From 9237490619c60fad45ed24964fa339d74ca3a939 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 11 Apr 2024 22:12:06 +0300 Subject: [PATCH 01/53] feat: download file in json format --- lean/components/api/data_client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lean/components/api/data_client.py b/lean/components/api/data_client.py index 60355fab..70767631 100644 --- a/lean/components/api/data_client.py +++ b/lean/components/api/data_client.py @@ -92,6 +92,14 @@ def download_public_file(self, data_endpoint: str) -> bytes: :return: the content of the file """ return self._http_client.get(data_endpoint).content + + def download_public_file_json(self, data_endpoint: str) -> Any: + """Downloads the content of a downloadable public file in json format. + + :param data_endpoint: the url of the public file + :return: the content of the file in json format + """ + return self._http_client.get(data_endpoint).json() def list_files(self, prefix: str) -> List[str]: """Lists all remote files with a given prefix. From 9f4526246efd277ceb8e2c408a1df030ac434af9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 11 Apr 2024 22:13:44 +0300 Subject: [PATCH 02/53] feat: download no-/interactive with new command data-provider-historical --- lean/commands/data/download.py | 119 +++++++++++++++++++++++++++------ 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 59209053..479a6388 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -12,13 +12,14 @@ # limitations under the License. from typing import Iterable, List, Optional -from click import command, option, confirm, pass_context, Context - +from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery -from lean.models.data import Dataset, DataFile, Product +from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product from lean.models.logger import Option +from lean.models.cli import cli_data_downloaders +from datetime import datetime _data_information: Optional[QCDataInformation] = None _presigned_terms=""" @@ -95,7 +96,6 @@ def _get_data_files(organization: QCFullOrganization, products: List[Product]) - unique_data_files = sorted(list(set(chain(*[product.get_data_files() for product in products])))) return _map_data_files_to_vendors(organization, unique_data_files) - def _display_products(organization: QCFullOrganization, products: List[Product]) -> None: """Previews a list of products in pretty tables. @@ -159,7 +159,6 @@ def _get_security_master_warn(url: str) -> str: f"You can add the subscription at https://www.quantconnect.com/datasets/{url}/pricing" ]) - def _select_products_interactive(organization: QCFullOrganization, datasets: List[Dataset], force: bool, ask_for_more_data: bool) -> List[Product]: """Asks the user for the products that should be purchased and downloaded. @@ -408,15 +407,45 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: return available_datasets - @command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True) +@option("--data-provider-historical", + type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), + help="The name of the downloader data provider.") @option("--dataset", type=str, help="The name of the dataset to download non-interactively") @option("--overwrite", is_flag=True, default=False, help="Overwrite existing local data") @option("--force", is_flag=True, default=False, hidden=True) @option("--yes", "-y", "auto_confirm", is_flag=True, default=False, help="Automatically confirm payment confirmation prompts") +@option("--historical-data-type", type=Choice(["Trade", "Quote", "OpenInterest"]), help="Specify the type of historical data") +@option("--historical-resolution", type=Choice(["Tick", "Second", "Minute", "Hour", "Daily"]), help="Specify the resolution of the historical data") +@option("--historical-ticker-security-type", type=Choice( + [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ]), + help="Specify the security type of the historical data") +@option("--historical-tickers", + type=str, + default="", + help="Specify comma separated list of tickers to use for historical data request.") +@option("--historical-start-date", + type=str, + help="Specify the start date for the historical data request in the format yyyyMMdd.") +@option("--historical-end-date", + type=str, + default=datetime.today().strftime("%Y%m%d"), + help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)") @pass_context -def download(ctx: Context, dataset: Optional[str], overwrite: bool, force: bool, auto_confirm: bool, **kwargs) -> None: +def download(ctx: Context, + dataset: Optional[str], + overwrite: bool, + force: bool, + auto_confirm: bool, + data_provider_historical: Optional[str], + historical_data_type: Optional[str], + historical_resolution: Optional[str], + historical_ticker_security_type: Optional[str], + historical_tickers: Optional[str], + historical_start_date: Optional[str], + historical_end_date: Optional[str], + **kwargs) -> None: """Purchase and download data from QuantConnect Datasets. An interactive wizard will show to walk you through the process of selecting data, @@ -433,20 +462,70 @@ def download(ctx: Context, dataset: Optional[str], overwrite: bool, force: bool, """ organization = _get_organization() - is_interactive = dataset is None - if not is_interactive: - ensure_options(["dataset"]) - datasets = _get_available_datasets(organization) - products = _select_products_non_interactive(organization, datasets, ctx, force) + if data_provider_historical is None: + data_provider_historical = _get_historical_data_providers() + + if data_provider_historical == 'QuantConnect': + is_interactive = dataset is None + if not is_interactive: + ensure_options(["dataset"]) + datasets = _get_available_datasets(organization) + products = _select_products_non_interactive(organization, datasets, ctx, force) + else: + datasets = _get_available_datasets(organization) + products = _select_products_interactive(organization, datasets, force, ask_for_more_data=not auto_confirm) + + _confirm_organization_balance(organization, products) + _verify_accept_agreement(organization, is_interactive) + + if is_interactive and not auto_confirm: + _confirm_payment(organization, products) + + all_data_files = _get_data_files(organization, products) + container.data_downloader.download_files(all_data_files, overwrite, organization.id) else: - datasets = _get_available_datasets(organization) - products = _select_products_interactive(organization, datasets, force, ask_for_more_data=not auto_confirm) + logger = container.logger - _confirm_organization_balance(organization, products) - _verify_accept_agreement(organization, is_interactive) + # download config by specific --data-provider-historical + # config_json = container.api_client.data.download_public_file_json("https://raw.githubusercontent.com/QuantConnect/Lean.DataSource.Polygon/master/polygon.json") + # logger.info(f'2configs {config_json["data-supported"]}') - if is_interactive and not auto_confirm: - _confirm_payment(organization, products) + # validate data_type exists + if not historical_data_type: + historical_data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in [ "Trade", "Quote", "OpenInterest" ]]) + + logger.info(f'data_type: {historical_data_type}') - all_data_files = _get_data_files(organization, products) - container.data_downloader.download_files(all_data_files, overwrite, organization.id) + if not historical_resolution: + historical_resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in ["Tick", "Second", "Minute", "Hour", "Daily"]]) + + logger.info(f'resolution: {historical_resolution}') + + if not historical_ticker_security_type: + historical_ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ]]) + + logger.info(f'historical_ticker_security_type: {historical_ticker_security_type}') + + if not historical_tickers: + historical_tickers = DatasetTextOption(id="id", + label="Enter comma separated list of tickers to use for historical data request.", + description="description", + transform=DatasetTextOptionTransform.Lowercase, + multiple=True).configure_interactive() + + logger.info(f'historical_tickers: {historical_tickers}') + + start_data_option = DatasetDateOption(id="start", label="Start date", description="Enter the start date for the historical data request in the format YYYYMMDD.") + if not historical_start_date: + historical_start_date = start_data_option.configure_interactive() + else: + historical_start_date = start_data_option.configure_non_interactive(historical_start_date) + + end_date_option = DatasetDateOption(id="end", label="End date",description="Enter the end date for the historical data request in the format YYYYMMDD.") + if not historical_end_date: + historical_end_date = end_date_option.configure_interactive() + else: + historical_end_date = end_date_option.configure_non_interactive(historical_end_date) + +def _get_historical_data_providers() -> str: + return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) \ No newline at end of file From 2c5598b60d89f9ce5d2b93579031ea40e2e65751 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 12 Apr 2024 19:23:38 +0300 Subject: [PATCH 03/53] feat: constant data types --- lean/constants.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lean/constants.py b/lean/constants.py index f2fe3482..7564f313 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -107,3 +107,15 @@ # platforms MODULE_CLI_PLATFORM = "cli" MODULE_CLOUD_PLATFORM = "cloud" + +# Lean Resolution +RESOLUTIONS = ["Tick", "Second", "Minute", "Hour", "Daily"] + +# Lean Resolution +RESOLUTIONS = ["Tick", "Second", "Minute", "Hour", "Daily"] + +# Lean Data Types +DATA_TYPES = ["Trade", "Quote", "OpenInterest"] + +# Lean Security Types +SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] \ No newline at end of file From 0fe8412b179f3b75e2d0a13168d76555177955c2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 12 Apr 2024 19:24:05 +0300 Subject: [PATCH 04/53] feat: new property in JsonModule --- lean/models/json_module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index dca7b556..ba0d6a43 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -37,6 +37,7 @@ def __init__(self, json_module_data: Dict[str, Any], module_type: str, platform: self._product_id: int = json_module_data["product-id"] if "product-id" in json_module_data else 0 self._id: str = json_module_data["id"] self._display_name: str = json_module_data["display-id"] + self._specifications_url: str = json_module_data["specifications"] if "specifications" in json_module_data else None self._installs: bool = json_module_data["installs"] if ("installs" in json_module_data and platform == MODULE_CLI_PLATFORM) else False self._lean_configs: List[Configuration] = [] From 6f01c936c732ba0eb5f85d67b043b8dd177fdda3 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 12 Apr 2024 19:25:07 +0300 Subject: [PATCH 05/53] feat: validate of no-/interactive in download command --- lean/commands/data/download.py | 61 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 479a6388..5dd2dc0a 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -14,12 +14,13 @@ from typing import Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options +from lean.components.util.json_modules_handler import config_build_for_name, non_interactive_config_build_for_name +from lean.constants import DATA_TYPES, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product from lean.models.logger import Option from lean.models.cli import cli_data_downloaders -from datetime import datetime _data_information: Optional[QCDataInformation] = None _presigned_terms=""" @@ -408,7 +409,7 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: return available_datasets @command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True) -@option("--data-provider-historical", +@option("--data-provider-historical", type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), help="The name of the downloader data provider.") @option("--dataset", type=str, help="The name of the dataset to download non-interactively") @@ -416,29 +417,26 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: @option("--force", is_flag=True, default=False, hidden=True) @option("--yes", "-y", "auto_confirm", is_flag=True, default=False, help="Automatically confirm payment confirmation prompts") -@option("--historical-data-type", type=Choice(["Trade", "Quote", "OpenInterest"]), help="Specify the type of historical data") -@option("--historical-resolution", type=Choice(["Tick", "Second", "Minute", "Hour", "Daily"]), help="Specify the resolution of the historical data") -@option("--historical-ticker-security-type", type=Choice( - [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ]), +@option("--historical-data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") +@option("--historical-resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") +@option("--historical-ticker-security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), help="Specify the security type of the historical data") @option("--historical-tickers", type=str, - default="", help="Specify comma separated list of tickers to use for historical data request.") @option("--historical-start-date", type=str, help="Specify the start date for the historical data request in the format yyyyMMdd.") @option("--historical-end-date", type=str, - default=datetime.today().strftime("%Y%m%d"), help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)") @pass_context def download(ctx: Context, + data_provider_historical: Optional[str], dataset: Optional[str], overwrite: bool, force: bool, auto_confirm: bool, - data_provider_historical: Optional[str], historical_data_type: Optional[str], historical_resolution: Optional[str], historical_ticker_security_type: Optional[str], @@ -463,7 +461,7 @@ def download(ctx: Context, organization = _get_organization() if data_provider_historical is None: - data_provider_historical = _get_historical_data_providers() + data_provider_historical = _get_historical_data_provider() if data_provider_historical == 'QuantConnect': is_interactive = dataset is None @@ -485,26 +483,28 @@ def download(ctx: Context, container.data_downloader.download_files(all_data_files, overwrite, organization.id) else: logger = container.logger + + data_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) - # download config by specific --data-provider-historical - # config_json = container.api_client.data.download_public_file_json("https://raw.githubusercontent.com/QuantConnect/Lean.DataSource.Polygon/master/polygon.json") - # logger.info(f'2configs {config_json["data-supported"]}') + data_provider_config_json = None + if data_provider._specifications_url is not None: + data_provider_config_json = container.api_client.data.download_public_file_json(data_provider._specifications_url) + data_provider_support_security_types = SECURITY_TYPES + if data_provider_config_json is not None: + data_provider_support_security_types = data_provider_config_json["data-supported"] + # validate data_type exists if not historical_data_type: - historical_data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in [ "Trade", "Quote", "OpenInterest" ]]) - - logger.info(f'data_type: {historical_data_type}') + historical_data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in DATA_TYPES]) if not historical_resolution: - historical_resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in ["Tick", "Second", "Minute", "Hour", "Daily"]]) - - logger.info(f'resolution: {historical_resolution}') + historical_resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in RESOLUTIONS]) if not historical_ticker_security_type: - historical_ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ]]) - - logger.info(f'historical_ticker_security_type: {historical_ticker_security_type}') + historical_ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in data_provider_support_security_types]) + elif historical_ticker_security_type not in data_provider_support_security_types: + raise ValueError(f"The {data_provider_historical} data provider does not support {historical_ticker_security_type}. Please choose a supported security type from: {data_provider_support_security_types}.") if not historical_tickers: historical_tickers = DatasetTextOption(id="id", @@ -512,8 +512,6 @@ def download(ctx: Context, description="description", transform=DatasetTextOptionTransform.Lowercase, multiple=True).configure_interactive() - - logger.info(f'historical_tickers: {historical_tickers}') start_data_option = DatasetDateOption(id="start", label="Start date", description="Enter the start date for the historical data request in the format YYYYMMDD.") if not historical_start_date: @@ -527,5 +525,18 @@ def download(ctx: Context, else: historical_end_date = end_date_option.configure_non_interactive(historical_end_date) -def _get_historical_data_providers() -> str: + if historical_start_date.value >= historical_end_date.value: + raise ValueError("Historical start date cannot be greater than or equal to historical end date.") + + lean_config = container.lean_config_manager.get_lean_config() + data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, True) + data_provider.ensure_module_installed(organization.id) + container.lean_config_manager.set_properties(data_provider.get_settings()) + # TODO: copy / paste to new Lean.proj + paths_to_mount = data_provider.get_paths_to_mount() + + lean_config_manager = container.lean_config_manager + data_dir = lean_config_manager.get_data_directory() + +def _get_historical_data_provider() -> str: return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) \ No newline at end of file From 8171b2f9ad51191da308cd14fdac4b50cec154c7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 13 Apr 2024 00:01:41 +0300 Subject: [PATCH 06/53] feat: constant data folder path --- lean/constants.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lean/constants.py b/lean/constants.py index 7564f313..b339c8a7 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -118,4 +118,7 @@ DATA_TYPES = ["Trade", "Quote", "OpenInterest"] # Lean Security Types -SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] \ No newline at end of file +SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] + +# Lean Data folder path, where keeps tickers data +DATA_FOLDER_PATH = "/Lean/Data" \ No newline at end of file From cc4f7bd44b76a859f1e4befe9ce597c9c2ab7ba1 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 13 Apr 2024 00:02:18 +0300 Subject: [PATCH 07/53] feat: entry point to new download project --- lean/commands/data/download.py | 47 ++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 5dd2dc0a..a4aefcfb 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -15,7 +15,7 @@ from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name, non_interactive_config_build_for_name -from lean.constants import DATA_TYPES, RESOLUTIONS, SECURITY_TYPES +from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product @@ -430,6 +430,13 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: @option("--historical-end-date", type=str, help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)") +@option("--image", + type=str, + help=f"The LEAN engine image to use (defaults to {DEFAULT_ENGINE_IMAGE})") +@option("--update", + is_flag=True, + default=False, + help="Pull the LEAN engine image before running the Downloader Data Provider") @pass_context def download(ctx: Context, data_provider_historical: Optional[str], @@ -443,6 +450,8 @@ def download(ctx: Context, historical_tickers: Optional[str], historical_start_date: Optional[str], historical_end_date: Optional[str], + image: Optional[str], + update: bool, **kwargs) -> None: """Purchase and download data from QuantConnect Datasets. @@ -532,11 +541,45 @@ def download(ctx: Context, data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, True) data_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_provider.get_settings()) - # TODO: copy / paste to new Lean.proj paths_to_mount = data_provider.get_paths_to_mount() + for key, value in paths_to_mount.items(): + logger.info(f'paths_to_mount {key}: {value}') + lean_config_manager = container.lean_config_manager data_dir = lean_config_manager.get_data_directory() + + entrypoint = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", + "--data-provider", data_provider.get_name(), + "--destination-dir", DATA_FOLDER_PATH, + "--data-type", historical_data_type, + "--start-date", historical_start_date.value.strftime("%Y%m%d"), + "--end-date", historical_end_date.value.strftime("%Y%m%d"), + "--security-type", historical_ticker_security_type, + "--resolution", historical_resolution, + "--tickers", historical_tickers] + + run_options = { + "entrypoint": entrypoint, + "volumes": { + str(data_dir): { + "bind": DATA_FOLDER_PATH, + "mode": "rw" + } + } + } + + engine_image = container.cli_config_manager.get_engine_image(image) + + logger.info(f'engine_image: {engine_image.name}') + + container.update_manager.pull_docker_image_if_necessary(engine_image, update) + + success = container.docker_manager.run_image(engine_image, **run_options) + + if not success: + raise RuntimeError( + "Something went wrong while running the downloader data provider, see the logs above for more information") def _get_historical_data_provider() -> str: return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) \ No newline at end of file From 6b3c15841b8219e9c07c2dfd2effba129afb97a3 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 15 Apr 2024 17:26:09 +0300 Subject: [PATCH 08/53] feat: working_dir in docker remove: not used path mount feat: const for download config key --- lean/commands/data/download.py | 9 +++------ lean/constants.py | 5 ++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index a4aefcfb..8684ca22 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -15,7 +15,7 @@ from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name, non_interactive_config_build_for_name -from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, RESOLUTIONS, SECURITY_TYPES +from lean.constants import DATA_FOLDER_PATH, DATA_PROVIDER_DOWNLOAD_CONFIG_KEY, DATA_TYPES, DEFAULT_ENGINE_IMAGE, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product @@ -541,16 +541,12 @@ def download(ctx: Context, data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, True) data_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_provider.get_settings()) - paths_to_mount = data_provider.get_paths_to_mount() - - for key, value in paths_to_mount.items(): - logger.info(f'paths_to_mount {key}: {value}') lean_config_manager = container.lean_config_manager data_dir = lean_config_manager.get_data_directory() entrypoint = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-provider", data_provider.get_name(), + "--data-provider", data_provider.get_config_value_from_name(DATA_PROVIDER_DOWNLOAD_CONFIG_KEY), "--destination-dir", DATA_FOLDER_PATH, "--data-type", historical_data_type, "--start-date", historical_start_date.value.strftime("%Y%m%d"), @@ -560,6 +556,7 @@ def download(ctx: Context, "--tickers", historical_tickers] run_options = { + "working_dir": "/Lean/DownloaderDataProvider/bin/Debug/", "entrypoint": entrypoint, "volumes": { str(data_dir): { diff --git a/lean/constants.py b/lean/constants.py index b339c8a7..41d08d2a 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -121,4 +121,7 @@ SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] # Lean Data folder path, where keeps tickers data -DATA_FOLDER_PATH = "/Lean/Data" \ No newline at end of file +DATA_FOLDER_PATH = "/Lean/Data" + +# Key in JSON configuration for data downloader settings. +DATA_PROVIDER_DOWNLOAD_CONFIG_KEY = "data-downloader" \ No newline at end of file From 76aa61a594dd1d794a44f8cbaf62e8c4ae8c586d Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 16 Apr 2024 00:39:58 +0300 Subject: [PATCH 09/53] feat: get full title with namespace remove: extra const --- lean/commands/data/download.py | 5 +++-- lean/constants.py | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 8684ca22..49d708c4 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -11,11 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path from typing import Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name, non_interactive_config_build_for_name -from lean.constants import DATA_FOLDER_PATH, DATA_PROVIDER_DOWNLOAD_CONFIG_KEY, DATA_TYPES, DEFAULT_ENGINE_IMAGE, RESOLUTIONS, SECURITY_TYPES +from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product @@ -546,7 +547,7 @@ def download(ctx: Context, data_dir = lean_config_manager.get_data_directory() entrypoint = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-provider", data_provider.get_config_value_from_name(DATA_PROVIDER_DOWNLOAD_CONFIG_KEY), + "--data-provider", data_provider.get_config_value_from_name(MODULE_DATA_DOWNLOADER), "--destination-dir", DATA_FOLDER_PATH, "--data-type", historical_data_type, "--start-date", historical_start_date.value.strftime("%Y%m%d"), diff --git a/lean/constants.py b/lean/constants.py index 41d08d2a..b339c8a7 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -121,7 +121,4 @@ SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] # Lean Data folder path, where keeps tickers data -DATA_FOLDER_PATH = "/Lean/Data" - -# Key in JSON configuration for data downloader settings. -DATA_PROVIDER_DOWNLOAD_CONFIG_KEY = "data-downloader" \ No newline at end of file +DATA_FOLDER_PATH = "/Lean/Data" \ No newline at end of file From 2fa5da01c8e7278b2674c84640ae6f5f235b8e36 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 17 Apr 2024 19:58:37 +0300 Subject: [PATCH 10/53] feat: get basic docker config without path refactor: param target_path in set up csharp options feat: run download project --- lean/commands/data/download.py | 50 +++---- lean/components/docker/lean_runner.py | 188 +++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 30 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 49d708c4..aef61b64 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -542,37 +542,37 @@ def download(ctx: Context, data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, True) data_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_provider.get_settings()) + # Info: I don't understand why it returns empty result + paths_to_mount = data_provider.get_paths_to_mount() - lean_config_manager = container.lean_config_manager - data_dir = lean_config_manager.get_data_directory() + engine_image = container.cli_config_manager.get_engine_image(image) - entrypoint = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-provider", data_provider.get_config_value_from_name(MODULE_DATA_DOWNLOADER), - "--destination-dir", DATA_FOLDER_PATH, - "--data-type", historical_data_type, - "--start-date", historical_start_date.value.strftime("%Y%m%d"), - "--end-date", historical_end_date.value.strftime("%Y%m%d"), - "--security-type", historical_ticker_security_type, - "--resolution", historical_resolution, - "--tickers", historical_tickers] + no_update = False + if str(engine_image) != DEFAULT_ENGINE_IMAGE: + # Custom engine image should not be updated. + no_update = True + logger.warn(f'A custom engine image: "{engine_image}" is being used!') + + container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) + + downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" - run_options = { - "working_dir": "/Lean/DownloaderDataProvider/bin/Debug/", - "entrypoint": entrypoint, - "volumes": { - str(data_dir): { - "bind": DATA_FOLDER_PATH, - "mode": "rw" - } - } - } + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, True, False, engine_image, downloader_data_provider_path_dll, paths_to_mount) - engine_image = container.cli_config_manager.get_engine_image(image) - - logger.info(f'engine_image: {engine_image.name}') + run_options["working_dir"] = downloader_data_provider_path_dll - container.update_manager.pull_docker_image_if_necessary(engine_image, update) + dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", + "--data-provider", data_provider.get_config_value_from_name(MODULE_DATA_DOWNLOADER), + "--destination-dir", DATA_FOLDER_PATH, + "--data-type", historical_data_type, + "--start-date", historical_start_date.value.strftime("%Y%m%d"), + "--end-date", historical_end_date.value.strftime("%Y%m%d"), + "--security-type", historical_ticker_security_type, + "--resolution", historical_resolution, + "--tickers", historical_tickers] + run_options["commands"].append(' '.join(dll_arguments)) + success = container.docker_manager.run_image(engine_image, **run_options) if not success: diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 4c247869..d994788f 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -375,6 +375,174 @@ def get_basic_docker_config(self, return run_options + def get_basic_docker_config_without_algo(self, + lean_config: Dict[str, Any], + debugging_method: Optional[DebuggingMethod], + detach: bool, + image: DockerImage, + target_path: str, + paths_to_mount: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """Creates a basic Docker config to run the engine with. + + This method constructs the parts of the Docker config that is the same for both the engine and the optimizer. + + :param lean_config: the LEAN configuration to use + :param debugging_method: the debugging method if debugging needs to be enabled, None if not + :param detach: whether LEAN should run in a detached container + :param image: The docker image that will be used + :return: the Docker configuration containing basic configuration to run Lean + :param paths_to_mount: additional paths to mount to the container + """ + from docker.types import Mount + from uuid import uuid4 + from json import dumps + + docker_project_config = { + "docker": {} + } + # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider + data_dir = self._lean_config_manager.get_data_directory() + if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": + self._force_disk_provider_if_necessary(lean_config, + "map-file-provider", + "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider", + "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider", + data_dir / "equity" / "usa" / "map_files") + self._force_disk_provider_if_necessary(lean_config, + "factor-file-provider", + "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider", + "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider", + data_dir / "equity" / "usa" / "factor_files") + + # Create the storage directory if it doesn't exist yet + storage_dir = self._lean_config_manager.get_cli_root_directory() / "storage" + if not storage_dir.exists(): + storage_dir.mkdir(parents=True) + + lean_config["debug-mode"] = self._logger.debug_logging_enabled + lean_config["data-folder"] = "/Lean/Data" + lean_config["results-destination-folder"] = "/Results" + lean_config["object-store-root"] = "/Storage" + + # The dict containing all options passed to `docker run` + # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html + run_options: Dict[str, Any] = { + "detach": detach, + "commands": [], + "environment": docker_project_config.get("environment", {}), + "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", + "mounts": [], + "volumes": {}, + "ports": docker_project_config.get("ports", {}) + } + + # mount the paths passed in + self.mount_paths(paths_to_mount, lean_config, run_options) + + # mount the project and library directories + #self.mount_project_and_library_directories(project_dir, run_options) + + # Mount the data directory + run_options["volumes"][str(data_dir)] = { + "bind": "/Lean/Data", + "mode": "rw" + } + + # Mount the local object store directory + run_options["volumes"][str(storage_dir)] = { + "bind": "/Storage", + "mode": "rw" + } + + # Mount all local files referenced in the Lean config + cli_root_dir = self._lean_config_manager.get_cli_root_directory() + files_to_mount = [ + ("transaction-log", cli_root_dir), + ("terminal-link-symbol-map-file", cli_root_dir / DEFAULT_DATA_DIRECTORY_NAME / "symbol-properties") + ] + for key, base_path in files_to_mount: + if key not in lean_config or lean_config[key] == "": + continue + + lean_config_entry = Path(lean_config[key]) + local_path = lean_config_entry if lean_config_entry.is_absolute() else base_path / lean_config_entry + if not local_path.exists(): + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.touch() + + run_options["mounts"].append(Mount(target=f"/Files/{key}", + source=str(local_path), + type="bind", + read_only=False)) + + lean_config[key] = f"/Files/{key}" + + # Update all hosts that need to point to the host's localhost to host.docker.internal so they resolve properly + for key in ["terminal-link-server-host"]: + if key not in lean_config: + continue + + if lean_config[key] == "localhost" or lean_config[key] == "127.0.0.1": + lean_config[key] = "host.docker.internal" + + # Set up modules + installed_packages = self._module_manager.get_installed_packages() + if len(installed_packages) > 0: + self._logger.debug(f"LeanRunner.run_lean(): installed packages {len(installed_packages)}") + self.set_up_common_csharp_options(run_options, target_path) + + # Mount the modules directory + run_options["volumes"][MODULES_DIRECTORY] = { + "bind": "/Modules", + "mode": "ro" + } + + # Add the modules directory as a NuGet source root + run_options["commands"].append("dotnet nuget add source /Modules") + + # Create a C# project used to resolve the dependencies of the modules + run_options["commands"].append("mkdir /ModulesProject") + run_options["commands"].append("dotnet new sln -o /ModulesProject") + + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', + DEFAULT_LEAN_DOTNET_FRAMEWORK) + run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") + run_options["commands"].append("rm /ModulesProject/Class1.cs") + + # Add all modules to the project, automatically resolving all dependencies + for package in installed_packages: + self._logger.debug(f"LeanRunner.run_lean(): Adding module {package} to the project") + run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") + run_options["commands"].append( + f"dotnet add /ModulesProject package {package.name} --version {package.version}") + + # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists + run_options["commands"].append( + "python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") + + # Save the final Lean config to a temporary file so we can mount it into the container + config_path = self._temp_manager.create_temporary_directory() / "config.json" + with config_path.open("w+", encoding="utf-8") as file: + file.write(dumps(lean_config, indent=4)) + + # Mount the Lean config + run_options["mounts"].append(Mount(target=f"{LEAN_ROOT_PATH}/config.json", + source=str(config_path), + type="bind", + read_only=True)) + + # Assign the container a name and store it in the output directory's configuration + if "container-name" in lean_config: + run_options["name"] = lean_config["container-name"] + else: + run_options["name"] = f"lean_cli_{str(uuid4()).replace('-', '')}" + + # set the hostname + if "hostname" in lean_config: + run_options["hostname"] = lean_config["hostname"] + + return run_options + def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None: """Sets up Docker run options specific to Python projects. @@ -547,12 +715,22 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], run_options["commands"].append( f'python /copy_csharp_dependencies.py "/Compile/obj/{project_file.stem}/project.assets.json"') - def set_up_common_csharp_options(self, run_options: Dict[str, Any]) -> None: - """Sets up common Docker run options that is needed for all C# work. + def set_up_common_csharp_options(self, run_options: Dict[str, Any], target_path: str = "/Lean/Launcher/bin/Debug") -> None: + """ + Sets up common Docker run options that is needed for all C# work. + + This method prepares the Docker run options required to run C# projects inside a Docker container. It is called + when the user has installed specific modules or when the project to run is written in C#. - This method is only called if the user has installed modules and/or if the project to run is written in C#. + Parameters: + - run_options (Dict[str, Any]): A dictionary to which the Docker run options will be appended. + - target_path (str, optional): The target path inside the Docker container where the C# project should be located. + Default value is "/Lean/Launcher/bin/Debug". + A Python script is typically used to copy the right C# dependencies to this path. + This script ensures that the correct DLLs are copied, even if they are OS-specific. - :param run_options: the dictionary to append run options to + Returns: + - None: This function does not return anything. It modifies the `run_options` dictionary in place. """ from docker.types import Mount # Mount a volume to NuGet's cache directory so we only download packages once @@ -623,7 +801,7 @@ def copy_file(library_id, partial_path, file_data): output_name = file_data.get("outputPath", full_path.name) - target_path = Path("/Lean/Launcher/bin/Debug") / output_name + target_path = Path("""+ f'"{target_path}"' +""") / output_name if not target_path.exists(): target_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy(full_path, target_path) From 97d44dbffe20f8398ee82e031dc7343395098308 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 17 Apr 2024 22:15:14 +0300 Subject: [PATCH 11/53] refactor: split up get_basic_docker_config on small parts --- lean/components/docker/lean_runner.py | 437 +++++++++++--------------- tests/commands/data/test_download.py | 64 +++- 2 files changed, 240 insertions(+), 261 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index d994788f..a793eb92 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -197,9 +197,6 @@ def get_basic_docker_config(self, :return: the Docker configuration containing basic configuration to run Lean :param paths_to_mount: additional paths to mount to the container """ - from docker.types import Mount - from uuid import uuid4 - from json import dumps project_dir = algorithm_file.parent project_config = self._project_config_manager.get_project_config(project_dir) @@ -207,92 +204,27 @@ def get_basic_docker_config(self, # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider data_dir = self._lean_config_manager.get_data_directory() - if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": - self._force_disk_provider_if_necessary(lean_config, - "map-file-provider", - "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider", - "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider", - data_dir / "equity" / "usa" / "map_files") - self._force_disk_provider_if_necessary(lean_config, - "factor-file-provider", - "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider", - "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider", - data_dir / "equity" / "usa" / "factor_files") + self._handle_data_providers(lean_config, data_dir) - # Create the output directory if it doesn't exist yet - if not output_dir.exists(): - output_dir.mkdir(parents=True) - - # Create the storage directory if it doesn't exist yet storage_dir = self._lean_config_manager.get_cli_root_directory() / "storage" - if not storage_dir.exists(): - storage_dir.mkdir(parents=True) - - lean_config["debug-mode"] = self._logger.debug_logging_enabled - lean_config["data-folder"] = "/Lean/Data" - lean_config["results-destination-folder"] = "/Results" - lean_config["object-store-root"] = "/Storage" - - # The dict containing all options passed to `docker run` - # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html - run_options: Dict[str, Any] = { - "detach": detach, - "commands": [], - "environment": docker_project_config.get("environment", {}), - "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", - "mounts": [], - "volumes": {}, - "ports": docker_project_config.get("ports", {}) - } - - # mount the paths passed in - self.mount_paths(paths_to_mount, lean_config, run_options) - - # mount the project and library directories - self.mount_project_and_library_directories(project_dir, run_options) - - # Mount the data directory - run_options["volumes"][str(data_dir)] = { - "bind": "/Lean/Data", - "mode": "rw" - } - - # Mount the output directory - run_options["volumes"][str(output_dir)] = { - "bind": "/Results", - "mode": "rw" - } - - # Mount the local object store directory - run_options["volumes"][str(storage_dir)] = { - "bind": "/Storage", - "mode": "rw" - } - - # Mount all local files referenced in the Lean config - cli_root_dir = self._lean_config_manager.get_cli_root_directory() - files_to_mount = [ - ("transaction-log", cli_root_dir), - ("terminal-link-symbol-map-file", cli_root_dir / DEFAULT_DATA_DIRECTORY_NAME / "symbol-properties") - ] - for key, base_path in files_to_mount: - if key not in lean_config or lean_config[key] == "": - continue - lean_config_entry = Path(lean_config[key]) - local_path = lean_config_entry if lean_config_entry.is_absolute() else base_path / lean_config_entry - if not local_path.exists(): - local_path.parent.mkdir(parents=True, exist_ok=True) - local_path.touch() + # Create the output directory if it doesn't exist yet + # Create the storage directory if it doesn't exist yet + self._ensure_directories_exist([output_dir, storage_dir]) + + lean_config.update({ + "debug-mode": self._logger.debug_logging_enabled, + "data-folder": "/Lean/Data", + "results-destination-folder": "/Results", + "object-store-root": "/Storage" + }) - run_options["mounts"].append(Mount(target=f"/Files/{key}", - source=str(local_path), - type="bind", - read_only=False)) + run_options = self._initialize_run_options(detach, docker_project_config, debugging_method) - lean_config[key] = f"/Files/{key}" + self._mount_common_directories(run_options, paths_to_mount, lean_config, data_dir, storage_dir, project_dir, output_dir) # Update all hosts that need to point to the host's localhost to host.docker.internal so they resolve properly + # TODO: we should remove it or add to config json for key in ["terminal-link-server-host"]: if key not in lean_config: continue @@ -300,78 +232,15 @@ def get_basic_docker_config(self, if lean_config[key] == "localhost" or lean_config[key] == "127.0.0.1": lean_config[key] = "host.docker.internal" - set_up_common_csharp_options_called = False - # Set up modules - installed_packages = self._module_manager.get_installed_packages() - if len(installed_packages) > 0: - self._logger.debug(f"LeanRunner.run_lean(): installed packages {len(installed_packages)}") - self.set_up_common_csharp_options(run_options) - set_up_common_csharp_options_called = True - - # Mount the modules directory - run_options["volumes"][MODULES_DIRECTORY] = { - "bind": "/Modules", - "mode": "ro" - } - - # Add the modules directory as a NuGet source root - run_options["commands"].append("dotnet nuget add source /Modules") - - # Create a C# project used to resolve the dependencies of the modules - run_options["commands"].append("mkdir /ModulesProject") - run_options["commands"].append("dotnet new sln -o /ModulesProject") - - framework_ver = self._docker_manager.get_image_label(image, 'target_framework', - DEFAULT_LEAN_DOTNET_FRAMEWORK) - run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") - run_options["commands"].append("rm /ModulesProject/Class1.cs") - - # Add all modules to the project, automatically resolving all dependencies - for package in installed_packages: - self._logger.debug(f"LeanRunner.run_lean(): Adding module {package} to the project") - run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") - run_options["commands"].append( - f"dotnet add /ModulesProject package {package.name} --version {package.version}") - - # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists - run_options["commands"].append( - "python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") + set_up_common_csharp_options_called = self._setup_installed_packages(run_options, image) # Set up language-specific run options self.setup_language_specific_run_options(run_options, project_dir, algorithm_file, set_up_common_csharp_options_called, release, image) - # Save the final Lean config to a temporary file so we can mount it into the container - config_path = self._temp_manager.create_temporary_directory() / "config.json" - with config_path.open("w+", encoding="utf-8") as file: - file.write(dumps(lean_config, indent=4)) - - # Mount the Lean config - run_options["mounts"].append(Mount(target=f"{LEAN_ROOT_PATH}/config.json", - source=str(config_path), - type="bind", - read_only=True)) - - # Assign the container a name and store it in the output directory's configuration - if "container-name" in lean_config: - run_options["name"] = lean_config["container-name"] - else: - run_options["name"] = f"lean_cli_{str(uuid4()).replace('-', '')}" - - # set the hostname - if "hostname" in lean_config: - run_options["hostname"] = lean_config["hostname"] - - output_config = self._output_config_manager.get_output_config(output_dir) - output_config.set("container", run_options["name"]) - if "backtest-name" in lean_config: - output_config.set("backtest-name", lean_config["backtest-name"]) - if "environment" in lean_config and "environments" in lean_config: - environment = lean_config["environments"][lean_config["environment"]] - if "live-mode-brokerage" in environment: - output_config.set("brokerage", environment["live-mode-brokerage"].split(".")[-1]) + self._mount_lean_config_and_finalize(run_options, lean_config, output_dir) return run_options @@ -393,91 +262,29 @@ def get_basic_docker_config_without_algo(self, :return: the Docker configuration containing basic configuration to run Lean :param paths_to_mount: additional paths to mount to the container """ - from docker.types import Mount - from uuid import uuid4 - from json import dumps - docker_project_config = { - "docker": {} - } + docker_project_config = { "docker": {} } # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider data_dir = self._lean_config_manager.get_data_directory() - if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": - self._force_disk_provider_if_necessary(lean_config, - "map-file-provider", - "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider", - "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider", - data_dir / "equity" / "usa" / "map_files") - self._force_disk_provider_if_necessary(lean_config, - "factor-file-provider", - "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider", - "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider", - data_dir / "equity" / "usa" / "factor_files") + self._handle_data_providers(lean_config, data_dir) # Create the storage directory if it doesn't exist yet storage_dir = self._lean_config_manager.get_cli_root_directory() / "storage" - if not storage_dir.exists(): - storage_dir.mkdir(parents=True) + self._ensure_directories_exist([storage_dir]) - lean_config["debug-mode"] = self._logger.debug_logging_enabled - lean_config["data-folder"] = "/Lean/Data" - lean_config["results-destination-folder"] = "/Results" - lean_config["object-store-root"] = "/Storage" + lean_config.update({ + "debug-mode": self._logger.debug_logging_enabled, + "data-folder": "/Lean/Data", + "results-destination-folder": "/Results", + "object-store-root": "/Storage" + }) - # The dict containing all options passed to `docker run` - # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html - run_options: Dict[str, Any] = { - "detach": detach, - "commands": [], - "environment": docker_project_config.get("environment", {}), - "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", - "mounts": [], - "volumes": {}, - "ports": docker_project_config.get("ports", {}) - } - - # mount the paths passed in - self.mount_paths(paths_to_mount, lean_config, run_options) - - # mount the project and library directories - #self.mount_project_and_library_directories(project_dir, run_options) - - # Mount the data directory - run_options["volumes"][str(data_dir)] = { - "bind": "/Lean/Data", - "mode": "rw" - } - - # Mount the local object store directory - run_options["volumes"][str(storage_dir)] = { - "bind": "/Storage", - "mode": "rw" - } - - # Mount all local files referenced in the Lean config - cli_root_dir = self._lean_config_manager.get_cli_root_directory() - files_to_mount = [ - ("transaction-log", cli_root_dir), - ("terminal-link-symbol-map-file", cli_root_dir / DEFAULT_DATA_DIRECTORY_NAME / "symbol-properties") - ] - for key, base_path in files_to_mount: - if key not in lean_config or lean_config[key] == "": - continue - - lean_config_entry = Path(lean_config[key]) - local_path = lean_config_entry if lean_config_entry.is_absolute() else base_path / lean_config_entry - if not local_path.exists(): - local_path.parent.mkdir(parents=True, exist_ok=True) - local_path.touch() + run_options = self._initialize_run_options(detach, docker_project_config, debugging_method) - run_options["mounts"].append(Mount(target=f"/Files/{key}", - source=str(local_path), - type="bind", - read_only=False)) - - lean_config[key] = f"/Files/{key}" + self._mount_common_directories(run_options, paths_to_mount, lean_config, data_dir, storage_dir, None, None) # Update all hosts that need to point to the host's localhost to host.docker.internal so they resolve properly + # TODO: we should remove it or add to config json for key in ["terminal-link-server-host"]: if key not in lean_config: continue @@ -486,62 +293,172 @@ def get_basic_docker_config_without_algo(self, lean_config[key] = "host.docker.internal" # Set up modules - installed_packages = self._module_manager.get_installed_packages() - if len(installed_packages) > 0: - self._logger.debug(f"LeanRunner.run_lean(): installed packages {len(installed_packages)}") - self.set_up_common_csharp_options(run_options, target_path) - - # Mount the modules directory - run_options["volumes"][MODULES_DIRECTORY] = { - "bind": "/Modules", - "mode": "ro" - } - - # Add the modules directory as a NuGet source root - run_options["commands"].append("dotnet nuget add source /Modules") + self._setup_installed_packages(run_options, image, target_path) - # Create a C# project used to resolve the dependencies of the modules - run_options["commands"].append("mkdir /ModulesProject") - run_options["commands"].append("dotnet new sln -o /ModulesProject") + self._mount_lean_config_and_finalize(run_options, lean_config, None) - framework_ver = self._docker_manager.get_image_label(image, 'target_framework', - DEFAULT_LEAN_DOTNET_FRAMEWORK) - run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") - run_options["commands"].append("rm /ModulesProject/Class1.cs") - - # Add all modules to the project, automatically resolving all dependencies - for package in installed_packages: - self._logger.debug(f"LeanRunner.run_lean(): Adding module {package} to the project") - run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") - run_options["commands"].append( - f"dotnet add /ModulesProject package {package.name} --version {package.version}") + return run_options + + def _mount_lean_config_and_finalize(self, run_options: Dict[str, Any], lean_config: Dict[str, Any], output_dir: Optional[Path]): + """Mounts Lean config and finalizes.""" + from docker.types import Mount + from uuid import uuid4 + from json import dumps - # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists - run_options["commands"].append( - "python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") - # Save the final Lean config to a temporary file so we can mount it into the container config_path = self._temp_manager.create_temporary_directory() / "config.json" with config_path.open("w+", encoding="utf-8") as file: file.write(dumps(lean_config, indent=4)) - + # Mount the Lean config run_options["mounts"].append(Mount(target=f"{LEAN_ROOT_PATH}/config.json", source=str(config_path), type="bind", read_only=True)) - + # Assign the container a name and store it in the output directory's configuration - if "container-name" in lean_config: - run_options["name"] = lean_config["container-name"] - else: - run_options["name"] = f"lean_cli_{str(uuid4()).replace('-', '')}" - + run_options["name"] = lean_config.get("container-name", f"lean_cli_{str(uuid4()).replace('-', '')}") + # set the hostname if "hostname" in lean_config: run_options["hostname"] = lean_config["hostname"] - return run_options + if output_dir: + output_config = self._output_config_manager.get_output_config(output_dir) + output_config.set("container", run_options["name"]) + if "backtest-name" in lean_config: + output_config.set("backtest-name", lean_config["backtest-name"]) + if "environment" in lean_config and "environments" in lean_config: + environment = lean_config["environments"][lean_config["environment"]] + if "live-mode-brokerage" in environment: + output_config.set("brokerage", environment["live-mode-brokerage"].split(".")[-1]) + + def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerImage, target_path: str = "/Lean/Launcher/bin/Debug"): + """Sets up installed packages.""" + installed_packages = self._module_manager.get_installed_packages() + if installed_packages: + self._logger.debug(f"LeanRunner._setup_installed_packages(): installed packages {len(installed_packages)}") + self.set_up_common_csharp_options(run_options, target_path) + + # Mount the modules directory + run_options["volumes"][MODULES_DIRECTORY] = {"bind": "/Modules", "mode": "ro"} + + # Add the modules directory as a NuGet source root + run_options["commands"].append("dotnet nuget add source /Modules") + # Create a C# project used to resolve the dependencies of the modules + run_options["commands"].append("mkdir /ModulesProject") + run_options["commands"].append("dotnet new sln -o /ModulesProject") + + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', DEFAULT_LEAN_DOTNET_FRAMEWORK) + run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") + run_options["commands"].append("rm /ModulesProject/Class1.cs") + + # Add all modules to the project, automatically resolving all dependencies + for package in installed_packages: + self._logger.debug(f"LeanRunner._setup_installed_packages(): Adding module {package} to the project") + run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") + run_options["commands"].append(f"dotnet add /ModulesProject package {package.name} --version {package.version}") + + # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists + run_options["commands"].append("python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") + + return bool(installed_packages) + + def _mount_common_directories(self, + run_options: Dict[str, Any], + paths_to_mount: Optional[Dict[str, str]], + lean_config: Dict[str, Any], + data_dir: Path, + storage_dir: Path, + project_dir: Optional[Path], + output_dir: Optional[Path]): + """ + Mounts common directories. + + 1. mount the paths passed in + 2. mount the project and library directories (param: `project_dir` is not None) + 3. mount the data directory + 4. mount the output directory (param: `output_dir` is not None) + 5. mount the local object store directory + 6. mount all local files referenced in the Lean config + """ + from docker.types import Mount + + # 1 + self.mount_paths(paths_to_mount, lean_config, run_options) + + # 2 + if project_dir: + self.mount_project_and_library_directories(project_dir, run_options) + + # 3 + run_options["volumes"][str(data_dir)] = {"bind": "/Lean/Data", "mode": "rw"} + + # 4 + if output_dir: + run_options["volumes"][str(output_dir)] = {"bind": "/Results", "mode": "rw"} + # 5 + run_options["volumes"][str(storage_dir)] = { "bind": "/Storage", "mode": "rw"} + + # 6 + cli_root_dir = self._lean_config_manager.get_cli_root_directory() + files_to_mount = [ + ("transaction-log", cli_root_dir), + ("terminal-link-symbol-map-file", cli_root_dir / DEFAULT_DATA_DIRECTORY_NAME / "symbol-properties") + ] + for key, base_path in files_to_mount: + if key not in lean_config or lean_config[key] == "": + continue + + lean_config_entry = Path(lean_config[key]) + local_path = lean_config_entry if lean_config_entry.is_absolute() else base_path / lean_config_entry + if not local_path.exists(): + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.touch() + + run_options["mounts"].append(Mount(target=f"/Files/{key}", + source=str(local_path), + type="bind", + read_only=False)) + + lean_config[key] = f"/Files/{key}" + + def _initialize_run_options(self, detach: bool, docker_project_config: Dict[str, Any], debugging_method: Optional[DebuggingMethod]): + """ + Initializes run options. + + The dict containing all options passed to `docker run` + See all available options at https://docker-py.readthedocs.io/en/stable/containers.html + """ + return { + "detach": detach, + "commands": [], + "environment": docker_project_config.get("environment", {}), + "stop_signal": "SIGINT" if debugging_method is None else "SIGKILL", + "mounts": [], + "volumes": {}, + "ports": docker_project_config.get("ports", {}) + } + + def _ensure_directories_exist(self, dirs: List[Path]): + """Ensures directories exist.""" + for dir_path in dirs: + if not dir_path.exists(): + dir_path.mkdir(parents=True) + + def _handle_data_providers(self, lean_config: Dict[str, Any], data_dir: Path): + """Handles data provider logic.""" + if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": + self._force_disk_provider_if_necessary(lean_config, + "map-file-provider", + "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider", + "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider", + data_dir / "equity" / "usa" / "map_files") + self._force_disk_provider_if_necessary(lean_config, + "factor-file-provider", + "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider", + "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider", + data_dir / "equity" / "usa" / "factor_files") def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None: """Sets up Docker run options specific to Python projects. diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index be34f851..aeca36cf 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -1,16 +1,22 @@ import json from unittest import mock -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock +import click import pytest import os import re from pathlib import Path from lean.commands.data.download import * +from lean.commands.data.download import _select_products_non_interactive from lean.container import container from lean.models.api import QCDataset, QCOrganizationCredit, QCOrganizationData from tests.test_helpers import create_api_organization +from click.testing import CliRunner +from lean.commands import lean +from tests.test_helpers import create_fake_lean_cli_directory +from tests.conftest import initialize_container test_files = Path(os.path.join(os.path.dirname(os.path.realpath(__file__)), "testFiles")) @@ -31,6 +37,62 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) +def test_invoke_interactive(): + create_fake_lean_cli_directory() + result = CliRunner().invoke(lean, ["data", "download", "--data-provider-historical", "Binance"]) + + assert result.exit_code == 0 + +def test_select_products_non_interactive(): + organization = create_api_organization() + datasource = json.loads(bulk_datasource) + testDataSet = [Dataset(name="US Equity Options", + vendor="testVendor", + categories=["testData"], + options=datasource["options"], + paths=datasource["paths"], + requirements=datasource.get("requirements", {}))] + force = True + + + mock_context = Mock() + mock_context.params = {"dataset": "US Equity Options", "data-type": "Trade", "ticker": "AAPL", "resolution": "Daily", "start": "20240101", "end": "20240404"} + + products = _select_products_non_interactive(organization, testDataSet, mock_context, force) + + assert products + + # mocking + create_fake_lean_cli_directory() + + api_client = mock.MagicMock() + + test_datasets = [ + QCDataset(id=1, name="Pending Dataset", delivery=QCDatasetDelivery.DownloadOnly, vendorName="Vendor", tags=[], pending=True), + QCDataset(id=2, name="Non-Pending Dataset", delivery=QCDatasetDelivery.DownloadOnly, vendorName="Vendor", tags=[], pending=False) + ] + api_client.list_datasets = MagicMock(return_value=test_datasets) + container.api_client.market = api_client + container.api_client.organizations.get.return_value = create_api_organization() + + datasources = { + str(test_datasets[0].id): { + 'options': [], + 'paths': [], + 'requirements': {}, + }, + str(test_datasets[1].id): { + 'options': [], + 'paths': [], + 'requirements': {}, + } + } + container.api_client.data.get_info = MagicMock(return_value=QCDataInformation(datasources=datasources, prices=[], agreement="")) + + # initialize_container(api_client_to_use=api_client) + result = CliRunner().invoke(lean, ["data", "download", "--dataset", "US Equity Options"]) + + assert result.exit_code == 0 def test_non_interactive_bulk_select(): # TODO From 044e8c4c0ca350e4e1258e3b0c9d0c2fe7a7b37c Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 17 Apr 2024 22:18:43 +0300 Subject: [PATCH 12/53] revert: test_downloader file --- tests/commands/data/test_download.py | 64 +--------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index aeca36cf..be34f851 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -1,22 +1,16 @@ import json from unittest import mock -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock -import click import pytest import os import re from pathlib import Path from lean.commands.data.download import * -from lean.commands.data.download import _select_products_non_interactive from lean.container import container from lean.models.api import QCDataset, QCOrganizationCredit, QCOrganizationData from tests.test_helpers import create_api_organization -from click.testing import CliRunner -from lean.commands import lean -from tests.test_helpers import create_fake_lean_cli_directory -from tests.conftest import initialize_container test_files = Path(os.path.join(os.path.dirname(os.path.realpath(__file__)), "testFiles")) @@ -37,62 +31,6 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) -def test_invoke_interactive(): - create_fake_lean_cli_directory() - result = CliRunner().invoke(lean, ["data", "download", "--data-provider-historical", "Binance"]) - - assert result.exit_code == 0 - -def test_select_products_non_interactive(): - organization = create_api_organization() - datasource = json.loads(bulk_datasource) - testDataSet = [Dataset(name="US Equity Options", - vendor="testVendor", - categories=["testData"], - options=datasource["options"], - paths=datasource["paths"], - requirements=datasource.get("requirements", {}))] - force = True - - - mock_context = Mock() - mock_context.params = {"dataset": "US Equity Options", "data-type": "Trade", "ticker": "AAPL", "resolution": "Daily", "start": "20240101", "end": "20240404"} - - products = _select_products_non_interactive(organization, testDataSet, mock_context, force) - - assert products - - # mocking - create_fake_lean_cli_directory() - - api_client = mock.MagicMock() - - test_datasets = [ - QCDataset(id=1, name="Pending Dataset", delivery=QCDatasetDelivery.DownloadOnly, vendorName="Vendor", tags=[], pending=True), - QCDataset(id=2, name="Non-Pending Dataset", delivery=QCDatasetDelivery.DownloadOnly, vendorName="Vendor", tags=[], pending=False) - ] - api_client.list_datasets = MagicMock(return_value=test_datasets) - container.api_client.market = api_client - container.api_client.organizations.get.return_value = create_api_organization() - - datasources = { - str(test_datasets[0].id): { - 'options': [], - 'paths': [], - 'requirements': {}, - }, - str(test_datasets[1].id): { - 'options': [], - 'paths': [], - 'requirements': {}, - } - } - container.api_client.data.get_info = MagicMock(return_value=QCDataInformation(datasources=datasources, prices=[], agreement="")) - - # initialize_container(api_client_to_use=api_client) - result = CliRunner().invoke(lean, ["data", "download", "--dataset", "US Equity Options"]) - - assert result.exit_code == 0 def test_non_interactive_bulk_select(): # TODO From 1e9793088e0d4e1f932605b965df8216ac237c55 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 00:25:05 +0300 Subject: [PATCH 13/53] feat: support additional config in command line when user run `data download` --- lean/commands/data/download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index aef61b64..52259f37 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -19,6 +19,7 @@ from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery +from lean.models.click_options import get_configs_for_options, options_from_json from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product from lean.models.logger import Option from lean.models.cli import cli_data_downloaders @@ -409,10 +410,11 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: return available_datasets -@command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True) +@command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True, name="download") @option("--data-provider-historical", type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), help="The name of the downloader data provider.") +@options_from_json(get_configs_for_options("backtest")) @option("--dataset", type=str, help="The name of the dataset to download non-interactively") @option("--overwrite", is_flag=True, default=False, help="Overwrite existing local data") @option("--force", is_flag=True, default=False, hidden=True) From 26f96f6efe1bce075a105abbdc7b0430649758e2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 00:26:14 +0300 Subject: [PATCH 14/53] rename: remove historical prefix in command names --- lean/commands/data/download.py | 70 +++++++++++++++++----------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 52259f37..0ad93a1f 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -420,17 +420,17 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: @option("--force", is_flag=True, default=False, hidden=True) @option("--yes", "-y", "auto_confirm", is_flag=True, default=False, help="Automatically confirm payment confirmation prompts") -@option("--historical-data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") -@option("--historical-resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") -@option("--historical-ticker-security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), +@option("--data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") +@option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") +@option("--ticker-security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), help="Specify the security type of the historical data") -@option("--historical-tickers", +@option("--tickers", type=str, help="Specify comma separated list of tickers to use for historical data request.") -@option("--historical-start-date", +@option("--start-date", type=str, help="Specify the start date for the historical data request in the format yyyyMMdd.") -@option("--historical-end-date", +@option("--end-date", type=str, help="Specify the end date for the historical data request in the format yyyyMMdd. (defaults to today)") @option("--image", @@ -447,12 +447,12 @@ def download(ctx: Context, overwrite: bool, force: bool, auto_confirm: bool, - historical_data_type: Optional[str], - historical_resolution: Optional[str], - historical_ticker_security_type: Optional[str], - historical_tickers: Optional[str], - historical_start_date: Optional[str], - historical_end_date: Optional[str], + data_type: Optional[str], + resolution: Optional[str], + ticker_security_type: Optional[str], + tickers: Optional[str], + start_date: Optional[str], + end_date: Optional[str], image: Optional[str], update: bool, **kwargs) -> None: @@ -507,37 +507,37 @@ def download(ctx: Context, data_provider_support_security_types = data_provider_config_json["data-supported"] # validate data_type exists - if not historical_data_type: - historical_data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in DATA_TYPES]) + if not data_type: + data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in DATA_TYPES]) - if not historical_resolution: - historical_resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in RESOLUTIONS]) + if not resolution: + resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in RESOLUTIONS]) - if not historical_ticker_security_type: - historical_ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in data_provider_support_security_types]) - elif historical_ticker_security_type not in data_provider_support_security_types: - raise ValueError(f"The {data_provider_historical} data provider does not support {historical_ticker_security_type}. Please choose a supported security type from: {data_provider_support_security_types}.") + if not ticker_security_type: + ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in data_provider_support_security_types]) + elif ticker_security_type not in data_provider_support_security_types: + raise ValueError(f"The {data_provider_historical} data provider does not support {ticker_security_type}. Please choose a supported security type from: {data_provider_support_security_types}.") - if not historical_tickers: - historical_tickers = DatasetTextOption(id="id", + if not tickers: + tickers = DatasetTextOption(id="id", label="Enter comma separated list of tickers to use for historical data request.", description="description", transform=DatasetTextOptionTransform.Lowercase, multiple=True).configure_interactive() start_data_option = DatasetDateOption(id="start", label="Start date", description="Enter the start date for the historical data request in the format YYYYMMDD.") - if not historical_start_date: - historical_start_date = start_data_option.configure_interactive() + if not start_date: + start_date = start_data_option.configure_interactive() else: - historical_start_date = start_data_option.configure_non_interactive(historical_start_date) + start_date = start_data_option.configure_non_interactive(start_date) end_date_option = DatasetDateOption(id="end", label="End date",description="Enter the end date for the historical data request in the format YYYYMMDD.") - if not historical_end_date: - historical_end_date = end_date_option.configure_interactive() + if not end_date: + end_date = end_date_option.configure_interactive() else: - historical_end_date = end_date_option.configure_non_interactive(historical_end_date) + end_date = end_date_option.configure_non_interactive(end_date) - if historical_start_date.value >= historical_end_date.value: + if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") lean_config = container.lean_config_manager.get_lean_config() @@ -566,12 +566,12 @@ def download(ctx: Context, dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", "--data-provider", data_provider.get_config_value_from_name(MODULE_DATA_DOWNLOADER), "--destination-dir", DATA_FOLDER_PATH, - "--data-type", historical_data_type, - "--start-date", historical_start_date.value.strftime("%Y%m%d"), - "--end-date", historical_end_date.value.strftime("%Y%m%d"), - "--security-type", historical_ticker_security_type, - "--resolution", historical_resolution, - "--tickers", historical_tickers] + "--data-type", data_type, + "--start-date", start_date.value.strftime("%Y%m%d"), + "--end-date", end_date.value.strftime("%Y%m%d"), + "--security-type", ticker_security_type, + "--resolution", resolution, + "--tickers", tickers] run_options["commands"].append(' '.join(dll_arguments)) From 89559b8233f9d98e825596b70355747ccc62e1aa Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 14:53:29 +0300 Subject: [PATCH 15/53] feat: download run test --- tests/commands/data/test_download.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index be34f851..67937dc9 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -11,6 +11,10 @@ from lean.container import container from lean.models.api import QCDataset, QCOrganizationCredit, QCOrganizationData from tests.test_helpers import create_api_organization +from click.testing import CliRunner +from lean.commands import lean +from tests.test_helpers import create_fake_lean_cli_directory +from tests.conftest import initialize_container test_files = Path(os.path.join(os.path.dirname(os.path.realpath(__file__)), "testFiles")) @@ -31,6 +35,43 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) +def test_download_data_non_interactive(): + create_fake_lean_cli_directory() + + container = initialize_container() + + with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): + result = CliRunner().invoke(lean, ["data", "download", + "--data-provider-historical", "Polygon", + "--polygon-api-key", "123", + "--data-type", "Trade", + "--resolution", "Minute", + "--ticker-security-type", "Equity", + "--tickers", "AAPL", + "--start-date", "20240101", + "--end-date", "20240202"]) + + assert result.exit_code == 0 + +def test_download_data_non_interactive_data_provider_missed_param(): + create_fake_lean_cli_directory() + + container = initialize_container() + + with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): + result = CliRunner().invoke(lean, ["data", "download", + "--data-provider-historical", "Polygon", + "--data-type", "Trade", + "--resolution", "Minute", + "--ticker-security-type", "Equity", + "--tickers", "AAPL", + "--start-date", "20240101", + "--end-date", "20240202"]) + + assert result.exit_code == 1 + + error_msg = str(result.exc_info[1]) + assert "--polygon-api-key" in error_msg def test_non_interactive_bulk_select(): # TODO From 2ea806f4d9f335af88e7cb681b6a567e4e9ff29a Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 14:54:14 +0300 Subject: [PATCH 16/53] refactor: switch off interactive mode for `data-provider` param --- lean/commands/data/download.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 0ad93a1f..4c646edf 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -541,7 +541,7 @@ def download(ctx: Context, raise ValueError("Historical start date cannot be greater than or equal to historical end date.") lean_config = container.lean_config_manager.get_lean_config() - data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, True) + data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_provider.get_settings()) # Info: I don't understand why it returns empty result @@ -559,7 +559,12 @@ def download(ctx: Context, downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" - run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, True, False, engine_image, downloader_data_provider_path_dll, paths_to_mount) + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, + debugging_method=None, + detach=False, + image=engine_image, + target_path=downloader_data_provider_path_dll, + paths_to_mount=paths_to_mount) run_options["working_dir"] = downloader_data_provider_path_dll From 4474502649ec80a761e025cfd8e6bbeea8c666cd Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 20:54:43 +0300 Subject: [PATCH 17/53] remove: extra resolution --- lean/constants.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lean/constants.py b/lean/constants.py index b339c8a7..0219369e 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -111,9 +111,6 @@ # Lean Resolution RESOLUTIONS = ["Tick", "Second", "Minute", "Hour", "Daily"] -# Lean Resolution -RESOLUTIONS = ["Tick", "Second", "Minute", "Hour", "Daily"] - # Lean Data Types DATA_TYPES = ["Trade", "Quote", "OpenInterest"] From 752f24204efae1367df7fa3d12871789b5658caf Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 20:56:21 +0300 Subject: [PATCH 18/53] refactor: download json return like Dict[str, Any] --- lean/components/api/data_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lean/components/api/data_client.py b/lean/components/api/data_client.py index 70767631..da50fb93 100644 --- a/lean/components/api/data_client.py +++ b/lean/components/api/data_client.py @@ -13,7 +13,7 @@ from lean.components.api.api_client import * from lean.models.api import QCDataInformation -from typing import List, Callable +from typing import List, Callable, cast class DataClient: @@ -93,13 +93,13 @@ def download_public_file(self, data_endpoint: str) -> bytes: """ return self._http_client.get(data_endpoint).content - def download_public_file_json(self, data_endpoint: str) -> Any: + def download_public_file_json(self, data_endpoint: str) -> Dict[str, Any]: """Downloads the content of a downloadable public file in json format. :param data_endpoint: the url of the public file :return: the content of the file in json format """ - return self._http_client.get(data_endpoint).json() + return cast(Dict[str, Any], self._http_client.get(data_endpoint).json()) def list_files(self, prefix: str) -> List[str]: """Lists all remote files with a given prefix. From 0432392fb2236da81e33dacde4c41a97b26b9425 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 20:57:18 +0300 Subject: [PATCH 19/53] refactor: no-/interactive data download params --- lean/commands/data/download.py | 129 ++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 33 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 4c646edf..250bc564 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -11,11 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path -from typing import Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options -from lean.components.util.json_modules_handler import config_build_for_name, non_interactive_config_build_for_name +from lean.components.util.json_modules_handler import config_build_for_name from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery @@ -502,40 +501,23 @@ def download(ctx: Context, if data_provider._specifications_url is not None: data_provider_config_json = container.api_client.data.download_public_file_json(data_provider._specifications_url) - data_provider_support_security_types = SECURITY_TYPES - if data_provider_config_json is not None: - data_provider_support_security_types = data_provider_config_json["data-supported"] - - # validate data_type exists - if not data_type: - data_type = logger.prompt_list("Select a historical data type", [Option(id=data_type, label=data_type) for data_type in DATA_TYPES]) - - if not resolution: - resolution = logger.prompt_list("Select a historical resolution", [Option(id=data_type, label=data_type) for data_type in RESOLUTIONS]) + data_provider_support_security_types = _get_param_from_config(data_provider_config_json, SECURITY_TYPES, "data-supported") + data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") + data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") - if not ticker_security_type: - ticker_security_type = logger.prompt_list("Select a Ticker's security type", [Option(id=data_type, label=data_type) for data_type in data_provider_support_security_types]) - elif ticker_security_type not in data_provider_support_security_types: - raise ValueError(f"The {data_provider_historical} data provider does not support {ticker_security_type}. Please choose a supported security type from: {data_provider_support_security_types}.") + ticker_security_type = get_user_input_or_prompt(ticker_security_type, data_provider_support_security_types, data_provider_historical) + data_type = get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical) + resolution = get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical) if not tickers: - tickers = DatasetTextOption(id="id", + tickers = ','.join(DatasetTextOption(id="id", label="Enter comma separated list of tickers to use for historical data request.", description="description", transform=DatasetTextOptionTransform.Lowercase, - multiple=True).configure_interactive() - - start_data_option = DatasetDateOption(id="start", label="Start date", description="Enter the start date for the historical data request in the format YYYYMMDD.") - if not start_date: - start_date = start_data_option.configure_interactive() - else: - start_date = start_data_option.configure_non_interactive(start_date) - - end_date_option = DatasetDateOption(id="end", label="End date",description="Enter the end date for the historical data request in the format YYYYMMDD.") - if not end_date: - end_date = end_date_option.configure_interactive() - else: - end_date = end_date_option.configure_non_interactive(end_date) + multiple=True).configure_interactive().value) + + start_date = configure_date_option(start_date, "start", "Start date") + end_date = configure_date_option(end_date, "end", "End date") if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") @@ -578,7 +560,7 @@ def download(ctx: Context, "--resolution", resolution, "--tickers", tickers] - run_options["commands"].append(' '.join(dll_arguments)) + run_options["commands"].append(' '.join(dll_arguments)) success = container.docker_manager.run_image(engine_image, **run_options) @@ -587,4 +569,85 @@ def download(ctx: Context, "Something went wrong while running the downloader data provider, see the logs above for more information") def _get_historical_data_provider() -> str: - return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) \ No newline at end of file + return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) + +def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], key_config_data: str) -> List[str]: + """ + Get parameter from data provider config JSON or return default parameters. + + Args: + - data_provider_config_json (Dict[str, Any]): Configuration JSON. + - default_param (List[str]): Default parameters. + - key_config_data (str): Key to look for in the config JSON. + + Returns: + - List[str]: List of parameters. + """ + container.logger.info(f'data_provider_config_json: {data_provider_config_json}') + if data_provider_config_json is None: + return default_param + + return data_provider_config_json.get(key_config_data, default_param) + +def get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str) -> str: + """ + Get user input or prompt for selection based on data types. + + Args: + - user_input_data (str): User input data. + - data_types (List[str]): List of supported data types. + - data_provider_name (str): Name of the data provider. + + Returns: + - str: Selected data type or prompted choice. + + Raises: + - ValueError: If user input data is not in supported data types. + """ + + if not user_input_data: + # Prompt user to select a ticker's security type + return prompt_user_selection(data_types) + + elif user_input_data not in data_types: + # Raise ValueError for unsupported data type + raise ValueError( + f"The {data_provider_name} data provider does not support {user_input_data}. " + f"Please choose a supported data from: {data_types}." + ) + + return user_input_data + +def prompt_user_selection(data_types: List[str]) -> str: + """ + Prompt user to select a data type from a list. + + Args: + - data_types (List[str]): List of supported data types. + + Returns: + - str: Selected data type. + """ + + options = [Option(id=data_type, label=data_type) for data_type in data_types] + return container.logger.prompt_list("Select a Ticker's security type", options) + +def configure_date_option(date_value: str, option_id: str, option_label: str) -> str: + """ + Configure the date based on the provided date value, option ID, and option label. + + Args: + - date_value (str): Existing date value. + - option_id (str): Identifier for the date option. + - option_label (str): Label for the date option. + + Returns: + - str: Configured date. + """ + + date_option = DatasetDateOption(id=option_id, label=option_label, description=f"Enter the {option_label} for the historical data request in the format YYYYMMDD.") + + if not date_value: + return date_option.configure_interactive() + + return date_option.configure_non_interactive(date_value) \ No newline at end of file From 6cded4b2dda8468b00eadf6f26e7a1c027724a8f Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 20:57:59 +0300 Subject: [PATCH 20/53] feat: test mock REST get request --- tests/commands/data/test_download.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 67937dc9..5b78de4a 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -41,7 +41,8 @@ def test_download_data_non_interactive(): container = initialize_container() with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): - result = CliRunner().invoke(lean, ["data", "download", + with mock.patch.object(container.api_client.data, "download_public_file_json", return_value={ "data-supported" : [ "Equity", "Equity Options", "Indexes", "Index Options" ] }): + result = CliRunner().invoke(lean, ["data", "download", "--data-provider-historical", "Polygon", "--polygon-api-key", "123", "--data-type", "Trade", @@ -58,15 +59,19 @@ def test_download_data_non_interactive_data_provider_missed_param(): container = initialize_container() + # with mock.patch.object(container.api_client.projects, 'get_all', return_value=cloud_projects) as mock_get_all,\ + # mock.patch.object(container.api_client.projects, 'delete', return_value=None) as mock_delete: + with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): - result = CliRunner().invoke(lean, ["data", "download", - "--data-provider-historical", "Polygon", - "--data-type", "Trade", - "--resolution", "Minute", - "--ticker-security-type", "Equity", - "--tickers", "AAPL", - "--start-date", "20240101", - "--end-date", "20240202"]) + with mock.patch.object(container.api_client.data, "download_public_file_json", return_value={ "data-supported" : [ "Equity", "Equity Options", "Indexes", "Index Options" ] }): + result = CliRunner().invoke(lean, ["data", "download", + "--data-provider-historical", "Polygon", + "--data-type", "Trade", + "--resolution", "Minute", + "--ticker-security-type", "Equity", + "--tickers", "AAPL", + "--start-date", "20240101", + "--end-date", "20240202"]) assert result.exit_code == 1 From ac482d2a30181c0727e53ddb37bd389472fa0215 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 21:29:20 +0300 Subject: [PATCH 21/53] refactor: put local methods above main method rename: local methods --- lean/commands/data/download.py | 180 ++++++++++++++++----------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 250bc564..c7cc0f84 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -409,6 +409,90 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: return available_datasets +def _get_historical_data_provider() -> str: + return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) + +def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], key_config_data: str) -> List[str]: + """ + Get parameter from data provider config JSON or return default parameters. + + Args: + - data_provider_config_json (Dict[str, Any]): Configuration JSON. + - default_param (List[str]): Default parameters. + - key_config_data (str): Key to look for in the config JSON. + + Returns: + - List[str]: List of parameters. + """ + container.logger.info(f'data_provider_config_json: {data_provider_config_json}') + if data_provider_config_json is None: + return default_param + + return data_provider_config_json.get(key_config_data, default_param) + +def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str) -> str: + """ + Get user input or prompt for selection based on data types. + + Args: + - user_input_data (str): User input data. + - data_types (List[str]): List of supported data types. + - data_provider_name (str): Name of the data provider. + + Returns: + - str: Selected data type or prompted choice. + + Raises: + - ValueError: If user input data is not in supported data types. + """ + + if not user_input_data: + # Prompt user to select a ticker's security type + return _prompt_user_selection(data_types) + + elif user_input_data not in data_types: + # Raise ValueError for unsupported data type + raise ValueError( + f"The {data_provider_name} data provider does not support {user_input_data}. " + f"Please choose a supported data from: {data_types}." + ) + + return user_input_data + +def _prompt_user_selection(data_types: List[str]) -> str: + """ + Prompt user to select a data type from a list. + + Args: + - data_types (List[str]): List of supported data types. + + Returns: + - str: Selected data type. + """ + + options = [Option(id=data_type, label=data_type) for data_type in data_types] + return container.logger.prompt_list("Select a Ticker's security type", options) + +def _configure_date_option(date_value: str, option_id: str, option_label: str) -> str: + """ + Configure the date based on the provided date value, option ID, and option label. + + Args: + - date_value (str): Existing date value. + - option_id (str): Identifier for the date option. + - option_label (str): Label for the date option. + + Returns: + - str: Configured date. + """ + + date_option = DatasetDateOption(id=option_id, label=option_label, description=f"Enter the {option_label} for the historical data request in the format YYYYMMDD.") + + if not date_value: + return date_option.configure_interactive() + + return date_option.configure_non_interactive(date_value) + @command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True, name="download") @option("--data-provider-historical", type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), @@ -505,9 +589,9 @@ def download(ctx: Context, data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") - ticker_security_type = get_user_input_or_prompt(ticker_security_type, data_provider_support_security_types, data_provider_historical) - data_type = get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical) - resolution = get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical) + ticker_security_type = _get_user_input_or_prompt(ticker_security_type, data_provider_support_security_types, data_provider_historical) + data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical) + resolution = _get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical) if not tickers: tickers = ','.join(DatasetTextOption(id="id", @@ -516,8 +600,8 @@ def download(ctx: Context, transform=DatasetTextOptionTransform.Lowercase, multiple=True).configure_interactive().value) - start_date = configure_date_option(start_date, "start", "Start date") - end_date = configure_date_option(end_date, "end", "End date") + start_date = _configure_date_option(start_date, "start", "Start date") + end_date = _configure_date_option(end_date, "end", "End date") if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") @@ -566,88 +650,4 @@ def download(ctx: Context, if not success: raise RuntimeError( - "Something went wrong while running the downloader data provider, see the logs above for more information") - -def _get_historical_data_provider() -> str: - return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) - -def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], key_config_data: str) -> List[str]: - """ - Get parameter from data provider config JSON or return default parameters. - - Args: - - data_provider_config_json (Dict[str, Any]): Configuration JSON. - - default_param (List[str]): Default parameters. - - key_config_data (str): Key to look for in the config JSON. - - Returns: - - List[str]: List of parameters. - """ - container.logger.info(f'data_provider_config_json: {data_provider_config_json}') - if data_provider_config_json is None: - return default_param - - return data_provider_config_json.get(key_config_data, default_param) - -def get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str) -> str: - """ - Get user input or prompt for selection based on data types. - - Args: - - user_input_data (str): User input data. - - data_types (List[str]): List of supported data types. - - data_provider_name (str): Name of the data provider. - - Returns: - - str: Selected data type or prompted choice. - - Raises: - - ValueError: If user input data is not in supported data types. - """ - - if not user_input_data: - # Prompt user to select a ticker's security type - return prompt_user_selection(data_types) - - elif user_input_data not in data_types: - # Raise ValueError for unsupported data type - raise ValueError( - f"The {data_provider_name} data provider does not support {user_input_data}. " - f"Please choose a supported data from: {data_types}." - ) - - return user_input_data - -def prompt_user_selection(data_types: List[str]) -> str: - """ - Prompt user to select a data type from a list. - - Args: - - data_types (List[str]): List of supported data types. - - Returns: - - str: Selected data type. - """ - - options = [Option(id=data_type, label=data_type) for data_type in data_types] - return container.logger.prompt_list("Select a Ticker's security type", options) - -def configure_date_option(date_value: str, option_id: str, option_label: str) -> str: - """ - Configure the date based on the provided date value, option ID, and option label. - - Args: - - date_value (str): Existing date value. - - option_id (str): Identifier for the date option. - - option_label (str): Label for the date option. - - Returns: - - str: Configured date. - """ - - date_option = DatasetDateOption(id=option_id, label=option_label, description=f"Enter the {option_label} for the historical data request in the format YYYYMMDD.") - - if not date_value: - return date_option.configure_interactive() - - return date_option.configure_non_interactive(date_value) \ No newline at end of file + "Something went wrong while running the downloader data provider, see the logs above for more information") \ No newline at end of file From c67d2f275daded4d959cad45ff112a18128a3ae2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 21:36:44 +0300 Subject: [PATCH 22/53] feat: new config param options `download` --- lean/commands/data/download.py | 2 +- lean/models/click_options.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index c7cc0f84..a24b5ba1 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -497,7 +497,7 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - @option("--data-provider-historical", type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), help="The name of the downloader data provider.") -@options_from_json(get_configs_for_options("backtest")) +@options_from_json(get_configs_for_options("download")) @option("--dataset", type=str, help="The name of the dataset to download non-interactively") @option("--overwrite", is_flag=True, default=False, help="Overwrite existing local data") @option("--force", is_flag=True, default=False, hidden=True) diff --git a/lean/models/click_options.py b/lean/models/click_options.py index 5e56f444..15d95349 100644 --- a/lean/models/click_options.py +++ b/lean/models/click_options.py @@ -29,6 +29,8 @@ def get_configs_for_options(env: str) -> List[Configuration]: brokerage = cli_data_downloaders elif env == "research": brokerage = cli_data_downloaders + elif env == "download": + brokerage = cli_data_downloaders else: raise ValueError("Acceptable values for 'env' are: 'live-cloud', 'live-cli', 'backtest', 'research'") From 586dada7f8bf714d719ad340f035467584d6a924 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 21:37:06 +0300 Subject: [PATCH 23/53] remove: extra log.info --- lean/commands/data/download.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index a24b5ba1..eda16627 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -424,7 +424,6 @@ def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_pa Returns: - List[str]: List of parameters. """ - container.logger.info(f'data_provider_config_json: {data_provider_config_json}') if data_provider_config_json is None: return default_param @@ -576,9 +575,7 @@ def download(ctx: Context, all_data_files = _get_data_files(organization, products) container.data_downloader.download_files(all_data_files, overwrite, organization.id) - else: - logger = container.logger - + else: data_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) data_provider_config_json = None @@ -605,7 +602,8 @@ def download(ctx: Context, if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") - + + logger = container.logger lean_config = container.lean_config_manager.get_lean_config() data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_provider.ensure_module_installed(organization.id) From 7b36bac8380371de96e7da0d934e22a063c33dda Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 18 Apr 2024 23:41:40 +0300 Subject: [PATCH 24/53] fix:bug: if provider has more than 2 download providers --- lean/commands/data/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index eda16627..0aed9bf4 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -633,7 +633,7 @@ def download(ctx: Context, run_options["working_dir"] = downloader_data_provider_path_dll dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-provider", data_provider.get_config_value_from_name(MODULE_DATA_DOWNLOADER), + "--data-provider", data_provider.get_settings()[MODULE_DATA_DOWNLOADER], "--destination-dir", DATA_FOLDER_PATH, "--data-type", data_type, "--start-date", start_date.value.strftime("%Y%m%d"), From 1d31bb64f9f4354deca02355645d401553702bbd Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 19 Apr 2024 00:07:34 +0300 Subject: [PATCH 25/53] feat: additional test download process refactor: encapsulate data download data runner in tests --- tests/commands/data/test_download.py | 136 ++++++++++++++++++++------- 1 file changed, 102 insertions(+), 34 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 5b78de4a..5a42d9d6 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -35,48 +35,116 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) -def test_download_data_non_interactive(): - create_fake_lean_cli_directory() +def _get_data_provider_config() -> Dict[str, Any]: + """ + Retrieve the configuration settings for a financial data provider. + + This method encapsulates the configuration settings typically found in a data provider config JSON file, + as referenced by a file named .json in an example from a GitHub repository. + + Returns: + Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. + """ + data_provider_config_file_json: Dict[str, Any] = { + "data-types": [ "Trade", "Quote" ], # Supported data types: Trade and Quote + "data-resolutions": [ "Second", "Minute", "Hour", "Daily" ], # Supported data resolutions: Second, Minute, Hour, Daily + "data-supported": [ "Equity", "Equity Options", "Indexes", "Index Options" ] # Supported asset classes: Equity, Equity Options, Indexes, Index Options + } + + return data_provider_config_file_json +def _create_lean_data_download(data_provider_name: str, + data_type: str, + resolution: str, + security_type: str, + tickers: List[str], + start_date: str, + end_date: str, + data_provider_config_file_json: Dict[str, Any], + extra_run_command: List[str] = None): + """ + Create a data download command for the Lean algorithmic trading engine. + + This method constructs and invokes a Lean CLI command to download historical data from a specified data provider. + It utilizes a mock data provider configuration JSON and may include extra run commands if provided. + + Args: + data_provider_name (str): Name of the data provider. + data_type (str): Type of data to download (e.g., Trade, Quote). + resolution (str): Time resolution of the data (e.g., Second, Minute). + security_type (str): Type of security (e.g., Equity, Equity Options). + tickers (List[str]): List of tickers to download data for. + start_date (str): Start date of the data download in YYYY-MM-DD format. + end_date (str): End date of the data download in YYYY-MM-DD format. + data_provider_config_file_json (Dict[str, Any]): Mock data provider configuration JSON. + extra_run_command (List[str], optional): Extra run commands to be included in the Lean CLI command. + + Returns: + CompletedProcess: Result of the Lean CLI command execution. + """ + create_fake_lean_cli_directory() container = initialize_container() with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): - with mock.patch.object(container.api_client.data, "download_public_file_json", return_value={ "data-supported" : [ "Equity", "Equity Options", "Indexes", "Index Options" ] }): - result = CliRunner().invoke(lean, ["data", "download", - "--data-provider-historical", "Polygon", - "--polygon-api-key", "123", - "--data-type", "Trade", - "--resolution", "Minute", - "--ticker-security-type", "Equity", - "--tickers", "AAPL", - "--start-date", "20240101", - "--end-date", "20240202"]) - - assert result.exit_code == 0 - -def test_download_data_non_interactive_data_provider_missed_param(): - create_fake_lean_cli_directory() + with mock.patch.object(container.api_client.data, "download_public_file_json", return_value=data_provider_config_file_json): + run_parameters = [ + "data", "download", + "--data-provider-historical", data_provider_name, + "--data-type", data_type, + "--resolution", resolution, + "--ticker-security-type", security_type, + "--tickers", ','.join(tickers), + "--start-date", start_date, + "--end-date", end_date, + ] + if extra_run_command: + run_parameters += extra_run_command - container = initialize_container() + return CliRunner().invoke(lean, run_parameters) - # with mock.patch.object(container.api_client.projects, 'get_all', return_value=cloud_projects) as mock_get_all,\ - # mock.patch.object(container.api_client.projects, 'delete', return_value=None) as mock_delete: +@pytest.mark.parametrize("data_provider,data_provider_parameters", + [("Polygon", ["--polygon-api-key", "123"]), + ("Binance", ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", "--binanceus-api-secret", "123"]), + ("Interactive Brokers", ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) +def test_download_data_non_interactive(data_provider: str, data_provider_parameters: List[str]): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), data_provider_parameters) + assert run_data_download.exit_code == 0 - with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): - with mock.patch.object(container.api_client.data, "download_public_file_json", return_value={ "data-supported" : [ "Equity", "Equity Options", "Indexes", "Index Options" ] }): - result = CliRunner().invoke(lean, ["data", "download", - "--data-provider-historical", "Polygon", - "--data-type", "Trade", - "--resolution", "Minute", - "--ticker-security-type", "Equity", - "--tickers", "AAPL", - "--start-date", "20240101", - "--end-date", "20240202"]) - - assert result.exit_code == 1 +@pytest.mark.parametrize("data_provider,missed_parameters", + [("Polygon", "--polygon-api-key"), + ("Binance", "--binance-exchange-name"), + ("Interactive Brokers", "--ib-user-name, --ib-account, --ib-password")]) +def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, missed_parameters: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config()) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert missed_parameters in error_msg + +@pytest.mark.parametrize("data_provider,wrong_security_type", + [("Polygon", "Future"),("Polygon", "Crypto"),("Polygon", "Forex")]) +def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert f"The {data_provider} data provider does not support {wrong_security_type}." in error_msg + +@pytest.mark.parametrize("data_provider,start_date,end_date", [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")]) +def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, end_date, _get_data_provider_config(), ["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert f"Historical start date cannot be greater than or equal to historical end date." in error_msg + +@pytest.mark.parametrize("wrong_data_type",[("OpenInterest")]) +def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): + run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 - error_msg = str(result.exc_info[1]) - assert "--polygon-api-key" in error_msg + error_msg = str(run_data_download.exc_info[1]) + assert f"The Polygon data provider does not support {wrong_data_type}." in error_msg def test_non_interactive_bulk_select(): # TODO From fbd2d7e0e999a40a0d82a8e0455698cdfc706480 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 19 Apr 2024 16:25:53 +0300 Subject: [PATCH 26/53] fix: wrong condition tabulation --- lean/components/docker/lean_runner.py | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index a793eb92..ffe2c216 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -340,27 +340,27 @@ def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerIm self._logger.debug(f"LeanRunner._setup_installed_packages(): installed packages {len(installed_packages)}") self.set_up_common_csharp_options(run_options, target_path) - # Mount the modules directory - run_options["volumes"][MODULES_DIRECTORY] = {"bind": "/Modules", "mode": "ro"} - - # Add the modules directory as a NuGet source root - run_options["commands"].append("dotnet nuget add source /Modules") - # Create a C# project used to resolve the dependencies of the modules - run_options["commands"].append("mkdir /ModulesProject") - run_options["commands"].append("dotnet new sln -o /ModulesProject") + # Mount the modules directory + run_options["volumes"][MODULES_DIRECTORY] = {"bind": "/Modules", "mode": "ro"} + + # Add the modules directory as a NuGet source root + run_options["commands"].append("dotnet nuget add source /Modules") + # Create a C# project used to resolve the dependencies of the modules + run_options["commands"].append("mkdir /ModulesProject") + run_options["commands"].append("dotnet new sln -o /ModulesProject") - framework_ver = self._docker_manager.get_image_label(image, 'target_framework', DEFAULT_LEAN_DOTNET_FRAMEWORK) - run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") - run_options["commands"].append("rm /ModulesProject/Class1.cs") - - # Add all modules to the project, automatically resolving all dependencies - for package in installed_packages: - self._logger.debug(f"LeanRunner._setup_installed_packages(): Adding module {package} to the project") - run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") - run_options["commands"].append(f"dotnet add /ModulesProject package {package.name} --version {package.version}") - - # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists - run_options["commands"].append("python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', DEFAULT_LEAN_DOTNET_FRAMEWORK) + run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") + run_options["commands"].append("rm /ModulesProject/Class1.cs") + + # Add all modules to the project, automatically resolving all dependencies + for package in installed_packages: + self._logger.debug(f"LeanRunner._setup_installed_packages(): Adding module {package} to the project") + run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") + run_options["commands"].append(f"dotnet add /ModulesProject package {package.name} --version {package.version}") + + # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists + run_options["commands"].append("python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") return bool(installed_packages) From e3396612b72dd4ea89e7efa5ac3a9904f02f4480 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 19 Apr 2024 17:43:05 +0300 Subject: [PATCH 27/53] refactor: use provider_config live internal variable --- tests/commands/data/test_download.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 5a42d9d6..d58503f2 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -124,7 +124,11 @@ def test_download_data_non_interactive_data_provider_missed_param(data_provider: @pytest.mark.parametrize("data_provider,wrong_security_type", [("Polygon", "Future"),("Polygon", "Crypto"),("Polygon", "Forex")]) def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) + data_provider_config = { + "data-supported": [ "Equity", "Equity Options", "Indexes", "Index Options" ] + } + + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", data_provider_config, ["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) From fe4a76ee60a9c1e37741cf967a6aa2fb91188861 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 19 Apr 2024 17:53:26 +0300 Subject: [PATCH 28/53] Revert "refactor: use provider_config live internal variable" This reverts commit 3c9b704423f43099fc87c0951856bddcbc81ce20. --- tests/commands/data/test_download.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index d58503f2..5a42d9d6 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -124,11 +124,7 @@ def test_download_data_non_interactive_data_provider_missed_param(data_provider: @pytest.mark.parametrize("data_provider,wrong_security_type", [("Polygon", "Future"),("Polygon", "Crypto"),("Polygon", "Forex")]) def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): - data_provider_config = { - "data-supported": [ "Equity", "Equity Options", "Indexes", "Index Options" ] - } - - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", data_provider_config, ["--polygon-api-key", "123"]) + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) From b3a8fc9ffe2fd001b8fc0dd0c843975b10cd00ad Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 19 Apr 2024 19:50:44 +0300 Subject: [PATCH 29/53] fix: test initialization of new property in mock config file --- tests/commands/data/test_download.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 5a42d9d6..7e50a2e2 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -82,6 +82,10 @@ def _create_lean_data_download(data_provider_name: str, Returns: CompletedProcess: Result of the Lean CLI command execution. """ + # add additional property in module config file + for data_provider in cli_data_downloaders: + data_provider.__setattr__("_specifications_url", "") + create_fake_lean_cli_directory() container = initialize_container() From 6acb4f2a872a4fa55c7e5724005d765a133997a7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 20 Apr 2024 00:21:40 +0300 Subject: [PATCH 30/53] feat: add market param rename: input command market/security-type refactor: add prompt msg helper remove: prompt use selection method --- lean/commands/data/download.py | 36 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 0aed9bf4..364d1f6e 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -429,7 +429,7 @@ def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_pa return data_provider_config_json.get(key_config_data, default_param) -def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str) -> str: +def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str, prompt_message_helper: str) -> str: """ Get user input or prompt for selection based on data types. @@ -447,7 +447,8 @@ def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_ if not user_input_data: # Prompt user to select a ticker's security type - return _prompt_user_selection(data_types) + options = [Option(id=data_type, label=data_type) for data_type in data_types] + return container.logger.prompt_list(prompt_message_helper, options) elif user_input_data not in data_types: # Raise ValueError for unsupported data type @@ -458,20 +459,6 @@ def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_ return user_input_data -def _prompt_user_selection(data_types: List[str]) -> str: - """ - Prompt user to select a data type from a list. - - Args: - - data_types (List[str]): List of supported data types. - - Returns: - - str: Selected data type. - """ - - options = [Option(id=data_type, label=data_type) for data_type in data_types] - return container.logger.prompt_list("Select a Ticker's security type", options) - def _configure_date_option(date_value: str, option_id: str, option_label: str) -> str: """ Configure the date based on the provided date value, option ID, and option label. @@ -504,8 +491,9 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - help="Automatically confirm payment confirmation prompts") @option("--data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") @option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") -@option("--ticker-security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), +@option("--security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), help="Specify the security type of the historical data") +@option("--market", type=str, help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") @option("--tickers", type=str, help="Specify comma separated list of tickers to use for historical data request.") @@ -531,7 +519,8 @@ def download(ctx: Context, auto_confirm: bool, data_type: Optional[str], resolution: Optional[str], - ticker_security_type: Optional[str], + security_type: Optional[str], + market: Optional[str], tickers: Optional[str], start_date: Optional[str], end_date: Optional[str], @@ -585,10 +574,12 @@ def download(ctx: Context, data_provider_support_security_types = _get_param_from_config(data_provider_config_json, SECURITY_TYPES, "data-supported") data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") + data_provider_support_markets = _get_param_from_config(data_provider_config_json, [ "USA" ], "data-markets") - ticker_security_type = _get_user_input_or_prompt(ticker_security_type, data_provider_support_security_types, data_provider_historical) - data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical) - resolution = _get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical) + security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") + data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical, "Select a Data type") + resolution = _get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical, "Select a Resolution") + market = _get_user_input_or_prompt(market, data_provider_support_markets, data_provider_historical, "Select a Market") if not tickers: tickers = ','.join(DatasetTextOption(id="id", @@ -638,7 +629,8 @@ def download(ctx: Context, "--data-type", data_type, "--start-date", start_date.value.strftime("%Y%m%d"), "--end-date", end_date.value.strftime("%Y%m%d"), - "--security-type", ticker_security_type, + "--security-type", security_type, + "--market", market, "--resolution", resolution, "--tickers", tickers] From 57496a284c3b86ea1c1314f32218f38a1e9f5c83 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 20 Apr 2024 00:22:16 +0300 Subject: [PATCH 31/53] refactor:test: data download --- tests/commands/data/test_download.py | 71 +++++++++++++++++----------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 7e50a2e2..f8c9d7dd 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -35,23 +35,33 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) -def _get_data_provider_config() -> Dict[str, Any]: - """ - Retrieve the configuration settings for a financial data provider. - - This method encapsulates the configuration settings typically found in a data provider config JSON file, - as referenced by a file named .json in an example from a GitHub repository. +def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any]: + """ + Retrieve the configuration settings for a financial data provider. - Returns: - Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. - """ - data_provider_config_file_json: Dict[str, Any] = { + This method encapsulates the configuration settings typically found in a data provider config JSON file, + as referenced by a file named .json in an example from a GitHub repository. + + Returns: + Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. + """ + + if is_crypto_configs: + return { + "data-types": [ "Trade", "Quote" ], + "data-resolutions": [ "Minute", "Hour", "Daily" ], + "data-supported": [ "Crypto", "CryptoFuture" ], + "data-markets": [ "Binance", "Kraken"] + } + + data_provider_config_file_json: Dict[str, Any] = { "data-types": [ "Trade", "Quote" ], # Supported data types: Trade and Quote "data-resolutions": [ "Second", "Minute", "Hour", "Daily" ], # Supported data resolutions: Second, Minute, Hour, Daily - "data-supported": [ "Equity", "Equity Options", "Indexes", "Index Options" ] # Supported asset classes: Equity, Equity Options, Indexes, Index Options + "data-supported": [ "Equity", "Option", "Index", "IndexOption" ], # Supported asset classes: Equity, Equity Options, Indexes, Index Options + "data-markets": [ "NYSE", "USA"] } - return data_provider_config_file_json + return data_provider_config_file_json def _create_lean_data_download(data_provider_name: str, data_type: str, @@ -61,6 +71,7 @@ def _create_lean_data_download(data_provider_name: str, start_date: str, end_date: str, data_provider_config_file_json: Dict[str, Any], + market: str = None, extra_run_command: List[str] = None): """ Create a data download command for the Lean algorithmic trading engine. @@ -96,30 +107,34 @@ def _create_lean_data_download(data_provider_name: str, "--data-provider-historical", data_provider_name, "--data-type", data_type, "--resolution", resolution, - "--ticker-security-type", security_type, + "--security-type", security_type, "--tickers", ','.join(tickers), "--start-date", start_date, "--end-date", end_date, ] + if market: + run_parameters.extend(["--market", market]) if extra_run_command: run_parameters += extra_run_command return CliRunner().invoke(lean, run_parameters) -@pytest.mark.parametrize("data_provider,data_provider_parameters", - [("Polygon", ["--polygon-api-key", "123"]), - ("Binance", ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", "--binanceus-api-secret", "123"]), - ("Interactive Brokers", ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) -def test_download_data_non_interactive(data_provider: str, data_provider_parameters: List[str]): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), data_provider_parameters) +@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,tickers,data_provider_parameters", + [("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]), + ("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"], ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", "--binanceus-api-secret", "123"]), + ("CoinApi", "Kraken", True, "Crypto", ["BTCUSDC", "ETHUSD"], ["--coinapi-api-key", "123", "--coinapi-product", "Free"]), + ("Interactive Brokers", "USA", False, "Index", ["INTL","NVDA"], ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) +def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str, tickers: List[str], data_provider_parameters: List[str]): + run_data_download = _create_lean_data_download( + data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202", _get_data_provider_config(is_crypto), market, data_provider_parameters) assert run_data_download.exit_code == 0 -@pytest.mark.parametrize("data_provider,missed_parameters", - [("Polygon", "--polygon-api-key"), - ("Binance", "--binance-exchange-name"), - ("Interactive Brokers", "--ib-user-name, --ib-account, --ib-password")]) -def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, missed_parameters: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config()) +@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,missed_parameters", + [("Polygon", "NYSE", False, "Equity", "--polygon-api-key"), + ("Binance", "Binance", True, "Crypto", "--binance-exchange-name"), + ("Interactive Brokers", "USA", False, "Equity", "--ib-user-name, --ib-account, --ib-password")]) +def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, market: str, is_crypto: bool, security_type: str, missed_parameters: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(is_crypto), market) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) @@ -128,7 +143,7 @@ def test_download_data_non_interactive_data_provider_missed_param(data_provider: @pytest.mark.parametrize("data_provider,wrong_security_type", [("Polygon", "Future"),("Polygon", "Crypto"),("Polygon", "Forex")]) def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), extra_run_command=["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) @@ -136,7 +151,7 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w @pytest.mark.parametrize("data_provider,start_date,end_date", [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")]) def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, end_date, _get_data_provider_config(), ["--polygon-api-key", "123"]) + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, end_date, _get_data_provider_config(), "USA", extra_run_command=["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) @@ -144,7 +159,7 @@ def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, @pytest.mark.parametrize("wrong_data_type",[("OpenInterest")]) def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): - run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), ["--polygon-api-key", "123"]) + run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), extra_run_command=["--polygon-api-key", "123"]) assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) From c7707dc594d3a436170fc9cf1b9ae381c53418db Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 23 Apr 2024 21:31:07 +0300 Subject: [PATCH 32/53] feat: create config file to run download provider project + mount rename: data_provider => data_downloader remove: destination-dir args --- lean/commands/data/download.py | 52 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 364d1f6e..19b6e3bf 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -11,15 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +from docker.types import Mount from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name -from lean.constants import DATA_FOLDER_PATH, DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES +from lean.constants import DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery from lean.models.click_options import get_configs_for_options, options_from_json -from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, Product +from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, OptionResult, Product from lean.models.logger import Option from lean.models.cli import cli_data_downloaders @@ -459,7 +461,7 @@ def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_ return user_input_data -def _configure_date_option(date_value: str, option_id: str, option_label: str) -> str: +def _configure_date_option(date_value: str, option_id: str, option_label: str) -> OptionResult: """ Configure the date based on the provided date value, option ID, and option label. @@ -493,7 +495,7 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - @option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") @option("--security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), help="Specify the security type of the historical data") -@option("--market", type=str, help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") +@option("--market", type=str, default="USA", help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") @option("--tickers", type=str, help="Specify comma separated list of tickers to use for historical data request.") @@ -565,16 +567,16 @@ def download(ctx: Context, all_data_files = _get_data_files(organization, products) container.data_downloader.download_files(all_data_files, overwrite, organization.id) else: - data_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) + data_downloader_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) data_provider_config_json = None - if data_provider._specifications_url is not None: - data_provider_config_json = container.api_client.data.download_public_file_json(data_provider._specifications_url) + if data_downloader_provider._specifications_url is not None: + data_provider_config_json = container.api_client.data.download_public_file_json(data_downloader_provider._specifications_url) data_provider_support_security_types = _get_param_from_config(data_provider_config_json, SECURITY_TYPES, "data-supported") data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") - data_provider_support_markets = _get_param_from_config(data_provider_config_json, [ "USA" ], "data-markets") + data_provider_support_markets = _get_param_from_config(data_provider_config_json, [ market ], "data-markets") security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical, "Select a Data type") @@ -596,11 +598,12 @@ def download(ctx: Context, logger = container.logger lean_config = container.lean_config_manager.get_lean_config() - data_provider = config_build_for_name(lean_config, data_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) - data_provider.ensure_module_installed(organization.id) - container.lean_config_manager.set_properties(data_provider.get_settings()) + + data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=True) + data_downloader_provider.ensure_module_installed(organization.id) + container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) # Info: I don't understand why it returns empty result - paths_to_mount = data_provider.get_paths_to_mount() + paths_to_mount = data_downloader_provider.get_paths_to_mount() engine_image = container.cli_config_manager.get_engine_image(image) @@ -614,6 +617,18 @@ def download(ctx: Context, downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" + # Create config dictionary with credentials + config: Dict[str, str] = { + "job-user-id": lean_config.get("job-user-id"), + "api-access-token": lean_config.get("api-access-token"), + "job-organization-id": organization.id + } + config.update(data_downloader_provider.get_settings()) + + config_path = container.temp_manager.create_temporary_directory() / "config.json" + with config_path.open("w+", encoding="utf-8") as file: + json.dump(config, file) + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, debugging_method=None, detach=False, @@ -624,8 +639,7 @@ def download(ctx: Context, run_options["working_dir"] = downloader_data_provider_path_dll dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-provider", data_provider.get_settings()[MODULE_DATA_DOWNLOADER], - "--destination-dir", DATA_FOLDER_PATH, + "--data-downloader", data_downloader_provider.get_settings()[MODULE_DATA_DOWNLOADER], "--data-type", data_type, "--start-date", start_date.value.strftime("%Y%m%d"), "--end-date", end_date.value.strftime("%Y%m%d"), @@ -634,7 +648,15 @@ def download(ctx: Context, "--resolution", resolution, "--tickers", tickers] - run_options["commands"].append(' '.join(dll_arguments)) + run_options["commands"].append(' '.join(dll_arguments)) + + # mount our created above config with work directory + run_options["mounts"].append( + Mount(target=f"{downloader_data_provider_path_dll}/config.json", + source=str(config_path), + type="bind", + read_only=True) + ) success = container.docker_manager.run_image(engine_image, **run_options) From fe1b9b6735d3e0ccafb59aab517794aa45859cc9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 02:03:58 +0300 Subject: [PATCH 33/53] feat: mock get_organization refactor: switch off interactive for data providers feat: description of some param get_basic_docker_config_without_algo --- lean/commands/data/download.py | 77 +++++++++++++++------------ lean/components/docker/lean_runner.py | 5 +- tests/commands/data/test_download.py | 65 +++++++++++----------- 3 files changed, 81 insertions(+), 66 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 19b6e3bf..99e8038b 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +from json import dump from docker.types import Mount from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice @@ -428,7 +428,7 @@ def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_pa """ if data_provider_config_json is None: return default_param - + return data_provider_config_json.get(key_config_data, default_param) def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str, prompt_message_helper: str) -> str: @@ -442,23 +442,23 @@ def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_ Returns: - str: Selected data type or prompted choice. - + Raises: - ValueError: If user input data is not in supported data types. """ - + if not user_input_data: # Prompt user to select a ticker's security type options = [Option(id=data_type, label=data_type) for data_type in data_types] return container.logger.prompt_list(prompt_message_helper, options) - + elif user_input_data not in data_types: # Raise ValueError for unsupported data type raise ValueError( f"The {data_provider_name} data provider does not support {user_input_data}. " f"Please choose a supported data from: {data_types}." ) - + return user_input_data def _configure_date_option(date_value: str, option_id: str, option_label: str) -> OptionResult: @@ -473,14 +473,15 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - Returns: - str: Configured date. """ - + date_option = DatasetDateOption(id=option_id, label=option_label, description=f"Enter the {option_label} for the historical data request in the format YYYYMMDD.") - + if not date_value: return date_option.configure_interactive() - + return date_option.configure_non_interactive(date_value) + @command(cls=LeanCommand, requires_lean_config=True, allow_unknown_options=True, name="download") @option("--data-provider-historical", type=Choice([data_downloader.get_name() for data_downloader in cli_data_downloaders], case_sensitive=False), @@ -492,10 +493,12 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - @option("--yes", "-y", "auto_confirm", is_flag=True, default=False, help="Automatically confirm payment confirmation prompts") @option("--data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") -@option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), help="Specify the resolution of the historical data") -@option("--security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), +@option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), + help="Specify the resolution of the historical data") +@option("--security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), help="Specify the security type of the historical data") -@option("--market", type=str, default="USA", help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") +@option("--market", type=str, default="USA", + help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") @option("--tickers", type=str, help="Specify comma separated list of tickers to use for historical data request.") @@ -514,7 +517,7 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - help="Pull the LEAN engine image before running the Downloader Data Provider") @pass_context def download(ctx: Context, - data_provider_historical: Optional[str], + data_provider_historical: Optional[str], dataset: Optional[str], overwrite: bool, force: bool, @@ -566,7 +569,7 @@ def download(ctx: Context, all_data_files = _get_data_files(organization, products) container.data_downloader.download_files(all_data_files, overwrite, organization.id) - else: + else: data_downloader_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) data_provider_config_json = None @@ -578,10 +581,14 @@ def download(ctx: Context, data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") data_provider_support_markets = _get_param_from_config(data_provider_config_json, [ market ], "data-markets") - security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") - data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, data_provider_historical, "Select a Data type") - resolution = _get_user_input_or_prompt(resolution, data_provider_support_resolutions, data_provider_historical, "Select a Resolution") - market = _get_user_input_or_prompt(market, data_provider_support_markets, data_provider_historical, "Select a Market") + security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, + data_provider_historical, "Select a Ticker's security type") + data_type = _get_user_input_or_prompt(data_type, data_provider_support_data_types, + data_provider_historical, "Select a Data type") + resolution = _get_user_input_or_prompt(resolution, data_provider_support_resolutions, + data_provider_historical, "Select a Resolution") + market = _get_user_input_or_prompt(market, data_provider_support_markets, + data_provider_historical,"Select a Market") if not tickers: tickers = ','.join(DatasetTextOption(id="id", @@ -589,34 +596,36 @@ def download(ctx: Context, description="description", transform=DatasetTextOptionTransform.Lowercase, multiple=True).configure_interactive().value) - + start_date = _configure_date_option(start_date, "start", "Start date") end_date = _configure_date_option(end_date, "end", "End date") if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") - + logger = container.logger lean_config = container.lean_config_manager.get_lean_config() data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=True) + data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), + cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) - # Info: I don't understand why it returns empty result + # Info: I don't understand why it returns empty result paths_to_mount = data_downloader_provider.get_paths_to_mount() - + engine_image = container.cli_config_manager.get_engine_image(image) - + no_update = False if str(engine_image) != DEFAULT_ENGINE_IMAGE: # Custom engine image should not be updated. no_update = True logger.warn(f'A custom engine image: "{engine_image}" is being used!') - + container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" - + # Create config dictionary with credentials config: Dict[str, str] = { "job-user-id": lean_config.get("job-user-id"), @@ -624,20 +633,20 @@ def download(ctx: Context, "job-organization-id": organization.id } config.update(data_downloader_provider.get_settings()) - + config_path = container.temp_manager.create_temporary_directory() / "config.json" with config_path.open("w+", encoding="utf-8") as file: - json.dump(config, file) - + dump(config, file) + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, debugging_method=None, detach=False, image=engine_image, target_path=downloader_data_provider_path_dll, paths_to_mount=paths_to_mount) - + run_options["working_dir"] = downloader_data_provider_path_dll - + dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", "--data-downloader", data_downloader_provider.get_settings()[MODULE_DATA_DOWNLOADER], "--data-type", data_type, @@ -647,9 +656,9 @@ def download(ctx: Context, "--market", market, "--resolution", resolution, "--tickers", tickers] - + run_options["commands"].append(' '.join(dll_arguments)) - + # mount our created above config with work directory run_options["mounts"].append( Mount(target=f"{downloader_data_provider_path_dll}/config.json", @@ -659,7 +668,7 @@ def download(ctx: Context, ) success = container.docker_manager.run_image(engine_image, **run_options) - + if not success: raise RuntimeError( - "Something went wrong while running the downloader data provider, see the logs above for more information") \ No newline at end of file + "Something went wrong while running the downloader data provider, see the logs above for more information") diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index ffe2c216..fc0df42a 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -259,11 +259,12 @@ def get_basic_docker_config_without_algo(self, :param debugging_method: the debugging method if debugging needs to be enabled, None if not :param detach: whether LEAN should run in a detached container :param image: The docker image that will be used - :return: the Docker configuration containing basic configuration to run Lean + :param target_path: The target path inside the Docker container where the C# project should be located. :param paths_to_mount: additional paths to mount to the container + :return: the Docker configuration containing basic configuration to run Lean """ - docker_project_config = { "docker": {} } + docker_project_config = {"docker": {}} # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider data_dir = self._lean_config_manager.get_data_directory() self._handle_data_providers(lean_config, data_dir) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index f8c9d7dd..e58ca726 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -36,7 +36,7 @@ def test_bulk_extraction(setup): assert os.path.exists(file) def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any]: - """ + """ Retrieve the configuration settings for a financial data provider. This method encapsulates the configuration settings typically found in a data provider config JSON file, @@ -46,19 +46,21 @@ def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any] Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. """ - if is_crypto_configs: - return { - "data-types": [ "Trade", "Quote" ], - "data-resolutions": [ "Minute", "Hour", "Daily" ], - "data-supported": [ "Crypto", "CryptoFuture" ], - "data-markets": [ "Binance", "Kraken"] - } - - data_provider_config_file_json: Dict[str, Any] = { - "data-types": [ "Trade", "Quote" ], # Supported data types: Trade and Quote - "data-resolutions": [ "Second", "Minute", "Hour", "Daily" ], # Supported data resolutions: Second, Minute, Hour, Daily - "data-supported": [ "Equity", "Option", "Index", "IndexOption" ], # Supported asset classes: Equity, Equity Options, Indexes, Index Options - "data-markets": [ "NYSE", "USA"] + if is_crypto_configs: + return { + "data-types": ["Trade", "Quote"], + "data-resolutions": ["Minute", "Hour", "Daily"], + "data-supported": ["Crypto", "CryptoFuture"], + "data-markets": ["Binance", "Kraken"] + } + + data_provider_config_file_json: Dict[str, Any] = { + "data-types": ["Trade", "Quote"], # Supported data types: Trade and Quote + "data-resolutions": ["Second", "Minute", "Hour", "Daily"], + # Supported data resolutions: Second, Minute, Hour, Daily + "data-supported": ["Equity", "Option", "Index", "IndexOption"], + # Supported asset classes: Equity, Equity Options, Indexes, Index Options + "data-markets": ["NYSE", "USA"] } return data_provider_config_file_json @@ -100,22 +102,25 @@ def _create_lean_data_download(data_provider_name: str, create_fake_lean_cli_directory() container = initialize_container() - with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={ "commands": [] }): - with mock.patch.object(container.api_client.data, "download_public_file_json", return_value=data_provider_config_file_json): - run_parameters = [ - "data", "download", - "--data-provider-historical", data_provider_name, - "--data-type", data_type, - "--resolution", resolution, - "--security-type", security_type, - "--tickers", ','.join(tickers), - "--start-date", start_date, - "--end-date", end_date, - ] - if market: - run_parameters.extend(["--market", market]) - if extra_run_command: - run_parameters += extra_run_command + with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", + return_value={"commands": [], "mounts": []}): + with mock.patch.object(container.api_client.data, "download_public_file_json", + return_value=data_provider_config_file_json): + with mock.patch.object(container.api_client.organizations, "get", return_value=create_api_organization()): + run_parameters = [ + "data", "download", + "--data-provider-historical", data_provider_name, + "--data-type", data_type, + "--resolution", resolution, + "--security-type", security_type, + "--tickers", ','.join(tickers), + "--start-date", start_date, + "--end-date", end_date, + ] + if market: + run_parameters.extend(["--market", market]) + if extra_run_command: + run_parameters += extra_run_command return CliRunner().invoke(lean, run_parameters) From 0d4da4efc1172aab45a9780ca10c1520960f6c8a Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 02:12:19 +0300 Subject: [PATCH 34/53] refactor: styling --- tests/commands/data/test_download.py | 177 ++++++++++++++++----------- 1 file changed, 105 insertions(+), 72 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index e58ca726..826869bc 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -18,12 +18,14 @@ test_files = Path(os.path.join(os.path.dirname(os.path.realpath(__file__)), "testFiles")) + # Load in our test files into fake filesystem @pytest.fixture def setup(fs): fs.add_real_directory(test_files, read_only=False) yield fs + def test_bulk_extraction(setup): fake_tar = Path(os.path.join(test_files, "20220222_coinapi_crypto_ftx_price_aggregation.tar")) out = Path("/tmp/out") @@ -35,16 +37,17 @@ def test_bulk_extraction(setup): file = os.path.join(out, "crypto/ftx/daily/imxusd_trade.zip") assert os.path.exists(file) + def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any]: """ - Retrieve the configuration settings for a financial data provider. + Retrieve the configuration settings for a financial data provider. - This method encapsulates the configuration settings typically found in a data provider config JSON file, - as referenced by a file named .json in an example from a GitHub repository. + This method encapsulates the configuration settings typically found in a data provider config JSON file, + as referenced by a file named .json in an example from a GitHub repository. - Returns: - Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. - """ + Returns: + Dict[str, Any]: Configuration settings including supported data types, resolutions, and asset classes. + """ if is_crypto_configs: return { @@ -62,22 +65,23 @@ def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any] # Supported asset classes: Equity, Equity Options, Indexes, Index Options "data-markets": ["NYSE", "USA"] } - - return data_provider_config_file_json + + return data_provider_config_file_json + def _create_lean_data_download(data_provider_name: str, - data_type: str, - resolution: str, - security_type: str, - tickers: List[str], - start_date: str, - end_date: str, - data_provider_config_file_json: Dict[str, Any], - market: str = None, - extra_run_command: List[str] = None): - """ + data_type: str, + resolution: str, + security_type: str, + tickers: List[str], + start_date: str, + end_date: str, + data_provider_config_file_json: Dict[str, Any], + market: str = None, + extra_run_command: List[str] = None): + """ Create a data download command for the Lean algorithmic trading engine. - + This method constructs and invokes a Lean CLI command to download historical data from a specified data provider. It utilizes a mock data provider configuration JSON and may include extra run commands if provided. @@ -95,12 +99,12 @@ def _create_lean_data_download(data_provider_name: str, Returns: CompletedProcess: Result of the Lean CLI command execution. """ - # add additional property in module config file - for data_provider in cli_data_downloaders: - data_provider.__setattr__("_specifications_url", "") + # add additional property in module config file + for data_provider in cli_data_downloaders: + data_provider.__setattr__("_specifications_url", "") - create_fake_lean_cli_directory() - container = initialize_container() + create_fake_lean_cli_directory() + container = initialize_container() with mock.patch.object(container.lean_runner, "get_basic_docker_config_without_algo", return_value={"commands": [], "mounts": []}): @@ -122,58 +126,81 @@ def _create_lean_data_download(data_provider_name: str, if extra_run_command: run_parameters += extra_run_command - return CliRunner().invoke(lean, run_parameters) + return CliRunner().invoke(lean, run_parameters) + @pytest.mark.parametrize("data_provider,market,is_crypto,security_type,tickers,data_provider_parameters", - [("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]), - ("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"], ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", "--binanceus-api-secret", "123"]), - ("CoinApi", "Kraken", True, "Crypto", ["BTCUSDC", "ETHUSD"], ["--coinapi-api-key", "123", "--coinapi-product", "Free"]), - ("Interactive Brokers", "USA", False, "Index", ["INTL","NVDA"], ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) -def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str, tickers: List[str], data_provider_parameters: List[str]): - run_data_download = _create_lean_data_download( - data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202", _get_data_provider_config(is_crypto), market, data_provider_parameters) - assert run_data_download.exit_code == 0 + [("Polygon", "NYSE", False, "Equity", ["AAPL"], ["--polygon-api-key", "123"]), + ("Binance", "Binance", True, "CryptoFuture", ["BTCUSDT"], + ["--binance-exchange-name", "BinanceUS", "--binanceus-api-key", "123", + "--binanceus-api-secret", "123"]), + ("CoinApi", "Kraken", True, "Crypto", ["BTCUSDC", "ETHUSD"], + ["--coinapi-api-key", "123", "--coinapi-product", "Free"]), + ("Interactive Brokers", "USA", False, "Index", ["INTL", "NVDA"], + ["--ib-user-name", "123", "--ib-account", "Individual", "--ib-password", "123"])]) +def test_download_data_non_interactive(data_provider: str, market: str, is_crypto: bool, security_type: str, + tickers: List[str], data_provider_parameters: List[str]): + run_data_download = _create_lean_data_download( + data_provider, "Trade", "Minute", security_type, tickers, "20240101", "20240202", + _get_data_provider_config(is_crypto), market, data_provider_parameters) + assert run_data_download.exit_code == 0 + @pytest.mark.parametrize("data_provider,market,is_crypto,security_type,missed_parameters", - [("Polygon", "NYSE", False, "Equity", "--polygon-api-key"), - ("Binance", "Binance", True, "Crypto", "--binance-exchange-name"), - ("Interactive Brokers", "USA", False, "Equity", "--ib-user-name, --ib-account, --ib-password")]) -def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, market: str, is_crypto: bool, security_type: str, missed_parameters: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(is_crypto), market) - assert run_data_download.exit_code == 1 - - error_msg = str(run_data_download.exc_info[1]) - assert missed_parameters in error_msg + [("Polygon", "NYSE", False, "Equity", "--polygon-api-key"), + ("Binance", "Binance", True, "Crypto", "--binance-exchange-name"), + ("Interactive Brokers", "USA", False, "Equity", + "--ib-user-name, --ib-account, --ib-password")]) +def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, market: str, is_crypto: bool, + security_type: str, missed_parameters: str): + run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", security_type, ["AAPL"], + "20240101", "20240202", _get_data_provider_config(is_crypto), market) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert missed_parameters in error_msg + @pytest.mark.parametrize("data_provider,wrong_security_type", - [("Polygon", "Future"),("Polygon", "Crypto"),("Polygon", "Forex")]) + [("Polygon", "Future"), ("Polygon", "Crypto"), ("Polygon", "Forex")]) def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], "20240101", "20240202", _get_data_provider_config(), extra_run_command=["--polygon-api-key", "123"]) - assert run_data_download.exit_code == 1 - - error_msg = str(run_data_download.exc_info[1]) - assert f"The {data_provider} data provider does not support {wrong_security_type}." in error_msg + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", wrong_security_type, ["AAPL"], + "20240101", "20240202", _get_data_provider_config(), + extra_run_command=["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 -@pytest.mark.parametrize("data_provider,start_date,end_date", [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")]) + error_msg = str(run_data_download.exc_info[1]) + assert f"The {data_provider} data provider does not support {wrong_security_type}." in error_msg + + +@pytest.mark.parametrize("data_provider,start_date,end_date", + [("Polygon", "20240101", "20230202"), ("Polygon", "2024-01-01", "2023-02-02")]) def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, start_date: str, end_date: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, end_date, _get_data_provider_config(), "USA", extra_run_command=["--polygon-api-key", "123"]) - assert run_data_download.exit_code == 1 + run_data_download = _create_lean_data_download(data_provider, "Trade", "Hour", "Equity", ["AAPL"], start_date, + end_date, _get_data_provider_config(), "USA", + extra_run_command=["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert f"Historical start date cannot be greater than or equal to historical end date." in error_msg - error_msg = str(run_data_download.exc_info[1]) - assert f"Historical start date cannot be greater than or equal to historical end date." in error_msg -@pytest.mark.parametrize("wrong_data_type",[("OpenInterest")]) +@pytest.mark.parametrize("wrong_data_type", [("OpenInterest")]) def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): - run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), extra_run_command=["--polygon-api-key", "123"]) - assert run_data_download.exit_code == 1 - - error_msg = str(run_data_download.exc_info[1]) - assert f"The Polygon data provider does not support {wrong_data_type}." in error_msg + run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", + "20240202", _get_data_provider_config(), + extra_run_command=["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 1 + + error_msg = str(run_data_download.exc_info[1]) + assert f"The Polygon data provider does not support {wrong_data_type}." in error_msg + def test_non_interactive_bulk_select(): - # TODO + # TODO pass + def test_interactive_bulk_select(): pytest.skip("This test is interactive") @@ -189,21 +216,26 @@ def test_interactive_bulk_select(): products = _select_products_interactive(organization, testSets) # No assertion, since interactive has multiple results + def test_dataset_requirements(): organization = create_api_organization() datasource = json.loads(bulk_datasource) testSet = Dataset(name="testSet", + organization = create_api_organization() + datasource = json.loads(bulk_datasource) + testSet = Dataset(name="testSet", vendor="testVendor", categories=["testData"], options=datasource["options"], paths=datasource["paths"], requirements=datasource.get("requirements", {})) - - for id, name in testSet.requirements.items(): - assert not organization.has_security_master_subscription(id) - assert id==39 -bulk_datasource=""" + for id, name in testSet.requirements.items(): + assert not organization.has_security_master_subscription(id) + assert id == 39 + + +bulk_datasource = """ { "requirements": { "39": "quantconnect-us-equity-security-master" @@ -389,15 +421,16 @@ def test_dataset_requirements(): } """ + def test_validate_datafile() -> None: + try: + value = "/^equity\\/usa\\/(factor_files|map_files)\\/[^\\/]+.zip$/m" + target = re.compile(value[value.index("/") + 1:value.rindex("/")]) + vendor = QCDataVendor(vendorName="Algoseek", regex=target) + DataFile(file='equity/usa/daily/aal.zip', vendor=vendor) + except Exception as err: + pytest.fail(f"{err}") - try: - value = "/^equity\\/usa\\/(factor_files|map_files)\\/[^\\/]+.zip$/m" - target = re.compile(value[value.index("/") + 1:value.rindex("/")]) - vendor = QCDataVendor(vendorName="Algoseek", regex=target) - DataFile(file='equity/usa/daily/aal.zip', vendor=vendor) - except Exception as err: - pytest.fail(f"{err}") def test_filter_pending_datasets() -> None: from lean.commands.data.download import _get_available_datasets, _get_data_information From b55d4c3a87fff2fb655a09e234d70dca94694008 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 02:24:13 +0300 Subject: [PATCH 35/53] fix: test_dataset_requirements --- tests/commands/data/test_download.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 826869bc..dd181719 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -218,9 +218,6 @@ def test_interactive_bulk_select(): def test_dataset_requirements(): - organization = create_api_organization() - datasource = json.loads(bulk_datasource) - testSet = Dataset(name="testSet", organization = create_api_organization() datasource = json.loads(bulk_datasource) testSet = Dataset(name="testSet", From 7b6668667c58e5fbf2feedab892d64a8f48c1354 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 16:24:37 +0300 Subject: [PATCH 36/53] refactor: input key's DataProvider interactive=False --- lean/commands/data/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 99e8038b..19c0dd67 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -606,7 +606,7 @@ def download(ctx: Context, logger = container.logger lean_config = container.lean_config_manager.get_lean_config() - data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=True) + data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider.ensure_module_installed(organization.id) From 1e8f5a6aa2bdf082ba69942e9d7b5fe23fd20bfa Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 23:19:49 +0300 Subject: [PATCH 37/53] rename: help message Select a historical Provider rename: variable to not confused with data_type (Trade, Quote, etc.) style: additional spaces and new lines --- lean/commands/data/download.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 19c0dd67..5cd9d947 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -412,7 +412,7 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: return available_datasets def _get_historical_data_provider() -> str: - return container.logger.prompt_list("Select a downloading mode", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) + return container.logger.prompt_list("Select a historical data provider", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], key_config_data: str) -> List[str]: """ @@ -431,13 +431,15 @@ def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_pa return data_provider_config_json.get(key_config_data, default_param) -def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_provider_name: str, prompt_message_helper: str) -> str: + +def _get_user_input_or_prompt(user_input_data: str, available_input_data: List[str], data_provider_name: str, + prompt_message_helper: str) -> str: """ Get user input or prompt for selection based on data types. Args: - user_input_data (str): User input data. - - data_types (List[str]): List of supported data types. + - available_input_data (List[str]): List of available input data options. - data_provider_name (str): Name of the data provider. Returns: @@ -449,14 +451,14 @@ def _get_user_input_or_prompt(user_input_data: str, data_types: List[str], data_ if not user_input_data: # Prompt user to select a ticker's security type - options = [Option(id=data_type, label=data_type) for data_type in data_types] + options = [Option(id=data_type, label=data_type) for data_type in available_input_data] return container.logger.prompt_list(prompt_message_helper, options) - elif user_input_data not in data_types: + elif user_input_data not in available_input_data: # Raise ValueError for unsupported data type raise ValueError( f"The {data_provider_name} data provider does not support {user_input_data}. " - f"Please choose a supported data from: {data_types}." + f"Please choose a supported data from: {available_input_data}." ) return user_input_data @@ -579,7 +581,7 @@ def download(ctx: Context, data_provider_support_security_types = _get_param_from_config(data_provider_config_json, SECURITY_TYPES, "data-supported") data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") - data_provider_support_markets = _get_param_from_config(data_provider_config_json, [ market ], "data-markets") + data_provider_support_markets = _get_param_from_config(data_provider_config_json, [market], "data-markets") security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") @@ -606,7 +608,6 @@ def download(ctx: Context, logger = container.logger lean_config = container.lean_config_manager.get_lean_config() - data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider.ensure_module_installed(organization.id) From e5cbf81fc0cd34bed47c05e91a6007a282b98e44 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 24 Apr 2024 23:30:55 +0300 Subject: [PATCH 38/53] feat: interactive param test --- tests/commands/data/test_download.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index dd181719..5bb44f5a 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -161,6 +161,15 @@ def test_download_data_non_interactive_data_provider_missed_param(data_provider: assert missed_parameters in error_msg +@pytest.mark.parametrize("data_type,resolution", + [("Trade", "Hour"), ("trade", "hour"), ("TRADE", "HOUR"), ("TrAdE", "HoUr")]) +def test_download_data_non_interactive_insensitive_input_param(data_type: str, resolution: str): + run_data_download = _create_lean_data_download( + "Polygon", data_type, resolution, "Equity", ["AAPL"], "20240101", "20240202", + _get_data_provider_config(False), "NYSE", ["--polygon-api-key", "123"]) + assert run_data_download.exit_code == 0 + + @pytest.mark.parametrize("data_provider,wrong_security_type", [("Polygon", "Future"), ("Polygon", "Crypto"), ("Polygon", "Forex")]) def test_download_data_non_interactive_wrong_security_type(data_provider: str, wrong_security_type: str): From 97f792ed521cb03a035460b2c214efef18d2448c Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Apr 2024 00:20:31 +0300 Subject: [PATCH 39/53] feat: new class QCDataType feat: new Security types in QCSecurityType feat: get_all_members for QC's classes --- lean/models/api.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/lean/models/api.py b/lean/models/api.py index 3033815c..4d552a6e 100644 --- a/lean/models/api.py +++ b/lean/models/api.py @@ -407,15 +407,54 @@ class QCMinimalOrganization(WrappedBaseModel): preferred: bool +class QCDataType(str, Enum): + Trade = "Trade" + Quote = "Quote" + OpenInterest = "OpenInterest" + + @classmethod + def get_all_members(cls): + """ + Retrieve all members (values) of the QCDataType enumeration. + + Returns: + list: A list containing all the values of the QCDataType enumeration. + + Example: + >>> all_data_types = QCDataType.get_all_members() + >>> print(all_data_types) + ['Trade', 'Quote', 'OpenInterest'] + """ + return list(cls.__members__.values()) + class QCSecurityType(str, Enum): Equity = "Equity" + Index = "Index" Forex = "Forex" CFD = "Cfd" Future = "Future" Crypto = "Crypto" + CryptoFuture = "CryptoFuture" Option = "Option" + IndexOption = "IndexOption" + Commodity = "Commodity" FutureOption = "FutureOption" + @classmethod + def get_all_members(cls): + """ + Retrieve all members (values) of the QCSecurityType enumeration. + + Returns: + list: A list containing all the values of the QCSecurityType enumeration. + + Example: + >>> all_security_types = QCSecurityType.get_all_members() + >>> print(all_security_types) + ['Equity', 'Index', 'Forex', 'Cfd', 'Future', 'Crypto', 'CryptoFuture', 'Option', 'IndexOption', 'Commodity', 'FutureOption'] + """ + return list(cls.__members__.values()) + class QCResolution(str, Enum): Tick = "Tick" @@ -436,6 +475,21 @@ def by_name(cls, name: str) -> 'QCResolution': return v raise ValueError(f"QCResolution has no member named '{name}'") + @classmethod + def get_all_members(cls): + """ + Retrieve all members (values) of the QCResolution enumeration. + + Returns: + list: A list containing all the values of the QCResolution enumeration. + + Example: + >>> all_resolutions = QCResolution.get_all_members() + >>> print(all_resolutions) + ['Tick', 'Second', 'Minute', 'Hour', 'Daily'] + """ + return list(cls.__members__.values()) + class QCLink(WrappedBaseModel): link: str From 49c2236171c2c698e0f8ca8ab8ff53ce606caaa1 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Apr 2024 00:22:01 +0300 Subject: [PATCH 40/53] refactor: use QC default types remove: extra constants fix: tests --- lean/commands/data/download.py | 26 ++++++++++++++++---------- lean/constants.py | 12 ------------ tests/commands/data/test_download.py | 4 ++-- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 5cd9d947..f32f90ef 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -17,11 +17,11 @@ from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name -from lean.constants import DATA_TYPES, DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER, RESOLUTIONS, SECURITY_TYPES +from lean.constants import DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER from lean.container import container -from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery +from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType from lean.models.click_options import get_configs_for_options, options_from_json -from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform, OptionResult, Product +from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform,OptionResult, Product from lean.models.logger import Option from lean.models.cli import cli_data_downloaders @@ -454,7 +454,7 @@ def _get_user_input_or_prompt(user_input_data: str, available_input_data: List[s options = [Option(id=data_type, label=data_type) for data_type in available_input_data] return container.logger.prompt_list(prompt_message_helper, options) - elif user_input_data not in available_input_data: + elif user_input_data.lower() not in [available_data.lower() for available_data in available_input_data]: # Raise ValueError for unsupported data type raise ValueError( f"The {data_provider_name} data provider does not support {user_input_data}. " @@ -494,10 +494,10 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - @option("--force", is_flag=True, default=False, hidden=True) @option("--yes", "-y", "auto_confirm", is_flag=True, default=False, help="Automatically confirm payment confirmation prompts") -@option("--data-type", type=Choice(DATA_TYPES, case_sensitive=False), help="Specify the type of historical data") -@option("--resolution", type=Choice(RESOLUTIONS, case_sensitive=False), +@option("--data-type", type=Choice(QCDataType.get_all_members(), case_sensitive=False), help="Specify the type of historical data") +@option("--resolution", type=Choice(QCResolution.get_all_members(), case_sensitive=False), help="Specify the resolution of the historical data") -@option("--security-type", type=Choice(SECURITY_TYPES, case_sensitive=False), +@option("--security-type", type=Choice(QCSecurityType.get_all_members(), case_sensitive=False), help="Specify the security type of the historical data") @option("--market", type=str, default="USA", help="Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance')") @@ -578,9 +578,15 @@ def download(ctx: Context, if data_downloader_provider._specifications_url is not None: data_provider_config_json = container.api_client.data.download_public_file_json(data_downloader_provider._specifications_url) - data_provider_support_security_types = _get_param_from_config(data_provider_config_json, SECURITY_TYPES, "data-supported") - data_provider_support_data_types = _get_param_from_config(data_provider_config_json, DATA_TYPES, "data-types") - data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, RESOLUTIONS, "data-resolutions") + data_provider_support_security_types = _get_param_from_config(data_provider_config_json, + QCSecurityType.get_all_members(), + "data-supported") + data_provider_support_data_types = _get_param_from_config(data_provider_config_json, + QCDataType.get_all_members(), + "data-types") + data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, + QCResolution.get_all_members(), + "data-resolutions") data_provider_support_markets = _get_param_from_config(data_provider_config_json, [market], "data-markets") security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, diff --git a/lean/constants.py b/lean/constants.py index 0219369e..f2fe3482 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -107,15 +107,3 @@ # platforms MODULE_CLI_PLATFORM = "cli" MODULE_CLOUD_PLATFORM = "cloud" - -# Lean Resolution -RESOLUTIONS = ["Tick", "Second", "Minute", "Hour", "Daily"] - -# Lean Data Types -DATA_TYPES = ["Trade", "Quote", "OpenInterest"] - -# Lean Security Types -SECURITY_TYPES = [ "Equity", "Index", "Option", "IndexOption", "Commodity", "Forex", "Future", "Cfd", "Crypto", "FutureOption", "CryptoFuture" ] - -# Lean Data folder path, where keeps tickers data -DATA_FOLDER_PATH = "/Lean/Data" \ No newline at end of file diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 5bb44f5a..af6d736d 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -179,7 +179,7 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) - assert f"The {data_provider} data provider does not support {wrong_security_type}." in error_msg + assert f"The {data_provider} data provider does not support QCSecurityType.{wrong_security_type}." in error_msg @pytest.mark.parametrize("data_provider,start_date,end_date", @@ -202,7 +202,7 @@ def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) - assert f"The Polygon data provider does not support {wrong_data_type}." in error_msg + assert f"The Polygon data provider does not support QCDataType.{wrong_data_type}." in error_msg def test_non_interactive_bulk_select(): From 0767582cef293da5c8b5f22ab356fe867432e7b2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Apr 2024 00:36:17 +0300 Subject: [PATCH 41/53] remove: extra argument data-download-name in pass to exe dll --- lean/commands/data/download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index f32f90ef..497ceab6 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -655,7 +655,6 @@ def download(ctx: Context, run_options["working_dir"] = downloader_data_provider_path_dll dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", - "--data-downloader", data_downloader_provider.get_settings()[MODULE_DATA_DOWNLOADER], "--data-type", data_type, "--start-date", start_date.value.strftime("%Y%m%d"), "--end-date", end_date.value.strftime("%Y%m%d"), From 8db79288c4eb6de63e12a96309edafb7b86086c8 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Apr 2024 00:43:12 +0300 Subject: [PATCH 42/53] refactor: --no-update flag in local using --- lean/commands/data/download.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 497ceab6..ec489465 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -17,7 +17,7 @@ from click import command, option, confirm, pass_context, Context, Choice from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name -from lean.constants import DEFAULT_ENGINE_IMAGE, MODULE_DATA_DOWNLOADER +from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType from lean.models.click_options import get_configs_for_options, options_from_json @@ -517,6 +517,10 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - is_flag=True, default=False, help="Pull the LEAN engine image before running the Downloader Data Provider") +@option("--no-update", + is_flag=True, + default=False, + help="Use the local LEAN engine image instead of pulling the latest version") @pass_context def download(ctx: Context, data_provider_historical: Optional[str], @@ -533,6 +537,7 @@ def download(ctx: Context, end_date: Optional[str], image: Optional[str], update: bool, + no_update: bool, **kwargs) -> None: """Purchase and download data from QuantConnect Datasets. @@ -623,10 +628,8 @@ def download(ctx: Context, engine_image = container.cli_config_manager.get_engine_image(image) - no_update = False if str(engine_image) != DEFAULT_ENGINE_IMAGE: # Custom engine image should not be updated. - no_update = True logger.warn(f'A custom engine image: "{engine_image}" is being used!') container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) From 4dee04130830d260af66243b60be448c26922f5b Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Apr 2024 01:47:01 +0300 Subject: [PATCH 43/53] fix: back compatibility assert test --- tests/commands/data/test_download.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index af6d736d..abf1c2e6 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -179,7 +179,8 @@ def test_download_data_non_interactive_wrong_security_type(data_provider: str, w assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) - assert f"The {data_provider} data provider does not support QCSecurityType.{wrong_security_type}." in error_msg + assert data_provider in error_msg + assert wrong_security_type in error_msg @pytest.mark.parametrize("data_provider,start_date,end_date", @@ -194,7 +195,7 @@ def test_download_data_non_interactive_wrong_start_end_date(data_provider: str, assert f"Historical start date cannot be greater than or equal to historical end date." in error_msg -@pytest.mark.parametrize("wrong_data_type", [("OpenInterest")]) +@pytest.mark.parametrize("wrong_data_type", ["OpenInterest"]) def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): run_data_download = _create_lean_data_download("Polygon", wrong_data_type, "Hour", "Equity", ["AAPL"], "20240101", "20240202", _get_data_provider_config(), @@ -202,7 +203,7 @@ def test_download_data_non_interactive_wrong_data_type(wrong_data_type: str): assert run_data_download.exit_code == 1 error_msg = str(run_data_download.exc_info[1]) - assert f"The Polygon data provider does not support QCDataType.{wrong_data_type}." in error_msg + assert wrong_data_type in error_msg def test_non_interactive_bulk_select(): From c08a48ccd58c499800153fee5cbd245487a663e5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 26 Apr 2024 01:04:28 +0300 Subject: [PATCH 44/53] rename: DownloaderDataProvider.dll --- lean/commands/data/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index ec489465..7f25b84f 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -657,7 +657,7 @@ def download(ctx: Context, run_options["working_dir"] = downloader_data_provider_path_dll - dll_arguments = ["dotnet", "QuantConnect.Lean.DownloaderDataProvider.dll", + dll_arguments = ["dotnet", "QuantConnect.DownloaderDataProvider.Launcher.dll", "--data-type", data_type, "--start-date", start_date.value.strftime("%Y%m%d"), "--end-date", end_date.value.strftime("%Y%m%d"), From 4569009e32259116bf4536f268ce45ffb6a544d9 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 26 Apr 2024 19:41:01 +0300 Subject: [PATCH 45/53] feat: update readme --- README.md | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4d64a20b..acb08d97 100644 --- a/README.md +++ b/README.md @@ -781,12 +781,26 @@ _See code: [lean/commands/create_project.py](lean/commands/create_project.py)_ ### `lean data download` -Purchase and download data from QuantConnect Datasets. +Purchase and download data directly from QuantConnect or download from Support Data Providers ``` Usage: lean data download [OPTIONS] - Purchase and download data from QuantConnect Datasets. + Purchase and download data directly from QuantConnect or download from Support Data Providers + + 1. Acquire Data from QuantConnect Datasets: Purchase and seamlessly download data directly from QuantConnect. + + 2. Streamlined Access from Support Data Providers: + + - Choose your preferred historical data provider. + + - Initiate hassle-free downloads from our supported providers. + + We have 2 options: + + - interactive (follow instruction in lean-cli) + + - no interactive (write arguments in command line) An interactive wizard will show to walk you through the process of selecting data, accepting the CLI API Access and Data Agreement and payment. After this wizard the selected data will be downloaded automatically. @@ -799,12 +813,90 @@ Usage: lean data download [OPTIONS] https://www.quantconnect.com/datasets Options: - --dataset TEXT The name of the dataset to download non-interactively - --overwrite Overwrite existing local data - -y, --yes Automatically confirm payment confirmation prompts - --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) - --verbose Enable debug logging - --help Show this message and exit. + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit] + The name of the downloader data provider. + --ib-user-name TEXT Your Interactive Brokers username + --ib-account TEXT Your Interactive Brokers account id + --ib-password TEXT Your Interactive Brokers password + --ib-weekly-restart-utc-time TEXT + Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at + this time, and will require 2FA verification. This is required by Interactive Brokers. + Use this option explicitly to override the default value. + --oanda-account-id TEXT Your OANDA account id + --oanda-access-token TEXT Your OANDA API token + --oanda-environment [Practice|Trade] + The environment to run in, Practice for fxTrade Practice, Trade for fxTrade + --bitfinex-api-key TEXT Your Bitfinex API key + --bitfinex-api-secret TEXT Your Bitfinex API secret + --coinbase-api-key TEXT Your Coinbase Advanced Trade API key + --coinbase-api-secret TEXT Your Coinbase Advanced Trade API secret + --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures] + Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures] + --binance-api-key TEXT Your Binance API key + --binanceus-api-key TEXT Your Binance API key + --binance-api-secret TEXT Your Binance API secret + --binanceus-api-secret TEXT Your Binance API secret + --kraken-api-key TEXT Your Kraken API key + --kraken-api-secret TEXT Your Kraken API secret + --kraken-verification-tier [Starter|Intermediate|Pro] + Your Kraken Verification Tier + --iqfeed-iqconnect TEXT The path to the IQConnect binary + --iqfeed-username TEXT Your IQFeed username + --iqfeed-password TEXT Your IQFeed password + --iqfeed-version TEXT The product version of your IQFeed developer account + --iqfeed-host TEXT The IQFeed host address + --polygon-api-key TEXT Your Polygon.io API Key + --factset-auth-config-file FILE + The path to the FactSet authentication configuration file + --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key + --iex-price-plan [Launch|Grow|Enterprise] + Your IEX Cloud Price plan + --alpha-vantage-api-key TEXT Your Alpha Vantage Api Key + --alpha-vantage-price-plan [Free|Plan30|Plan75|Plan150|Plan300|Plan600|Plan1200] + Your Alpha Vantage Premium API Key plan + --coinapi-api-key TEXT Your coinapi.io Api Key + --coinapi-product [Free|Startup|Streamer|Professional|Enterprise] + CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing) + --thetadata-ws-url TEXT The ThetaData host address + --thetadata-rest-url TEXT The ThetaData host address + --thetadata-subscription-plan [Free|Value|Standard|Pro] + Your ThetaData subscription price plan + --terminal-link-connection-type [DAPI|SAPI] + Terminal Link Connection Type [DAPI, SAPI] + --terminal-link-server-auth-id TEXT + The Auth ID of the TerminalLink server + --terminal-link-environment [Production|Beta] + The environment to run in + --terminal-link-server-host TEXT + The host of the TerminalLink server + --terminal-link-server-port INTEGER + The port of the TerminalLink server + --terminal-link-openfigi-api-key TEXT + The Open FIGI API key to use for mapping options + --bybit-api-key TEXT Your Bybit API key + --bybit-api-secret TEXT Your Bybit API secret + --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] + Your Bybit VIP Level + --dataset TEXT The name of the dataset to download non-interactively + --overwrite Overwrite existing local data + -y, --yes Automatically confirm payment confirmation prompts + --data-type [Trade|Quote|OpenInterest] + Specify the type of historical data + --resolution [Tick|Second|Minute|Hour|Daily] + Specify the resolution of the historical data + --security-type [Equity|Index|Forex|Cfd|Future|Crypto|CryptoFuture|Option|IndexOption|Commodity|FutureOption] + Specify the security type of the historical data + --market TEXT Specify the market name for tickers (e.g., 'USA', 'NYMEX', 'Binance') + --tickers TEXT Specify comma separated list of tickers to use for historical data request. + --start-date TEXT Specify the start date for the historical data request in the format yyyyMMdd. + --end-date TEXT Specify the end date for the historical data request in the format yyyyMMdd. (defaults + to today) + --image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest) + --update Pull the LEAN engine image before running the Downloader Data Provider + --no-update Use the local LEAN engine image instead of pulling the latest version + --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) + --verbose Enable debug logging + --help Show this message and exit. ``` _See code: [lean/commands/data/download.py](lean/commands/data/download.py)_ From f0a70d23112c6ca5ec47542eab9c9114ee6f2a12 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 26 Apr 2024 19:44:03 +0300 Subject: [PATCH 46/53] feat: description of data download command --- lean/commands/data/download.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 7f25b84f..1788b8ff 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -539,7 +539,16 @@ def download(ctx: Context, update: bool, no_update: bool, **kwargs) -> None: - """Purchase and download data from QuantConnect Datasets. + """Purchase and download data directly from QuantConnect or download from Support Data Providers + + 1. Acquire Data from QuantConnect Datasets: Purchase and seamlessly download data directly from QuantConnect.\n + 2. Streamlined Access from Support Data Providers:\n + - Choose your preferred historical data provider.\n + - Initiate hassle-free downloads from our supported providers. + + We have 2 options:\n + - interactive (follow instruction in lean-cli)\n + - no interactive (write arguments in command line) An interactive wizard will show to walk you through the process of selecting data, accepting the CLI API Access and Data Agreement and payment. @@ -623,7 +632,7 @@ def download(ctx: Context, cli_data_downloaders, kwargs, logger, interactive=False) data_downloader_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) - # Info: I don't understand why it returns empty result + # mounting additional data_downloader config files paths_to_mount = data_downloader_provider.get_paths_to_mount() engine_image = container.cli_config_manager.get_engine_image(image) From 01c541ced4c4146d4c50d5fee166cfb73a6d6408 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 27 Apr 2024 00:58:41 +0300 Subject: [PATCH 47/53] fix: mounting path of Data Providers' config --- lean/commands/data/download.py | 11 ++--- lean/components/docker/lean_runner.py | 61 ++++++++++++++------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 1788b8ff..8ae3a647 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -12,6 +12,7 @@ # limitations under the License. from json import dump + from docker.types import Mount from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice @@ -653,17 +654,17 @@ def download(ctx: Context, } config.update(data_downloader_provider.get_settings()) - config_path = container.temp_manager.create_temporary_directory() / "config.json" - with config_path.open("w+", encoding="utf-8") as file: - dump(config, file) - - run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, + run_options = container.lean_runner.get_basic_docker_config_without_algo(config, debugging_method=None, detach=False, image=engine_image, target_path=downloader_data_provider_path_dll, paths_to_mount=paths_to_mount) + config_path = container.temp_manager.create_temporary_directory() / "config.json" + with config_path.open("w+", encoding="utf-8") as file: + dump(config, file) + run_options["working_dir"] = downloader_data_provider_path_dll dll_arguments = ["dotnet", "QuantConnect.DownloaderDataProvider.Launcher.dll", diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index fc0df42a..699c1b30 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -211,7 +211,7 @@ def get_basic_docker_config(self, # Create the output directory if it doesn't exist yet # Create the storage directory if it doesn't exist yet self._ensure_directories_exist([output_dir, storage_dir]) - + lean_config.update({ "debug-mode": self._logger.debug_logging_enabled, "data-folder": "/Lean/Data", @@ -224,7 +224,7 @@ def get_basic_docker_config(self, self._mount_common_directories(run_options, paths_to_mount, lean_config, data_dir, storage_dir, project_dir, output_dir) # Update all hosts that need to point to the host's localhost to host.docker.internal so they resolve properly - # TODO: we should remove it or add to config json + # TODO: we should remove it or add to config json for key in ["terminal-link-server-host"]: if key not in lean_config: continue @@ -285,7 +285,7 @@ def get_basic_docker_config_without_algo(self, self._mount_common_directories(run_options, paths_to_mount, lean_config, data_dir, storage_dir, None, None) # Update all hosts that need to point to the host's localhost to host.docker.internal so they resolve properly - # TODO: we should remove it or add to config json + # TODO: we should remove it or add to config json for key in ["terminal-link-server-host"]: if key not in lean_config: continue @@ -299,7 +299,7 @@ def get_basic_docker_config_without_algo(self, self._mount_lean_config_and_finalize(run_options, lean_config, None) return run_options - + def _mount_lean_config_and_finalize(self, run_options: Dict[str, Any], lean_config: Dict[str, Any], output_dir: Optional[Path]): """Mounts Lean config and finalizes.""" from docker.types import Mount @@ -310,16 +310,16 @@ def _mount_lean_config_and_finalize(self, run_options: Dict[str, Any], lean_conf config_path = self._temp_manager.create_temporary_directory() / "config.json" with config_path.open("w+", encoding="utf-8") as file: file.write(dumps(lean_config, indent=4)) - + # Mount the Lean config run_options["mounts"].append(Mount(target=f"{LEAN_ROOT_PATH}/config.json", source=str(config_path), type="bind", read_only=True)) - + # Assign the container a name and store it in the output directory's configuration run_options["name"] = lean_config.get("container-name", f"lean_cli_{str(uuid4()).replace('-', '')}") - + # set the hostname if "hostname" in lean_config: run_options["hostname"] = lean_config["hostname"] @@ -333,7 +333,7 @@ def _mount_lean_config_and_finalize(self, run_options: Dict[str, Any], lean_conf environment = lean_config["environments"][lean_config["environment"]] if "live-mode-brokerage" in environment: output_config.set("brokerage", environment["live-mode-brokerage"].split(".")[-1]) - + def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerImage, target_path: str = "/Lean/Launcher/bin/Debug"): """Sets up installed packages.""" installed_packages = self._module_manager.get_installed_packages() @@ -349,7 +349,7 @@ def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerIm # Create a C# project used to resolve the dependencies of the modules run_options["commands"].append("mkdir /ModulesProject") run_options["commands"].append("dotnet new sln -o /ModulesProject") - + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', DEFAULT_LEAN_DOTNET_FRAMEWORK) run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") run_options["commands"].append("rm /ModulesProject/Class1.cs") @@ -359,19 +359,19 @@ def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerIm self._logger.debug(f"LeanRunner._setup_installed_packages(): Adding module {package} to the project") run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") run_options["commands"].append(f"dotnet add /ModulesProject package {package.name} --version {package.version}") - + # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists run_options["commands"].append("python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") - + return bool(installed_packages) - - def _mount_common_directories(self, - run_options: Dict[str, Any], - paths_to_mount: Optional[Dict[str, str]], - lean_config: Dict[str, Any], + + def _mount_common_directories(self, + run_options: Dict[str, Any], + paths_to_mount: Optional[Dict[str, str]], + lean_config: Dict[str, Any], data_dir: Path, storage_dir: Path, - project_dir: Optional[Path], + project_dir: Optional[Path], output_dir: Optional[Path]): """ Mounts common directories. @@ -387,20 +387,20 @@ def _mount_common_directories(self, # 1 self.mount_paths(paths_to_mount, lean_config, run_options) - + # 2 if project_dir: self.mount_project_and_library_directories(project_dir, run_options) - + # 3 run_options["volumes"][str(data_dir)] = {"bind": "/Lean/Data", "mode": "rw"} - + # 4 if output_dir: run_options["volumes"][str(output_dir)] = {"bind": "/Results", "mode": "rw"} # 5 - run_options["volumes"][str(storage_dir)] = { "bind": "/Storage", "mode": "rw"} - + run_options["volumes"][str(storage_dir)] = {"bind": "/Storage", "mode": "rw"} + # 6 cli_root_dir = self._lean_config_manager.get_cli_root_directory() files_to_mount = [ @@ -423,11 +423,12 @@ def _mount_common_directories(self, read_only=False)) lean_config[key] = f"/Files/{key}" - + + def _initialize_run_options(self, detach: bool, docker_project_config: Dict[str, Any], debugging_method: Optional[DebuggingMethod]): """ Initializes run options. - + The dict containing all options passed to `docker run` See all available options at https://docker-py.readthedocs.io/en/stable/containers.html """ @@ -440,13 +441,13 @@ def _initialize_run_options(self, detach: bool, docker_project_config: Dict[str, "volumes": {}, "ports": docker_project_config.get("ports", {}) } - + def _ensure_directories_exist(self, dirs: List[Path]): """Ensures directories exist.""" for dir_path in dirs: if not dir_path.exists(): dir_path.mkdir(parents=True) - + def _handle_data_providers(self, lean_config: Dict[str, Any], data_dir: Path): """Handles data provider logic.""" if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": @@ -637,7 +638,7 @@ def set_up_common_csharp_options(self, run_options: Dict[str, Any], target_path: """ Sets up common Docker run options that is needed for all C# work. - This method prepares the Docker run options required to run C# projects inside a Docker container. It is called + This method prepares the Docker run options required to run C# projects inside a Docker container. It is called when the user has installed specific modules or when the project to run is written in C#. Parameters: @@ -719,7 +720,7 @@ def copy_file(library_id, partial_path, file_data): output_name = file_data.get("outputPath", full_path.name) - target_path = Path("""+ f'"{target_path}"' +""") / output_name + target_path = Path(""" + f'"{target_path}"' + """) / output_name if not target_path.exists(): target_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy(full_path, target_path) @@ -895,7 +896,6 @@ def mount_paths(self, paths_to_mount, lean_config, run_options): environment = {} if "environment" in lean_config and "environments" in lean_config: environment = lean_config["environments"][lean_config["environment"]] - mounts = run_options["mounts"] for key, pathStr in paths_to_mount.items(): @@ -909,6 +909,9 @@ def mount_paths(self, paths_to_mount, lean_config, run_options): type="bind", read_only=True)) environment[key] = target + # update target in config too (not only in environment above) + if key in lean_config: + lean_config[key] = target @staticmethod def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None: From 29037d93853504499e2b9261e0e98a8224526b22 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 27 Apr 2024 01:15:55 +0300 Subject: [PATCH 48/53] feat: if lean_config without environment use lean_config in mount_paths --- lean/components/docker/lean_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 699c1b30..da770b64 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -896,6 +896,8 @@ def mount_paths(self, paths_to_mount, lean_config, run_options): environment = {} if "environment" in lean_config and "environments" in lean_config: environment = lean_config["environments"][lean_config["environment"]] + else: + environment = lean_config mounts = run_options["mounts"] for key, pathStr in paths_to_mount.items(): From 89ca968d61480b5543bd4345a3f889b33158baf2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 27 Apr 2024 01:22:15 +0300 Subject: [PATCH 49/53] feat: info to debug log in mount --- lean/components/docker/lean_runner.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index da770b64..4bbe3849 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -904,16 +904,13 @@ def mount_paths(self, paths_to_mount, lean_config, run_options): path = Path(pathStr).resolve() target = f"/Files/{Path(path).name}" - self._logger.info(f"Mounting {path} to {target}") + self._logger.debug(f"Mounting {path} to {target}") mounts.append(Mount(target=target, source=str(path), type="bind", read_only=True)) environment[key] = target - # update target in config too (not only in environment above) - if key in lean_config: - lean_config[key] = target @staticmethod def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None: From 21eb2c5f5f79c0acd40762985f2461df69668d2c Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 27 Apr 2024 02:15:27 +0300 Subject: [PATCH 50/53] feat: user friendly prompt end-date --- lean/commands/data/download.py | 16 +++++++++++----- lean/models/data.py | 5 +++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 8ae3a647..1868546a 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from datetime import datetime from json import dump from docker.types import Mount @@ -464,6 +464,7 @@ def _get_user_input_or_prompt(user_input_data: str, available_input_data: List[s return user_input_data + def _configure_date_option(date_value: str, option_id: str, option_label: str) -> OptionResult: """ Configure the date based on the provided date value, option ID, and option label. @@ -477,10 +478,15 @@ def _configure_date_option(date_value: str, option_id: str, option_label: str) - - str: Configured date. """ - date_option = DatasetDateOption(id=option_id, label=option_label, description=f"Enter the {option_label} for the historical data request in the format YYYYMMDD.") + date_option = DatasetDateOption(id=option_id, label=option_label, + description=f"Enter the {option_label} " + f"for the historical data request in the format YYYYMMDD.") if not date_value: - return date_option.configure_interactive() + if option_id == "end": + return date_option.configure_interactive_with_default(datetime.today().strftime("%Y%m%d")) + else: + return date_option.configure_interactive() return date_option.configure_non_interactive(date_value) @@ -620,8 +626,8 @@ def download(ctx: Context, transform=DatasetTextOptionTransform.Lowercase, multiple=True).configure_interactive().value) - start_date = _configure_date_option(start_date, "start", "Start date") - end_date = _configure_date_option(end_date, "end", "End date") + start_date = _configure_date_option(start_date, "start", "Please enter a Start Date in the format") + end_date = _configure_date_option(end_date, "end", "Please enter a End Date in the format") if start_date.value >= end_date.value: raise ValueError("Historical start date cannot be greater than or equal to historical end date.") diff --git a/lean/models/data.py b/lean/models/data.py index 19d4dcec..4e9e72f8 100644 --- a/lean/models/data.py +++ b/lean/models/data.py @@ -230,6 +230,11 @@ def configure_interactive(self) -> OptionResult: date = prompt(f"{self.label} (yyyyMMdd)", type=DateParameter()) return OptionResult(value=date, label=date.strftime("%Y-%m-%d")) + def configure_interactive_with_default(self, default_date: str) -> OptionResult: + date = prompt(f"{self.label} (yyyyMMdd) or just press Enter to use the default date [{default_date}]", + show_default=False, default=default_date, type=DateParameter()) + return OptionResult(value=date, label=date.strftime("%Y-%m-%d")) + def configure_non_interactive(self, user_input: str) -> OptionResult: for date_format in ["%Y%m%d", "%Y-%m-%d"]: try: From a902ac61764770d6908ce332f0c8e3b445b7ac0c Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 29 Apr 2024 21:35:16 +0300 Subject: [PATCH 51/53] fix: get all config for data downloader refactor: get_complete_lean_config without environment and algo_file remove: test, cuz we have used interactive input for user --- lean/commands/data/download.py | 16 ++---- lean/components/config/lean_config_manager.py | 54 ++++++++++--------- tests/commands/data/test_download.py | 15 ------ 3 files changed, 33 insertions(+), 52 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 1868546a..9f5427cb 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -633,10 +633,10 @@ def download(ctx: Context, raise ValueError("Historical start date cannot be greater than or equal to historical end date.") logger = container.logger - lean_config = container.lean_config_manager.get_lean_config() + lean_config = container.lean_config_manager.get_complete_lean_config(None, None, None) data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), - cli_data_downloaders, kwargs, logger, interactive=False) + cli_data_downloaders, kwargs, logger, interactive=True) data_downloader_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) # mounting additional data_downloader config files @@ -652,15 +652,7 @@ def download(ctx: Context, downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" - # Create config dictionary with credentials - config: Dict[str, str] = { - "job-user-id": lean_config.get("job-user-id"), - "api-access-token": lean_config.get("api-access-token"), - "job-organization-id": organization.id - } - config.update(data_downloader_provider.get_settings()) - - run_options = container.lean_runner.get_basic_docker_config_without_algo(config, + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, debugging_method=None, detach=False, image=engine_image, @@ -669,7 +661,7 @@ def download(ctx: Context, config_path = container.temp_manager.create_temporary_directory() / "config.json" with config_path.open("w+", encoding="utf-8") as file: - dump(config, file) + dump(lean_config, file) run_options["working_dir"] = downloader_data_provider_path_dll diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index a33e7bc7..5b533386 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -223,7 +223,9 @@ def get_complete_lean_config(self, """ config = self.get_lean_config() - config["environment"] = environment + if environment and len(environment) > 0: + config["environment"] = environment + config["close-automatically"] = True config["composer-dll-directory"] = "." @@ -241,7 +243,6 @@ def get_complete_lean_config(self, config_defaults = { "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), - "job-project-id": self._project_config_manager.get_local_id(algorithm_file.parent), "job-organization-id": get_organization(config), "ib-host": "127.0.0.1", @@ -256,29 +257,32 @@ def get_complete_lean_config(self, if config.get(key, "") == "": config[key] = value - if algorithm_file.name.endswith(".py"): - config["algorithm-type-name"] = algorithm_file.name.split(".")[0] - config["algorithm-language"] = "Python" - config["algorithm-location"] = f"/LeanCLI/{algorithm_file.name}" - else: - from re import findall - algorithm_text = algorithm_file.read_text(encoding="utf-8") - config["algorithm-type-name"] = findall(r"class\s*([^\s:]+)\s*:\s*QCAlgorithm", algorithm_text)[0] - config["algorithm-language"] = "CSharp" - config["algorithm-location"] = f"{algorithm_file.parent.name}.dll" - - project_config = self._project_config_manager.get_project_config(algorithm_file.parent) - config["parameters"] = project_config.get("parameters", {}) - - # Add libraries paths to python project - project_language = project_config.get("algorithm-language", None) - if project_language == "Python": - library_references = project_config.get("libraries", []) - python_paths = config.get("python-additional-paths", []) - python_paths.extend([(Path("/") / library["path"]).as_posix() for library in library_references]) - if len(python_paths) > 0: - python_paths.append("/Library") - config["python-additional-paths"] = python_paths + if algorithm_file and len(algorithm_file.name) > 0: + config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) + + if algorithm_file.name.endswith(".py"): + config["algorithm-type-name"] = algorithm_file.name.split(".")[0] + config["algorithm-language"] = "Python" + config["algorithm-location"] = f"/LeanCLI/{algorithm_file.name}" + else: + from re import findall + algorithm_text = algorithm_file.read_text(encoding="utf-8") + config["algorithm-type-name"] = findall(r"class\s*([^\s:]+)\s*:\s*QCAlgorithm", algorithm_text)[0] + config["algorithm-language"] = "CSharp" + config["algorithm-location"] = f"{algorithm_file.parent.name}.dll" + + project_config = self._project_config_manager.get_project_config(algorithm_file.parent) + config["parameters"] = project_config.get("parameters", {}) + + # Add libraries paths to python project + project_language = project_config.get("algorithm-language", None) + if project_language == "Python": + library_references = project_config.get("libraries", []) + python_paths = config.get("python-additional-paths", []) + python_paths.extend([(Path("/") / library["path"]).as_posix() for library in library_references]) + if len(python_paths) > 0: + python_paths.append("/Library") + config["python-additional-paths"] = python_paths # No real limit for the object store by default if "storage-limit-mb" not in config: diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index abf1c2e6..48c8ce2c 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -146,21 +146,6 @@ def test_download_data_non_interactive(data_provider: str, market: str, is_crypt assert run_data_download.exit_code == 0 -@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,missed_parameters", - [("Polygon", "NYSE", False, "Equity", "--polygon-api-key"), - ("Binance", "Binance", True, "Crypto", "--binance-exchange-name"), - ("Interactive Brokers", "USA", False, "Equity", - "--ib-user-name, --ib-account, --ib-password")]) -def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, market: str, is_crypto: bool, - security_type: str, missed_parameters: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", security_type, ["AAPL"], - "20240101", "20240202", _get_data_provider_config(is_crypto), market) - assert run_data_download.exit_code == 1 - - error_msg = str(run_data_download.exc_info[1]) - assert missed_parameters in error_msg - - @pytest.mark.parametrize("data_type,resolution", [("Trade", "Hour"), ("trade", "hour"), ("TRADE", "HOUR"), ("TrAdE", "HoUr")]) def test_download_data_non_interactive_insensitive_input_param(data_type: str, resolution: str): From 14c4638fb7479f5c6970b104e67e01b9c588c061 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 30 Apr 2024 00:43:06 +0300 Subject: [PATCH 52/53] refactor: specifications_url (like property) feat: get access to inner support of data sources test: refactor by new format --- lean/commands/data/download.py | 43 ++++++++++++++++------------ lean/models/json_module.py | 4 +++ tests/commands/data/test_download.py | 26 ++++++++++------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 9f5427cb..3cb5d819 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -415,7 +415,9 @@ def _get_available_datasets(organization: QCFullOrganization) -> List[Dataset]: def _get_historical_data_provider() -> str: return container.logger.prompt_list("Select a historical data provider", [Option(id=data_downloader.get_name(), label=data_downloader.get_name()) for data_downloader in cli_data_downloaders]) -def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], key_config_data: str) -> List[str]: + +def _get_download_specification_from_config(data_provider_config_json: Dict[str, Any], default_param: List[str], + key_config_data: str) -> List[str]: """ Get parameter from data provider config JSON or return default parameters. @@ -427,10 +429,12 @@ def _get_param_from_config(data_provider_config_json: Dict[str, Any], default_pa Returns: - List[str]: List of parameters. """ - if data_provider_config_json is None: - return default_param - return data_provider_config_json.get(key_config_data, default_param) + if data_provider_config_json and "module-specification" in data_provider_config_json: + if "download" in data_provider_config_json["module-specification"]: + return data_provider_config_json["module-specification"]["download"].get(key_config_data, default_param) + + return default_param def _get_user_input_or_prompt(user_input_data: str, available_input_data: List[str], data_provider_name: str, @@ -593,22 +597,25 @@ def download(ctx: Context, all_data_files = _get_data_files(organization, products) container.data_downloader.download_files(all_data_files, overwrite, organization.id) else: - data_downloader_provider = next(data_downloader for data_downloader in cli_data_downloaders if data_downloader.get_name() == data_provider_historical) + data_downloader_provider = next(data_downloader for data_downloader in cli_data_downloaders + if data_downloader.get_name() == data_provider_historical) data_provider_config_json = None - if data_downloader_provider._specifications_url is not None: - data_provider_config_json = container.api_client.data.download_public_file_json(data_downloader_provider._specifications_url) - - data_provider_support_security_types = _get_param_from_config(data_provider_config_json, - QCSecurityType.get_all_members(), - "data-supported") - data_provider_support_data_types = _get_param_from_config(data_provider_config_json, - QCDataType.get_all_members(), - "data-types") - data_provider_support_resolutions = _get_param_from_config(data_provider_config_json, - QCResolution.get_all_members(), - "data-resolutions") - data_provider_support_markets = _get_param_from_config(data_provider_config_json, [market], "data-markets") + if data_downloader_provider.specifications_url is not None: + data_provider_config_json = container.api_client.data.download_public_file_json( + data_downloader_provider.specifications_url) + + data_provider_support_security_types = _get_download_specification_from_config(data_provider_config_json, + QCSecurityType.get_all_members(), + "data-supported") + data_provider_support_data_types = _get_download_specification_from_config(data_provider_config_json, + QCDataType.get_all_members(), + "data-types") + data_provider_support_resolutions = _get_download_specification_from_config(data_provider_config_json, + QCResolution.get_all_members(), + "data-resolutions") + data_provider_support_markets = _get_download_specification_from_config(data_provider_config_json, + [market], "data-markets") security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") diff --git a/lean/models/json_module.py b/lean/models/json_module.py index ba0d6a43..14bde708 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -262,6 +262,10 @@ def _save_property(self, settings: Dict[str, Any]): from lean.container import container container.lean_config_manager.set_properties(settings) + @property + def specifications_url(self): + return self._specifications_url + class LiveInitialStateInput(str, Enum): Required = "required" diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index 48c8ce2c..c428218c 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -51,19 +51,25 @@ def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any] if is_crypto_configs: return { - "data-types": ["Trade", "Quote"], - "data-resolutions": ["Minute", "Hour", "Daily"], - "data-supported": ["Crypto", "CryptoFuture"], - "data-markets": ["Binance", "Kraken"] + "module-specification": { + "download": { + "data-types": ["Trade", "Quote"], + "data-resolutions": ["Minute", "Hour", "Daily"], + "data-supported": ["Crypto", "CryptoFuture"], + "data-markets": ["Binance", "Kraken"] + } + } } data_provider_config_file_json: Dict[str, Any] = { - "data-types": ["Trade", "Quote"], # Supported data types: Trade and Quote - "data-resolutions": ["Second", "Minute", "Hour", "Daily"], - # Supported data resolutions: Second, Minute, Hour, Daily - "data-supported": ["Equity", "Option", "Index", "IndexOption"], - # Supported asset classes: Equity, Equity Options, Indexes, Index Options - "data-markets": ["NYSE", "USA"] + "module-specification": { + "download": { + "data-types": ["Trade", "Quote"], + "data-resolutions": ["Second", "Minute", "Hour", "Daily"], + "data-supported": ["Equity", "Option", "Index", "IndexOption"], + "data-markets": ["NYSE", "USA"] + } + } } return data_provider_config_file_json From 451ca0f7f63cfb234fea508121e15cf6bdb27321 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 30 Apr 2024 15:29:24 +0300 Subject: [PATCH 53/53] rename: module specification properties --- lean/commands/data/download.py | 6 +++--- tests/commands/data/test_download.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 3cb5d819..bbc24a17 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -607,15 +607,15 @@ def download(ctx: Context, data_provider_support_security_types = _get_download_specification_from_config(data_provider_config_json, QCSecurityType.get_all_members(), - "data-supported") + "security-types") data_provider_support_data_types = _get_download_specification_from_config(data_provider_config_json, QCDataType.get_all_members(), "data-types") data_provider_support_resolutions = _get_download_specification_from_config(data_provider_config_json, QCResolution.get_all_members(), - "data-resolutions") + "resolutions") data_provider_support_markets = _get_download_specification_from_config(data_provider_config_json, - [market], "data-markets") + [market], "markets") security_type = _get_user_input_or_prompt(security_type, data_provider_support_security_types, data_provider_historical, "Select a Ticker's security type") diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index c428218c..a5c1e6b5 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -54,9 +54,9 @@ def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any] "module-specification": { "download": { "data-types": ["Trade", "Quote"], - "data-resolutions": ["Minute", "Hour", "Daily"], - "data-supported": ["Crypto", "CryptoFuture"], - "data-markets": ["Binance", "Kraken"] + "resolutions": ["Minute", "Hour", "Daily"], + "security-types": ["Crypto", "CryptoFuture"], + "markets": ["Binance", "Kraken"] } } } @@ -65,9 +65,9 @@ def _get_data_provider_config(is_crypto_configs: bool = False) -> Dict[str, Any] "module-specification": { "download": { "data-types": ["Trade", "Quote"], - "data-resolutions": ["Second", "Minute", "Hour", "Daily"], - "data-supported": ["Equity", "Option", "Index", "IndexOption"], - "data-markets": ["NYSE", "USA"] + "resolutions": ["Second", "Minute", "Hour", "Daily"], + "security-types": ["Equity", "Option", "Index", "IndexOption"], + "markets": ["NYSE", "USA"] } } }