From f99344cbae03f582de855ee70cdf3f232fc2302b Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Thu, 24 Aug 2023 03:56:10 +0530 Subject: [PATCH] Feature Implement Object Store Commands (#356) * initial implementation * object store readme updates * cleanup * add tests * self review * fix alignment * fix style and cleanup * add alias for get command * add local object store commands * update readme and tests * fix readme * address review * fix tests * fix tests --- README.md | 199 +++++++++++++++++- lean/commands/__init__.py | 2 + lean/commands/cloud/__init__.py | 3 +- lean/commands/cloud/object_store/__init__.py | 24 +++ lean/commands/cloud/object_store/delete.py | 29 +++ lean/commands/cloud/object_store/get.py | 55 +++++ lean/commands/cloud/object_store/list.py | 44 ++++ .../cloud/object_store/object_store.py | 23 ++ lean/commands/cloud/object_store/set.py | 34 +++ lean/commands/object_store/__init__.py | 24 +++ lean/commands/object_store/delete.py | 26 +++ lean/commands/object_store/get.py | 30 +++ lean/commands/object_store/list.py | 26 +++ lean/commands/object_store/object_store.py | 23 ++ lean/commands/object_store/set.py | 26 +++ lean/components/api/api_client.py | 7 +- lean/components/api/object_store_client.py | 87 ++++++++ lean/components/util/object_store_helper.py | 26 +++ .../cloud/object_store/test_delete.py | 30 +++ tests/commands/cloud/object_store/test_get.py | 30 +++ .../commands/cloud/object_store/test_list.py | 30 +++ tests/commands/cloud/object_store/test_set.py | 37 ++++ 22 files changed, 812 insertions(+), 3 deletions(-) create mode 100644 lean/commands/cloud/object_store/__init__.py create mode 100644 lean/commands/cloud/object_store/delete.py create mode 100644 lean/commands/cloud/object_store/get.py create mode 100644 lean/commands/cloud/object_store/list.py create mode 100644 lean/commands/cloud/object_store/object_store.py create mode 100644 lean/commands/cloud/object_store/set.py create mode 100644 lean/commands/object_store/__init__.py create mode 100644 lean/commands/object_store/delete.py create mode 100644 lean/commands/object_store/get.py create mode 100644 lean/commands/object_store/list.py create mode 100644 lean/commands/object_store/object_store.py create mode 100644 lean/commands/object_store/set.py create mode 100644 lean/components/api/object_store_client.py create mode 100644 lean/components/util/object_store_helper.py create mode 100644 tests/commands/cloud/object_store/test_delete.py create mode 100644 tests/commands/cloud/object_store/test_get.py create mode 100644 tests/commands/cloud/object_store/test_list.py create mode 100644 tests/commands/cloud/object_store/test_set.py diff --git a/README.md b/README.md index b5eec7d5..a3024c2c 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,11 @@ A locally-focused workflow (local development, local execution) with the CLI may - [`lean cloud live deploy`](#lean-cloud-live-deploy) - [`lean cloud live liquidate`](#lean-cloud-live-liquidate) - [`lean cloud live stop`](#lean-cloud-live-stop) +- [`lean cloud object-store delete`](#lean-cloud-object-store-delete) +- [`lean cloud object-store get`](#lean-cloud-object-store-get) +- [`lean cloud object-store list`](#lean-cloud-object-store-list) +- [`lean cloud object-store ls`](#lean-cloud-object-store-ls) +- [`lean cloud object-store set`](#lean-cloud-object-store-set) - [`lean cloud optimize`](#lean-cloud-optimize) - [`lean cloud pull`](#lean-cloud-pull) - [`lean cloud push`](#lean-cloud-push) @@ -101,6 +106,11 @@ A locally-focused workflow (local development, local execution) with the CLI may - [`lean login`](#lean-login) - [`lean logout`](#lean-logout) - [`lean logs`](#lean-logs) +- [`lean object-store delete`](#lean-object-store-delete) +- [`lean object-store get`](#lean-object-store-get) +- [`lean object-store list`](#lean-object-store-list) +- [`lean object-store ls`](#lean-object-store-ls) +- [`lean object-store set`](#lean-object-store-set) - [`lean optimize`](#lean-optimize) - [`lean project-create`](#lean-project-create) - [`lean project-delete`](#lean-project-delete) @@ -374,6 +384,88 @@ Options: _See code: [lean/commands/cloud/live/stop.py](lean/commands/cloud/live/stop.py)_ +### `lean cloud object-store delete` + +Delete a value from the organization's cloud object store. + +``` +Usage: lean cloud object-store delete [OPTIONS] KEY + + Delete a value from the organization's cloud object store. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/delete.py](lean/commands/cloud/object_store/delete.py)_ + +### `lean cloud object-store get` + +Get a value from the organization's cloud object store. + +``` +Usage: lean cloud object-store get [OPTIONS] KEY + + Get a value from the organization's cloud object store. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/get.py](lean/commands/cloud/object_store/get.py)_ + +### `lean cloud object-store list` + +List all values for the given root key in the organization's cloud object store. + +``` +Usage: lean cloud object-store list [OPTIONS] [KEY] + + List all values for the given root key in the organization's cloud object store. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/list.py](lean/commands/cloud/object_store/list.py)_ + +### `lean cloud object-store ls` + +Alias for 'list' + +``` +Usage: lean cloud object-store ls [OPTIONS] [KEY] + + List all values for the given root key in the organization's cloud object store. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/ls.py](lean/commands/cloud/object_store/ls.py)_ + +### `lean cloud object-store set` + +Sets the data to the given key in the organization's cloud object store. + +``` +Usage: lean cloud object-store set [OPTIONS] KEY PATH + + Sets the data to the given key in the organization's cloud object store. + + :param key: The key to set the data to. :param path: Path to the file containing the object data. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/set.py](lean/commands/cloud/object_store/set.py)_ + ### `lean cloud optimize` Optimize a project in the cloud. @@ -638,7 +730,7 @@ Usage: lean data generate [OPTIONS] Options: --start [yyyyMMdd] Start date for the data to generate in yyyyMMdd format [required] --end [yyyyMMdd] End date for the data to generate in yyyyMMdd format (defaults to today) - --symbol-count INTEGER RANGE The number of symbols to generate data for [x>=0; required] + --symbol-count INTEGER RANGE The number of symbols to generate data for [x>=0] --tickers TEXT Comma separated list of tickers to use for generated data --security-type [Equity|Forex|Cfd|Future|Crypto|Option] The security type to generate data for (defaults to Equity) @@ -648,6 +740,31 @@ Options: The density of the generated data (defaults to Dense) --include-coarse BOOLEAN Whether coarse universe data should be generated for Equity data (defaults to True) --market TEXT The market to generate data for (defaults to standard market for the security type) + --quote-trade-ratio FLOAT The ratio of generated quotes to generated trades. Values larger than 1 mean more + quotes than trades. Only used for Option, Future and Crypto (defaults to 1) + --random-seed INTEGER RANGE The random number generator seed. Defaults to None, which means no seed will be used + [x>=0] + --ipo-percentage FLOAT The probability each equity generated will have an IPO event. Note that this is not + the total probability for all symbols generated. Only used for Equity (defaults to + 5.0) + --rename-percentage FLOAT The probability each equity generated will have a rename event. Note that this is not + the total probability for all symbols generated. Only used for Equity (defaults to + 30.0) + --splits-percentage FLOAT The probability each equity generated will have a stock split event. Note that this is + not the total probability for all symbols generated. Only used for Equity (defaults to + 15.0) + --dividends-percentage FLOAT The probability each equity generated will have dividends. Note that this is not the + probability for all symbols genearted. Only used for Equity (defaults to 60.0) + --dividend-every-quarter-percentage FLOAT + The probability each equity generated will have a dividend event every quarter. Note + that this is not the total probability for all symbols generated. Only used for Equity + (defaults to 30.0) + --option-price-engine TEXT The stochastic process, and returns new pricing engine to run calculations for that + option (defaults to BaroneAdesiWhaleyApproximationEngine) + --volatility-model-resolution [Tick|Second|Minute|Hour|Daily] + The volatility model period span (defaults to Daily) + --chain-symbol-count INTEGER RANGE + The size of the option chain (defaults to 10) [x>=0] --image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest) --update Pull the LEAN engine image before running the generator --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) @@ -1129,6 +1246,86 @@ Options: _See code: [lean/commands/logs.py](lean/commands/logs.py)_ +### `lean object-store delete` + +Opens the local storage directory in the file explorer. + +``` +Usage: lean object-store delete [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/delete.py](lean/commands/object_store/delete.py)_ + +### `lean object-store get` + +Opens the local storage directory in the file explorer. + +``` +Usage: lean object-store get [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/get.py](lean/commands/object_store/get.py)_ + +### `lean object-store list` + +Opens the local storage directory in the file explorer. + +``` +Usage: lean object-store list [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/list.py](lean/commands/object_store/list.py)_ + +### `lean object-store ls` + +Alias for 'list' + +``` +Usage: lean object-store ls [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/ls.py](lean/commands/object_store/ls.py)_ + +### `lean object-store set` + +Opens the local storage directory in the file explorer. + +``` +Usage: lean object-store set [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/set.py](lean/commands/object_store/set.py)_ + ### `lean optimize` Optimize a project's parameters locally using Docker. diff --git a/lean/commands/__init__.py b/lean/commands/__init__.py index eea3807d..c517d8b4 100644 --- a/lean/commands/__init__.py +++ b/lean/commands/__init__.py @@ -30,6 +30,7 @@ from lean.commands.research import research from lean.commands.whoami import whoami from lean.commands.gui import gui +from lean.commands.object_store import object_store lean.add_command(config) lean.add_command(cloud) @@ -49,3 +50,4 @@ lean.add_command(build) lean.add_command(logs) lean.add_command(gui) +lean.add_command(object_store) diff --git a/lean/commands/cloud/__init__.py b/lean/commands/cloud/__init__.py index 63caef70..af5532bf 100644 --- a/lean/commands/cloud/__init__.py +++ b/lean/commands/cloud/__init__.py @@ -19,7 +19,7 @@ from lean.commands.cloud.pull import pull from lean.commands.cloud.push import push from lean.commands.cloud.status import status - +from lean.commands.cloud.object_store import object_store @group() def cloud() -> None: @@ -35,3 +35,4 @@ def cloud() -> None: cloud.add_command(optimize) cloud.add_command(live) cloud.add_command(status) +cloud.add_command(object_store) diff --git a/lean/commands/cloud/object_store/__init__.py b/lean/commands/cloud/object_store/__init__.py new file mode 100644 index 00000000..8d4e7297 --- /dev/null +++ b/lean/commands/cloud/object_store/__init__.py @@ -0,0 +1,24 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lean.commands.cloud.object_store.object_store import object_store +from lean.commands.cloud.object_store.get import get +from lean.commands.cloud.object_store.set import set +from lean.commands.cloud.object_store.list import list +from lean.commands.cloud.object_store.delete import delete + +object_store.add_command(get) +object_store.add_command(set) +object_store.add_command(list) +object_store.add_command(delete) + diff --git a/lean/commands/cloud/object_store/delete.py b/lean/commands/cloud/object_store/delete.py new file mode 100644 index 00000000..4d2633f2 --- /dev/null +++ b/lean/commands/cloud/object_store/delete.py @@ -0,0 +1,29 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument + +from lean.click import LeanCommand +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +def delete(key: str) -> str: + """ + Delete a value from the organization's cloud object store. + + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + api_client.object_store.delete(key, organization_id) \ No newline at end of file diff --git a/lean/commands/cloud/object_store/get.py b/lean/commands/cloud/object_store/get.py new file mode 100644 index 00000000..951a277b --- /dev/null +++ b/lean/commands/cloud/object_store/get.py @@ -0,0 +1,55 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument +from lean.click import LeanCommand +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +def get(key: str) -> str: + """ + Get a value from the organization's cloud object store. + + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + logger = container.logger + data = api_client.object_store.get(key, organization_id) + + try: + headers = ["size", "modified", "key", "preview"] + display_headers = ["Bytes", "Modified", "Filename", "Preview"] + data_row = [] + for header in headers: + if header == "preview": + value = str(data["metadata"].get(header, "N/A")) + data_row.append(_clean_up_preview(value)) + else: + value = str(data["metadata"].get(header, "")) + data_row.append(value) + all_rows = [display_headers] + [data_row] + column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] + for row in all_rows: + logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) + except KeyError as e: + logger.error(f"Key {key} not found.") + except Exception as e: + logger.error(f"Error: {e}") + + +def _clean_up_preview(preview: str) -> str: + return preview.rstrip()[:10] + + diff --git a/lean/commands/cloud/object_store/list.py b/lean/commands/cloud/object_store/list.py new file mode 100644 index 00000000..83a08b05 --- /dev/null +++ b/lean/commands/cloud/object_store/list.py @@ -0,0 +1,44 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import argument +from lean.commands.cloud.object_store import object_store +from lean.click import LeanCommand +from lean.container import container + + +@object_store.command(cls=LeanCommand, name="list", aliases=["ls"]) +@argument("key", type=str, default="/") +def list(key: str) -> str: + """ + List all values for the given root key in the organization's cloud object store. + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + logger = container.logger + data = api_client.object_store.list(key, organization_id) + + try: + headers = ["key", "size", "folder", "name"] + display_headers = ["Key", "Bytes", "Folder", "Filename"] + rows = [[str(obj.get(header, "")) for header in headers] for obj in data['objects']] + # sort rows by key + rows.sort(key=lambda x: x[0]) + all_rows = [display_headers] + rows + column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] + for row in all_rows: + logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) + except KeyError as e: + logger.error(f"Key {key} not found.") + except Exception as e: + logger.error(f"Error: {e}") \ No newline at end of file diff --git a/lean/commands/cloud/object_store/object_store.py b/lean/commands/cloud/object_store/object_store.py new file mode 100644 index 00000000..e0d08877 --- /dev/null +++ b/lean/commands/cloud/object_store/object_store.py @@ -0,0 +1,23 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import group +from lean.components.util.click_aliased_command_group import AliasedCommandGroup + +@group(cls=AliasedCommandGroup, invoke_without_command=True) +def object_store() -> None: + """Interact with the Organization's Cloud Object Store.""" + # This method is intentionally empty + # It is used as the command group for all `lean object-store ` commands + pass + diff --git a/lean/commands/cloud/object_store/set.py b/lean/commands/cloud/object_store/set.py new file mode 100644 index 00000000..6cfd7279 --- /dev/null +++ b/lean/commands/cloud/object_store/set.py @@ -0,0 +1,34 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument +from pathlib import Path +from lean.click import LeanCommand, PathParameter +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +@argument("path", type= PathParameter(exists=True, file_okay=True, dir_okay=False)) +def set(key: str, path: Path) -> None: + """Sets the data to the given key in the organization's cloud object store. + + :param key: The key to set the data to. + :param path: Path to the file containing the object data. + """ + organization_id = container.organization_manager.try_get_working_organization_id() + container.logger.info(f"Setting object {key} in organization {organization_id}") + api_client = container.api_client + with open(path, "rb") as file: + bytes_data: bytes = file.read() + api_client.object_store.set(key, bytes_data, organization_id) \ No newline at end of file diff --git a/lean/commands/object_store/__init__.py b/lean/commands/object_store/__init__.py new file mode 100644 index 00000000..d6108b87 --- /dev/null +++ b/lean/commands/object_store/__init__.py @@ -0,0 +1,24 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lean.commands.object_store.object_store import object_store +from lean.commands.object_store.get import get +from lean.commands.object_store.set import set +from lean.commands.object_store.list import list +from lean.commands.object_store.delete import delete + +object_store.add_command(get) +object_store.add_command(set) +object_store.add_command(list) +object_store.add_command(delete) + diff --git a/lean/commands/object_store/delete.py b/lean/commands/object_store/delete.py new file mode 100644 index 00000000..77d0b56f --- /dev/null +++ b/lean/commands/object_store/delete.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command +from lean.click import LeanCommand +from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer + + +@command(cls=LeanCommand) +def delete() -> str: + """ + Opens the local storage directory in the file explorer. + + """ + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/commands/object_store/get.py b/lean/commands/object_store/get.py new file mode 100644 index 00000000..46f342d3 --- /dev/null +++ b/lean/commands/object_store/get.py @@ -0,0 +1,30 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from click import command +from lean.click import LeanCommand +from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer + + +@command(cls=LeanCommand) +def get() -> str: + """ + Opens the local storage directory in the file explorer. + + """ + open_storage_directory_in_explorer(container.lean_config_manager) + + + diff --git a/lean/commands/object_store/list.py b/lean/commands/object_store/list.py new file mode 100644 index 00000000..c2c7e624 --- /dev/null +++ b/lean/commands/object_store/list.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from lean.click import LeanCommand +from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer +from lean.commands.object_store import object_store + +@object_store.command(cls=LeanCommand, name="list", aliases=["ls"]) +def list() -> str: + """ + Opens the local storage directory in the file explorer. + + """ + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/commands/object_store/object_store.py b/lean/commands/object_store/object_store.py new file mode 100644 index 00000000..cd228874 --- /dev/null +++ b/lean/commands/object_store/object_store.py @@ -0,0 +1,23 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import group +from lean.components.util.click_aliased_command_group import AliasedCommandGroup + +@group(cls=AliasedCommandGroup, invoke_without_command=True) +def object_store() -> None: + """Interact with the Organization's Local Object Store.""" + # This method is intentionally empty + # It is used as the command group for all `lean object-store ` commands + pass + diff --git a/lean/commands/object_store/set.py b/lean/commands/object_store/set.py new file mode 100644 index 00000000..0fd7e2c0 --- /dev/null +++ b/lean/commands/object_store/set.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command +from lean.click import LeanCommand +from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer + + +@command(cls=LeanCommand) +def set() -> None: + """ + Opens the local storage directory in the file explorer. + + """ + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/components/api/api_client.py b/lean/components/api/api_client.py index 36e21c35..73a5a1c4 100644 --- a/lean/components/api/api_client.py +++ b/lean/components/api/api_client.py @@ -23,6 +23,7 @@ from lean.components.api.market_client import MarketClient from lean.components.api.module_client import ModuleClient from lean.components.api.node_client import NodeClient +from lean.components.api.object_store_client import ObjectStoreClient from lean.components.api.optimization_client import OptimizationClient from lean.components.api.organization_client import OrganizationClient from lean.components.api.project_client import ProjectClient @@ -59,6 +60,7 @@ def __init__(self, logger: Logger, http_client: HTTPClient, user_id: str, api_to self.market = MarketClient(self) self.modules = ModuleClient(self) self.nodes = NodeClient(self) + self.object_store = ObjectStoreClient(self) self.optimizations = OptimizationClient(self) self.organizations = OrganizationClient(self) self.projects = ProjectClient(self) @@ -81,7 +83,7 @@ def get(self, endpoint: str, parameters: Dict[str, Any] = {}) -> Any: """ return self._request("get", endpoint, {"params": parameters}) - def post(self, endpoint: str, data: Dict[str, Any] = {}, data_as_json: bool = True) -> Any: + def post(self, endpoint: str, data: Dict[str, Any] = {}, data_as_json: bool = True, extra_options: Dict[str, Any] = None) -> Any: """Makes an authenticated POST request to the given endpoint with the given data. Raises an error if the request fails or if the current credentials are invalid. @@ -92,6 +94,8 @@ def post(self, endpoint: str, data: Dict[str, Any] = {}, data_as_json: bool = Tr :return: the parsed response of the request """ options = {"json": data} if data_as_json else {"data": data} + if extra_options is not None: + options.update(extra_options) return self._request("post", endpoint, options) def is_authenticated(self) -> bool: @@ -165,6 +169,7 @@ def _parse_response(self, response) -> Any: :param response: the response of the request :return: the data in the response """ + data = response.json() if data["success"]: diff --git a/lean/components/api/object_store_client.py b/lean/components/api/object_store_client.py new file mode 100644 index 00000000..fb43c216 --- /dev/null +++ b/lean/components/api/object_store_client.py @@ -0,0 +1,87 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lean.components.api.api_client import * + + +class ObjectStoreClient: + """The ObjectStoreClient class contains methods to interact with object/* API endpoints.""" + + def __init__(self, api_client: 'APIClient') -> None: + """Creates a new ObjectStoreClient instance. + + :param api_client: the APIClient instance to use when making requests + """ + self._api = api_client + + def get(self, key: str, organization_id: str) -> str: + """Returns the details of key from the object store. + + :param key: key of the object to retrieve + :param organization_id: the id of the organization who's object store to retrieve from + :return: the details of the specified object + """ + + payload = { + "organizationId": organization_id, + "key": key, + } + + data = self._api.post("object/get", payload) + + return data + + def set(self, key: str, objectData: bytes, organization_id: str) -> None: + """Sets the given key in the Object Store. + + :param key: key of the object to set + :param objectData: the data to set + :param organization_id: the id of the organization who's object store to set data in + """ + payload = { + "organizationId": organization_id, + "key": key + } + extra_options = { + "files": {"objectData": objectData} + } + self._api.post("object/set", payload, False, extra_options) + + def list(self, path: str, organization_id: str) -> str: + """List all values for the given root key in the Object Store. + + :param path: root key for which to list all objects + :param organization_id: the id of the organization who's object store to retrieve from + :return: all objects for the given root key + """ + payload = { + "organizationId": organization_id, + "path": path + } + + data = self._api.post("object/list", payload) + + return data + + def delete(self, key: str, organization_id: str) -> None: + """Deletes the given key from the Object Store. + + :param key: key of the object to delete + :param organization_id: the id of the organization who's object store to delete from + """ + payload = { + "organizationId": organization_id, + "key": key + } + + self._api.post("object/delete", payload) \ No newline at end of file diff --git a/lean/components/util/object_store_helper.py b/lean/components/util/object_store_helper.py new file mode 100644 index 00000000..549f0865 --- /dev/null +++ b/lean/components/util/object_store_helper.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lean.components.config.lean_config_manager import LeanConfigManager + +def open_storage_directory_in_explorer(lean_config_manager: LeanConfigManager): + """Opens the storage directory in the file explorer.""" + global_storage_directory_path = lean_config_manager.get_cli_root_directory() / "storage" + if not global_storage_directory_path.exists(): + global_storage_directory_path.mkdir(parents=True) + open_file_explorer(str(global_storage_directory_path)) + +def open_file_explorer(directory_path: str): + """Opens the given directory in the file explorer.""" + from webbrowser import open + open('file:///' + directory_path) \ No newline at end of file diff --git a/tests/commands/cloud/object_store/test_delete.py b/tests/commands/cloud/object_store/test_delete.py new file mode 100644 index 00000000..76ea1a1d --- /dev/null +++ b/tests/commands/cloud/object_store/test_delete.py @@ -0,0 +1,30 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from click.testing import CliRunner + +from lean.commands import lean +from lean.container import container +from tests.conftest import initialize_container + + +def test_delete_deletes_value_when_key_is_given() -> None: + api_client = mock.Mock() + api_client.is_authenticated.return_value = True + initialize_container(api_client_to_use=api_client) + + result = CliRunner().invoke(lean, ["cloud", "object-store", "delete", "test-key"]) + assert result.exit_code == 0 + container.api_client.object_store.delete.assert_called_once_with('test-key', 'abc') diff --git a/tests/commands/cloud/object_store/test_get.py b/tests/commands/cloud/object_store/test_get.py new file mode 100644 index 00000000..37ba75ce --- /dev/null +++ b/tests/commands/cloud/object_store/test_get.py @@ -0,0 +1,30 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from click.testing import CliRunner + +from lean.commands import lean +from lean.container import container +from tests.conftest import initialize_container + + +def test_get_gets_value_when_key_is_given() -> None: + api_client = mock.Mock() + api_client.is_authenticated.return_value = True + initialize_container(api_client_to_use=api_client) + + result = CliRunner().invoke(lean, ["cloud", "object-store", "get", "test-key"]) + assert result.exit_code == 0 + container.api_client.object_store.get.assert_called_once_with('test-key', 'abc') diff --git a/tests/commands/cloud/object_store/test_list.py b/tests/commands/cloud/object_store/test_list.py new file mode 100644 index 00000000..1e34a4fe --- /dev/null +++ b/tests/commands/cloud/object_store/test_list.py @@ -0,0 +1,30 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from click.testing import CliRunner + +from lean.commands import lean +from lean.container import container +from tests.conftest import initialize_container + + +def test_list_gets_value_when_key_is_given() -> None: + api_client = mock.Mock() + api_client.is_authenticated.return_value = True + initialize_container(api_client_to_use=api_client) + + result = CliRunner().invoke(lean, ["cloud", "object-store", "list", "test-key"]) + assert result.exit_code == 0 + container.api_client.object_store.list.assert_called_once_with('test-key', 'abc') diff --git a/tests/commands/cloud/object_store/test_set.py b/tests/commands/cloud/object_store/test_set.py new file mode 100644 index 00000000..f282c14b --- /dev/null +++ b/tests/commands/cloud/object_store/test_set.py @@ -0,0 +1,37 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock +from pathlib import Path +from click.testing import CliRunner + +from lean.commands import lean +from lean.container import container +from tests.conftest import initialize_container +from tests.test_helpers import create_fake_lean_cli_project + +def test_set_sets_value_when_path_is_given() -> None: + create_fake_lean_cli_project("test", "python") + api_client = mock.Mock() + api_client.is_authenticated.return_value = True + initialize_container(api_client_to_use=api_client) + + file_path = Path.joinpath(Path.cwd(), "lean.json") + assert (file_path).exists() + file_path = str(file_path) + + result = CliRunner().invoke(lean, ["cloud", "object-store", "set", "test-key", file_path]) + assert result.exit_code == 0 + with open(file_path, "rb") as file: + bytes_data: bytes = file.read() + container.api_client.object_store.set.assert_called_once_with('test-key', bytes_data, 'abc')