From dd15b1d796a5e46ef6e71270731d0b504fe0a4fb Mon Sep 17 00:00:00 2001 From: waterquarks Date: Sat, 17 Jun 2023 05:59:39 +0000 Subject: [PATCH 01/31] (feat) woo-x connector --- hummingbot/connector/connector_status.py | 2 + .../connector/exchange/woo_x/__init__.py | 0 hummingbot/connector/exchange/woo_x/dummy.pxd | 2 + hummingbot/connector/exchange/woo_x/dummy.pyx | 2 + .../woo_x/woo_x_api_order_book_data_source.py | 189 ++++ .../woo_x_api_user_stream_data_source.py | 110 +++ .../connector/exchange/woo_x/woo_x_auth.py | 58 ++ .../exchange/woo_x/woo_x_constants.py | 70 ++ .../exchange/woo_x/woo_x_exchange.py | 499 ++++++++++ .../exchange/woo_x/woo_x_order_book.py | 81 ++ .../connector/exchange/woo_x/woo_x_utils.py | 107 ++ .../exchange/woo_x/woo_x_web_utils.py | 58 ++ .../connector/exchange/woo_x/__init__.py | 0 .../test_woo_x_api_order_book_data_source.py | 464 +++++++++ .../test_woo_x_api_user_stream_data_source.py | 273 ++++++ .../exchange/woo_x/test_woo_x_auth.py | 67 ++ .../exchange/woo_x/test_woo_x_exchange.py | 920 ++++++++++++++++++ .../exchange/woo_x/test_woo_x_order_book.py | 105 ++ .../exchange/woo_x/test_woo_x_utils.py | 40 + .../exchange/woo_x/test_woo_x_web_utils.py | 11 + 20 files changed, 3058 insertions(+) create mode 100644 hummingbot/connector/exchange/woo_x/__init__.py create mode 100644 hummingbot/connector/exchange/woo_x/dummy.pxd create mode 100644 hummingbot/connector/exchange/woo_x/dummy.pyx create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_api_order_book_data_source.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_api_user_stream_data_source.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_auth.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_constants.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_exchange.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_order_book.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_utils.py create mode 100644 hummingbot/connector/exchange/woo_x/woo_x_web_utils.py create mode 100644 test/hummingbot/connector/exchange/woo_x/__init__.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_api_order_book_data_source.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_api_user_stream_data_source.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_auth.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_exchange.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_order_book.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_utils.py create mode 100644 test/hummingbot/connector/exchange/woo_x/test_woo_x_web_utils.py diff --git a/hummingbot/connector/connector_status.py b/hummingbot/connector/connector_status.py index da2e7ee361..2a61dd4673 100644 --- a/hummingbot/connector/connector_status.py +++ b/hummingbot/connector/connector_status.py @@ -38,6 +38,8 @@ 'perpetual_finance': 'bronze', 'probit': 'bronze', 'whitebit': 'bronze', + 'woo_x': 'bronze', + 'woo_x_testnet': 'bronze', 'uniswap': 'gold', 'uniswapLP': 'gold', 'pancakeswap': 'bronze', diff --git a/hummingbot/connector/exchange/woo_x/__init__.py b/hummingbot/connector/exchange/woo_x/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/exchange/woo_x/dummy.pxd b/hummingbot/connector/exchange/woo_x/dummy.pxd new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/dummy.pxd @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/connector/exchange/woo_x/dummy.pyx b/hummingbot/connector/exchange/woo_x/dummy.pyx new file mode 100644 index 0000000000..4b098d6f59 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/dummy.pyx @@ -0,0 +1,2 @@ +cdef class dummy(): + pass diff --git a/hummingbot/connector/exchange/woo_x/woo_x_api_order_book_data_source.py b/hummingbot/connector/exchange/woo_x/woo_x_api_order_book_data_source.py new file mode 100644 index 0000000000..d9301455f9 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_api_order_book_data_source.py @@ -0,0 +1,189 @@ +import asyncio +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_web_utils as web_utils +from hummingbot.connector.exchange.woo_x.woo_x_order_book import WooXOrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.core.web_assistant.connections.data_types import RESTMethod, WSJSONRequest +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory +from hummingbot.core.web_assistant.ws_assistant import WSAssistant +from hummingbot.logger import HummingbotLogger + +if TYPE_CHECKING: + from hummingbot.connector.exchange.woo_x.woo_x_exchange import WooXExchange + + +class WooXAPIOrderBookDataSource(OrderBookTrackerDataSource): + HEARTBEAT_TIME_INTERVAL = 30.0 + TRADE_STREAM_ID = 1 + DIFF_STREAM_ID = 2 + ONE_HOUR = 60 * 60 + + _logger: Optional[HummingbotLogger] = None + + def __init__( + self, + trading_pairs: List[str], + connector: 'WooXExchange', + api_factory: WebAssistantsFactory, + domain: str = CONSTANTS.DEFAULT_DOMAIN + ): + super().__init__(trading_pairs) + self._connector = connector + self._trade_messages_queue_key = CONSTANTS.TRADE_EVENT_TYPE + self._diff_messages_queue_key = CONSTANTS.DIFF_EVENT_TYPE + self._domain = domain + self._api_factory = api_factory + + async def get_last_traded_prices( + self, + trading_pairs: List[str], + domain: Optional[str] = None + ) -> Dict[str, float]: + return await self._connector.get_last_traded_prices(trading_pairs=trading_pairs) + + async def _request_order_book_snapshot(self, trading_pair: str) -> Dict[str, Any]: + """ + Retrieves a copy of the full order book from the exchange, for a particular trading pair. + + :param trading_pair: the trading pair for which the order book will be retrieved + + :return: the response from the exchange (JSON dictionary) + """ + + rest_assistant = await self._api_factory.get_rest_assistant() + + data = await rest_assistant.execute_request( + url=web_utils.public_rest_url( + path_url=f"{CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL}/{await self._connector.exchange_symbol_associated_to_pair(trading_pair=trading_pair)}", + domain=self._domain + ), + method=RESTMethod.GET, + throttler_limit_id=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, + ) + + return data + + async def _subscribe_channels(self, ws: WSAssistant): + """ + Subscribes to the trade events and diff orders events through the provided websocket connection. + :param ws: the websocket assistant used to connect to the exchange + """ + try: + channels = ['trade', 'orderbookupdate'] + + topics = [] + + for trading_pair in self._trading_pairs: + symbol = await self._connector.exchange_symbol_associated_to_pair(trading_pair=trading_pair) + + for channel in channels: + topics.append(f"{symbol}@{channel}") + + payloads = [ + { + "id": str(i), + "topic": topic, + "event": "subscribe" + } + for i, topic in enumerate(topics) + ] + + await asyncio.gather(*[ + ws.send(WSJSONRequest(payload=payload)) for payload in payloads + ]) + + self.logger().info("Subscribed to public order book and trade channels...") + except asyncio.CancelledError: + raise + except Exception: + self.logger().error( + "Unexpected error occurred subscribing to order book trading and delta streams...", + exc_info=True + ) + + raise + + async def _process_websocket_messages(self, websocket_assistant: WSAssistant): + async def ping(): + await websocket_assistant.send(WSJSONRequest(payload={'event': 'ping'})) + + async for ws_response in websocket_assistant.iter_messages(): + data: Dict[str, Any] = ws_response.data + + if data.get('event') == 'ping': + asyncio.ensure_future(ping()) + + if data is not None: # data will be None when the websocket is disconnected + channel: str = self._channel_originating_message(event_message=data) + valid_channels = self._get_messages_queue_keys() + if channel in valid_channels: + self._message_queue[channel].put_nowait(data) + else: + await self._process_message_for_unknown_channel( + event_message=data, websocket_assistant=websocket_assistant + ) + + async def _connected_websocket_assistant(self) -> WSAssistant: + ws: WSAssistant = await self._api_factory.get_ws_assistant() + + await ws.connect( + ws_url=web_utils.wss_public_url(self._domain).format(self._connector.application_id), + ping_timeout=CONSTANTS.WS_HEARTBEAT_TIME_INTERVAL + ) + + return ws + + async def _order_book_snapshot(self, trading_pair: str) -> OrderBookMessage: + snapshot: Dict[str, Any] = await self._request_order_book_snapshot(trading_pair) + + snapshot_timestamp: int = snapshot['timestamp'] + + snapshot_msg: OrderBookMessage = WooXOrderBook.snapshot_message_from_exchange( + snapshot, + snapshot_timestamp, + metadata={"trading_pair": trading_pair} + ) + + return snapshot_msg + + async def _parse_trade_message(self, raw_message: Dict[str, Any], message_queue: asyncio.Queue): + trading_pair = await self._connector.trading_pair_associated_to_exchange_symbol( + symbol=raw_message['topic'].split('@')[0] + ) + + trade_message = WooXOrderBook.trade_message_from_exchange( + raw_message, + {"trading_pair": trading_pair} + ) + + message_queue.put_nowait(trade_message) + + async def _parse_order_book_diff_message(self, raw_message: Dict[str, Any], message_queue: asyncio.Queue): + trading_pair = await self._connector.trading_pair_associated_to_exchange_symbol( + symbol=raw_message['topic'].split('@')[0] + ) + + order_book_message: OrderBookMessage = WooXOrderBook.diff_message_from_exchange( + raw_message, + raw_message['ts'], + {"trading_pair": trading_pair} + ) + + message_queue.put_nowait(order_book_message) + + def _channel_originating_message(self, event_message: Dict[str, Any]) -> str: + channel = "" + + if "topic" in event_message: + channel = event_message.get("topic").split('@')[1] + + relations = { + CONSTANTS.DIFF_EVENT_TYPE: self._diff_messages_queue_key, + CONSTANTS.TRADE_EVENT_TYPE: self._trade_messages_queue_key + } + + channel = relations.get(channel, "") + + return channel diff --git a/hummingbot/connector/exchange/woo_x/woo_x_api_user_stream_data_source.py b/hummingbot/connector/exchange/woo_x/woo_x_api_user_stream_data_source.py new file mode 100644 index 0000000000..a2c6f9bde6 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_api_user_stream_data_source.py @@ -0,0 +1,110 @@ +import asyncio +import json +import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_web_utils as web_utils +from hummingbot.connector.exchange.woo_x.woo_x_auth import WooXAuth +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.core.web_assistant.connections.data_types import WSJSONRequest +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory +from hummingbot.core.web_assistant.ws_assistant import WSAssistant +from hummingbot.logger import HummingbotLogger + +if TYPE_CHECKING: + from hummingbot.connector.exchange.woo_x.woo_x_exchange import WooXExchange + + +class WooXAPIUserStreamDataSource(UserStreamTrackerDataSource): + LISTEN_KEY_KEEP_ALIVE_INTERVAL = 1800 # Recommended to Ping/Update listen key to keep connection alive + + HEARTBEAT_TIME_INTERVAL = 30 + + _logger: Optional[HummingbotLogger] = None + + def __init__( + self, + auth: WooXAuth, + trading_pairs: List[str], + connector: 'WooXExchange', + api_factory: WebAssistantsFactory, + domain: str = CONSTANTS.DEFAULT_DOMAIN + ): + super().__init__() + + self._auth: WooXAuth = auth + self._trading_pairs = trading_pairs + self._connector = connector + self._api_factory = api_factory + self._domain = domain + + async def _connected_websocket_assistant(self) -> WSAssistant: + """ + Creates an instance of WSAssistant connected to the exchange + """ + websocket_assistant = await self._api_factory.get_ws_assistant() + + await websocket_assistant.connect( + ws_url=web_utils.wss_private_url(self._domain).format(self._connector.application_id), + message_timeout=CONSTANTS.SECONDS_TO_WAIT_TO_RECEIVE_MESSAGE + ) + + timestamp = int(time.time() * 1e3) + + await websocket_assistant.send(WSJSONRequest(payload={ + 'id': 'auth', + 'event': 'auth', + 'params': { + 'apikey': self._connector.api_key, + 'sign': self._auth.signature(timestamp), + 'timestamp': timestamp + } + })) + + response = await websocket_assistant.receive() + + if not response.data['success']: + self.logger().error(f"Error authenticating the private websocket connection: {json.dumps(response.data)}") + + raise IOError("Private websocket connection authentication failed") + + return websocket_assistant + + async def _subscribe_channels(self, websocket_assistant: WSAssistant): + """ + Subscribes to the trade events and diff orders events through the provided websocket connection. + + :param websocket_assistant: the websocket assistant used to connect to the exchange + """ + + channels = ['executionreport', 'balance'] + + for channel in channels: + await websocket_assistant.send(WSJSONRequest(payload={ + "id": channel, + "topic": channel, + "event": "subscribe" + })) + + response = await websocket_assistant.receive() + + if not response.data['success']: + raise IOError(f"Error subscribing to the {channel} channel: {json.dumps(response)}") + + self.logger().info("Subscribed to private account and orders channels...") + + async def _process_websocket_messages(self, websocket_assistant: WSAssistant, queue: asyncio.Queue): + async def ping(): + await websocket_assistant.send(WSJSONRequest(payload={'event': 'ping'})) + + async for ws_response in websocket_assistant.iter_messages(): + data = ws_response.data + + if data.get('event') == 'ping': + asyncio.ensure_future(ping()) + + await self._process_event_message(event_message=data, queue=queue) + + async def _process_event_message(self, event_message: Dict[str, Any], queue: asyncio.Queue): + if len(event_message) > 0: + queue.put_nowait(event_message) diff --git a/hummingbot/connector/exchange/woo_x/woo_x_auth.py b/hummingbot/connector/exchange/woo_x/woo_x_auth.py new file mode 100644 index 0000000000..30fcb4ac37 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_auth.py @@ -0,0 +1,58 @@ +import hashlib +import hmac +import json +from typing import Dict + +from hummingbot.connector.time_synchronizer import TimeSynchronizer +from hummingbot.core.web_assistant.auth import AuthBase +from hummingbot.core.web_assistant.connections.data_types import RESTMethod, RESTRequest, WSRequest + + +class WooXAuth(AuthBase): + def __init__(self, api_key: str, secret_key: str, time_provider: TimeSynchronizer): + self.api_key = api_key + self.secret_key = secret_key + self.time_provider = time_provider + + async def rest_authenticate(self, request: RESTRequest) -> RESTRequest: + """ + Adds authentication headers to the request + Adds the server time and the signature to the request, required for authenticated interactions. It also adds + the required parameter in the request header. + :param request: the request to be configured for authenticated interaction + """ + timestamp = str(int(self.time_provider.time() * 1e3)) + + if request.method == RESTMethod.POST: + request.headers = self.headers(timestamp, **json.loads(request.data or json.dumps({}))) + + request.data = json.loads(request.data or json.dumps({})) # Allow aiohttp to send as application/x-www-form-urlencoded + else: + request.headers = self.headers(timestamp, **(request.params or {})) + + return request + + async def ws_authenticate(self, request: WSRequest) -> WSRequest: + """ + This method is intended to configure a websocket request to be authenticated. + Woo X does not use this functionality + """ + return request # pass-through + + def signature(self, timestamp, **kwargs): + signable = '&'.join([f"{key}={value}" for key, value in sorted(kwargs.items())]) + f"|{timestamp}" + + return hmac.new( + bytes(self.secret_key, "utf-8"), + bytes(signable, "utf-8"), + hashlib.sha256 + ).hexdigest().upper() + + def headers(self, timestamp, **kwargs) -> Dict[str, str]: + return { + 'x-api-timestamp': timestamp, + 'x-api-key': self.api_key, + 'x-api-signature': self.signature(timestamp, **kwargs), + 'Content-Type': 'application/x-www-form-urlencoded', + 'Cache-Control': 'no-cache', + } diff --git a/hummingbot/connector/exchange/woo_x/woo_x_constants.py b/hummingbot/connector/exchange/woo_x/woo_x_constants.py new file mode 100644 index 0000000000..d17163532d --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_constants.py @@ -0,0 +1,70 @@ +from hummingbot.core.api_throttler.data_types import RateLimit +from hummingbot.core.data_type.in_flight_order import OrderState + +DEFAULT_DOMAIN = "woo_x" + +MAX_ORDER_ID_LEN = 19 + +HBOT_ORDER_ID_PREFIX = "" + +REST_URLS = { + "woo_x": "https://api.woo.org", + "woo_x_testnet": "https://api.staging.woo.org", +} + +WSS_PUBLIC_URLS = { + "woo_x": "wss://wss.woo.org/ws/stream/{}", + "woo_x_testnet": "wss://wss.staging.woo.org/ws/stream/{}" +} + +WSS_PRIVATE_URLS = { + "woo_x": "wss://wss.woo.org/v2/ws/private/stream/{}", + "woo_x_testnet": "wss://wss.staging.woo.org/v2/ws/private/stream/{}" +} + +WS_HEARTBEAT_TIME_INTERVAL = 30 + +EXCHANGE_INFO_PATH_URL = '/v1/public/info' +MARKET_TRADES_PATH = '/v1/public/market_trades' +ORDERBOOK_SNAPSHOT_PATH_URL = '/v1/public/orderbook' +ORDER_PATH_URL = '/v1/order' +CANCEL_ORDER_PATH_URL = '/v1/client/order' +ACCOUNTS_PATH_URL = '/v2/client/holding' +GET_TRADES_BY_ORDER_ID_PATH = '/v1/order/{}/trades' +GET_ORDER_BY_CLIENT_ORDER_ID_PATH = '/v1/client/order/{}' + + +RATE_LIMITS = [ + RateLimit(limit_id=EXCHANGE_INFO_PATH_URL, limit=10, time_interval=1), + RateLimit(limit_id=CANCEL_ORDER_PATH_URL, limit=10, time_interval=1), + RateLimit(limit_id=GET_TRADES_BY_ORDER_ID_PATH, limit=10, time_interval=1), + RateLimit(limit_id=MARKET_TRADES_PATH, limit=10, time_interval=1), + RateLimit(limit_id=ORDERBOOK_SNAPSHOT_PATH_URL, limit=10, time_interval=1), + RateLimit(limit_id=ORDER_PATH_URL, limit=10, time_interval=1), + RateLimit(limit_id=ACCOUNTS_PATH_URL, limit=10, time_interval=1), + RateLimit(limit_id=GET_ORDER_BY_CLIENT_ORDER_ID_PATH, limit=10, time_interval=1) +] + +# Websocket event types +DIFF_EVENT_TYPE = "orderbookupdate" +TRADE_EVENT_TYPE = "trade" + +SECONDS_TO_WAIT_TO_RECEIVE_MESSAGE = 20 # According to the documentation this has to be less than 30 seconds + +ORDER_STATE = { + "NEW": OrderState.OPEN, + "CANCELLED": OrderState.CANCELED, + "PARTIAL_FILLED": OrderState.PARTIALLY_FILLED, + "FILLED": OrderState.FILLED, + "REJECTED": OrderState.FAILED, + "INCOMPLETE": OrderState.OPEN, + "COMPLETED": OrderState.COMPLETED, +} + +ORDER_NOT_EXIST_ERROR_CODE = -1006 + +UNKNOWN_ORDER_ERROR_CODE = -1004 + +TIME_IN_FORCE_GTC = "GTC" # Good till cancelled +TIME_IN_FORCE_IOC = "IOC" # Immediate or cancel +TIME_IN_FORCE_FOK = "FOK" # Fill or kill diff --git a/hummingbot/connector/exchange/woo_x/woo_x_exchange.py b/hummingbot/connector/exchange/woo_x/woo_x_exchange.py new file mode 100644 index 0000000000..0ade660951 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_exchange.py @@ -0,0 +1,499 @@ +import asyncio +import secrets +from decimal import Decimal +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from bidict import bidict + +from hummingbot.connector.constants import s_decimal_NaN +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_utils, woo_x_web_utils as web_utils +from hummingbot.connector.exchange.woo_x.woo_x_api_order_book_data_source import WooXAPIOrderBookDataSource +from hummingbot.connector.exchange.woo_x.woo_x_api_user_stream_data_source import WooXAPIUserStreamDataSource +from hummingbot.connector.exchange.woo_x.woo_x_auth import WooXAuth +from hummingbot.connector.exchange_py_base import ExchangePyBase +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.connector.utils import combine_to_hb_trading_pair +from hummingbot.core.data_type.common import OrderType, TradeType +from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderUpdate, TradeUpdate +from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource +from hummingbot.core.data_type.trade_fee import DeductedFromReturnsTradeFee, TokenAmount, TradeFeeBase +from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource +from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.core.web_assistant.connections.data_types import RESTMethod +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory + +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + + +class WooXExchange(ExchangePyBase): + UPDATE_ORDER_STATUS_MIN_INTERVAL = 10.0 + + web_utils = web_utils + + def __init__( + self, + client_config_map: "ClientConfigAdapter", + public_api_key: str, + secret_api_key: str, + application_id: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True, + domain: str = CONSTANTS.DEFAULT_DOMAIN, + ): + self.api_key = public_api_key + self.secret_key = secret_api_key + self.application_id = application_id + self._domain = domain + self._trading_required = trading_required + self._trading_pairs = trading_pairs + self._last_trades_poll_woo_x_timestamp = 1.0 + super().__init__(client_config_map) + + @staticmethod + def woo_x_order_type(order_type: OrderType) -> str: + if order_type.name == 'LIMIT_MAKER': + return 'POST_ONLY' + else: + return order_type.name.upper() + + @staticmethod + def to_hb_order_type(woo_x_type: str) -> OrderType: + return OrderType[woo_x_type] + + @property + def authenticator(self) -> WooXAuth: + return WooXAuth( + api_key=self.api_key, + secret_key=self.secret_key, + time_provider=self._time_synchronizer + ) + + @property + def name(self) -> str: + return self._domain + + @property + def rate_limits_rules(self): + return CONSTANTS.RATE_LIMITS + + @property + def domain(self): + return self._domain + + @property + def client_order_id_max_length(self): + return CONSTANTS.MAX_ORDER_ID_LEN + + @property + def client_order_id_prefix(self): + return CONSTANTS.HBOT_ORDER_ID_PREFIX + + @property + def trading_rules_request_path(self): + return CONSTANTS.EXCHANGE_INFO_PATH_URL + + @property + def trading_pairs_request_path(self): + return CONSTANTS.EXCHANGE_INFO_PATH_URL + + @property + def check_network_request_path(self): + return CONSTANTS.EXCHANGE_INFO_PATH_URL + + @property + def trading_pairs(self): + return self._trading_pairs + + @property + def is_cancel_request_in_exchange_synchronous(self) -> bool: + return True + + @property + def is_trading_required(self) -> bool: + return self._trading_required + + def supported_order_types(self): + return [OrderType.LIMIT, OrderType.LIMIT_MAKER, OrderType.MARKET] + + def _is_request_exception_related_to_time_synchronizer(self, request_exception: Exception): + error_description = str(request_exception) + + is_time_synchronizer_related = ( + "-1021" in error_description and "Timestamp for this request" in error_description + ) + + return is_time_synchronizer_related + + def _is_order_not_found_during_status_update_error(self, status_update_exception: Exception) -> bool: + # TODO: implement this method correctly for the connector + # The default implementation was added when the functionality to detect not found orders was introduced in the + # ExchangePyBase class. Also fix the unit test test_lost_order_removed_if_not_found_during_order_status_update + # when replacing the dummy implementation + return False + + def _is_order_not_found_during_cancelation_error(self, cancelation_exception: Exception) -> bool: + # TODO: implement this method correctly for the connector + # The default implementation was added when the functionality to detect not found orders was introduced in the + # ExchangePyBase class. Also fix the unit test test_cancel_order_not_found_in_the_exchange when replacing the + # dummy implementation + return False + + def _create_web_assistants_factory(self) -> WebAssistantsFactory: + return web_utils.build_api_factory( + throttler=self._throttler, + time_synchronizer=self._time_synchronizer, + domain=self._domain, + auth=self._auth + ) + + def _create_order_book_data_source(self) -> OrderBookTrackerDataSource: + return WooXAPIOrderBookDataSource( + trading_pairs=self._trading_pairs, + connector=self, + domain=self.domain, + api_factory=self._web_assistants_factory + ) + + def _create_user_stream_data_source(self) -> UserStreamTrackerDataSource: + return WooXAPIUserStreamDataSource( + auth=self._auth, + trading_pairs=self._trading_pairs, + connector=self, + api_factory=self._web_assistants_factory, + domain=self.domain, + ) + + def _get_fee( + self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + amount: Decimal, + price: Decimal = s_decimal_NaN, + is_maker: Optional[bool] = None + ) -> TradeFeeBase: + is_maker = order_type is OrderType.LIMIT_MAKER + + return DeductedFromReturnsTradeFee(percent=self.estimate_fee_pct(is_maker)) + + def buy(self, + trading_pair: str, + amount: Decimal, + order_type=OrderType.LIMIT, + price: Decimal = s_decimal_NaN, + **kwargs) -> str: + """ + Creates a promise to create a buy order using the parameters + + :param trading_pair: the token pair to operate with + :param amount: the order amount + :param order_type: the type of order to create (MARKET, LIMIT, LIMIT_MAKER) + :param price: the order price + + :return: the id assigned by the connector to the order (the client id) + """ + order_id = str(secrets.randbelow(9223372036854775807)) + + safe_ensure_future(self._create_order( + trade_type=TradeType.BUY, + order_id=order_id, + trading_pair=trading_pair, + amount=amount, + order_type=order_type, + price=price, + **kwargs) + ) + + return order_id + + def sell(self, + trading_pair: str, + amount: Decimal, + order_type: OrderType = OrderType.LIMIT, + price: Decimal = s_decimal_NaN, + **kwargs) -> str: + """ + Creates a promise to create a sell order using the parameters. + :param trading_pair: the token pair to operate with + :param amount: the order amount + :param order_type: the type of order to create (MARKET, LIMIT, LIMIT_MAKER) + :param price: the order price + :return: the id assigned by the connector to the order (the client id) + """ + + order_id = str(secrets.randbelow(9223372036854775807)) + + safe_ensure_future(self._create_order( + trade_type=TradeType.SELL, + order_id=order_id, + trading_pair=trading_pair, + amount=amount, + order_type=order_type, + price=price, + **kwargs) + ) + + return order_id + + async def _place_order( + self, + order_id: str, + trading_pair: str, + amount: Decimal, + trade_type: TradeType, + order_type: OrderType, + price: Decimal, + **kwargs + ) -> Tuple[str, float]: + data = { + "symbol": await self.exchange_symbol_associated_to_pair(trading_pair=trading_pair), + "order_type": self.woo_x_order_type(order_type), + "side": trade_type.name.upper(), + "order_quantity": float(amount), + "client_order_id": order_id + } + + if order_type is OrderType.LIMIT or order_type is OrderType.LIMIT_MAKER: + data["order_price"] = float(price) + + response = await self._api_post( + path_url=CONSTANTS.ORDER_PATH_URL, + data=data, + is_auth_required=True + ) + + return str(response["order_id"]), int(float(response['timestamp']) * 1e3) + + async def _place_cancel(self, order_id: str, tracked_order: InFlightOrder): + params = { + "client_order_id": order_id, + "symbol": await self.exchange_symbol_associated_to_pair(trading_pair=tracked_order.trading_pair), + } + + cancel_result = await self._api_delete( + path_url=CONSTANTS.CANCEL_ORDER_PATH_URL, + params=params, + is_auth_required=True + ) + + if cancel_result.get("status") != "CANCEL_SENT": + raise IOError() + + return True + + async def _format_trading_rules(self, exchange_info: Dict[str, Any]) -> List[TradingRule]: + result = [] + + for entry in filter(woo_x_utils.is_exchange_information_valid, exchange_info.get("rows", [])): + try: + trading_pair = await self.trading_pair_associated_to_exchange_symbol(symbol=entry.get("symbol")) + trading_rule = TradingRule( + trading_pair=trading_pair, + min_order_size=Decimal(str(entry['base_min'])), + min_price_increment=Decimal(str(entry['quote_tick'])), + min_base_amount_increment=Decimal(str(entry['base_tick'])), + min_notional_size=Decimal(str(entry['min_notional'])) + ) + + result.append(trading_rule) + + except Exception: + self.logger().exception(f"Error parsing the trading pair rule {entry}. Skipping.") + return result + + async def _status_polling_loop_fetch_updates(self): + await super()._status_polling_loop_fetch_updates() + + async def _update_trading_fees(self): + """ + Update fees information from the exchange + """ + pass + + async def _user_stream_event_listener(self): + """ + This functions runs in background continuously processing the events received from the exchange by the user + stream data source. It keeps reading events from the queue until the task is interrupted. + The events received are balance updates, order updates and trade events. + """ + async for event_message in self._iter_user_event_queue(): + try: + event_type = event_message.get("topic") + + if event_type == "executionreport": + event_data = event_message.get("data") + + execution_type = event_data.get("status") + + client_order_id = event_data.get("clientOrderId") + + if execution_type in ["PARTIAL_FILLED", "FILLED"]: + tracked_order = self._order_tracker.all_fillable_orders.get(str(client_order_id)) + + if tracked_order is not None: + fee = TradeFeeBase.new_spot_fee( + fee_schema=self.trade_fee_schema(), + trade_type=tracked_order.trade_type, + percent_token=event_data["feeAsset"], + flat_fees=[ + TokenAmount( + amount=Decimal(event_data["fee"]), + token=event_data["feeAsset"] + ) + ] + ) + + trade_update = TradeUpdate( + trade_id=str(event_data["tradeId"]), + client_order_id=tracked_order.client_order_id, + exchange_order_id=str(event_data["orderId"]), + trading_pair=tracked_order.trading_pair, + fee=fee, + fill_base_amount=Decimal(str(event_data["executedQuantity"])), + fill_quote_amount=Decimal(str(event_data["executedQuantity"])) * Decimal(str(event_data["executedPrice"])), + fill_price=Decimal(str(event_data["executedPrice"])), + fill_timestamp=event_data["timestamp"] * 1e-3, + ) + + self._order_tracker.process_trade_update(trade_update) + + tracked_order = self._order_tracker.all_updatable_orders.get(str(client_order_id)) + + if tracked_order is not None: + order_update = OrderUpdate( + trading_pair=tracked_order.trading_pair, + update_timestamp=event_data["timestamp"] * 1e-3, + new_state=CONSTANTS.ORDER_STATE[event_data["status"]], + client_order_id=tracked_order.client_order_id, + exchange_order_id=tracked_order.exchange_order_id, + ) + + self._order_tracker.process_order_update(order_update=order_update) + elif event_type == "balance": + balances = event_message["data"]["balances"] + + for asset_name, balance_entry in balances.items(): + free, frozen = Decimal(str(balance_entry["holding"])), Decimal(str(balance_entry["frozen"])) + + total = free + frozen + + self._account_available_balances[asset_name] = free + + self._account_balances[asset_name] = total + except asyncio.CancelledError: + raise + except Exception: + self.logger().error("Unexpected error in user stream listener loop.", exc_info=True) + await self._sleep(5.0) + + async def _all_trade_updates_for_order(self, order: InFlightOrder) -> List[TradeUpdate]: + trade_updates = [] + + if order.exchange_order_id is not None: + symbol = await self.exchange_symbol_associated_to_pair(trading_pair=order.trading_pair) + + content = await self._api_get( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id), + limit_id=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH, + is_auth_required=True, + ) + + for trade in content['Transactions']: + fee = TradeFeeBase.new_spot_fee( + fee_schema=self.trade_fee_schema(), + trade_type=order.trade_type, + percent_token=trade["fee_asset"], + flat_fees=[ + TokenAmount( + amount=Decimal(str(trade["fee"])), + token=trade["fee_asset"] + ) + ] + ) + + trade_update = TradeUpdate( + trade_id=str(trade["id"]), + client_order_id=order.client_order_id, + exchange_order_id=order.exchange_order_id, + trading_pair=symbol, + fee=fee, + fill_base_amount=Decimal(str(trade["executed_quantity"])), + fill_quote_amount=Decimal(str(trade["executed_price"])) * Decimal(str(trade["executed_quantity"])), + fill_price=Decimal(str(trade["executed_price"])), + fill_timestamp=float(trade["executed_timestamp"]) * 1e-3, + ) + + trade_updates.append(trade_update) + + return trade_updates + + async def _request_order_status(self, tracked_order: InFlightOrder) -> OrderUpdate: + updated_order_data = await self._api_get( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(tracked_order.client_order_id), + is_auth_required=True, + limit_id=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH + ) + + new_state = CONSTANTS.ORDER_STATE[updated_order_data["status"]] + + order_update = OrderUpdate( + client_order_id=tracked_order.client_order_id, + exchange_order_id=str(updated_order_data["order_id"]), + trading_pair=tracked_order.trading_pair, + update_timestamp=float(updated_order_data["created_time"]), + new_state=new_state, + ) + + return order_update + + async def _update_balances(self): + local_asset_names = set(self._account_balances.keys()) + remote_asset_names = set() + + account_info = await self._api_get( + path_url=CONSTANTS.ACCOUNTS_PATH_URL, + is_auth_required=True + ) + + balances = account_info.get('holding', []) + + for balance_info in balances: + asset = balance_info['token'] + holding = balance_info['holding'] + frozen = balance_info['frozen'] + + self._account_available_balances[asset] = Decimal(holding) + self._account_balances[asset] = Decimal(holding) + Decimal(frozen) + remote_asset_names.add(asset) + + asset_names_to_remove = local_asset_names.difference(remote_asset_names) + + for asset_name in asset_names_to_remove: + del self._account_available_balances[asset_name] + del self._account_balances[asset_name] + + def _initialize_trading_pair_symbols_from_exchange_info(self, exchange_info: Dict[str, Any]): + mapping = bidict() + + for entry in filter(woo_x_utils.is_exchange_information_valid, exchange_info["rows"]): + base, quote = entry['symbol'].split('_')[1:] + + mapping[entry["symbol"]] = combine_to_hb_trading_pair( + base=base, + quote=quote + ) + + self._set_trading_pair_symbol_map(mapping) + + async def _get_last_traded_price(self, trading_pair: str) -> float: + content = await self._api_request( + method=RESTMethod.GET, + path_url=CONSTANTS.MARKET_TRADES_PATH, + params={ + "symbol": await self.exchange_symbol_associated_to_pair(trading_pair=trading_pair) + } + ) + + return content['rows'][0]['executed_price'] diff --git a/hummingbot/connector/exchange/woo_x/woo_x_order_book.py b/hummingbot/connector/exchange/woo_x/woo_x_order_book.py new file mode 100644 index 0000000000..548178a191 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_order_book.py @@ -0,0 +1,81 @@ +from typing import Dict, Optional + +from hummingbot.core.data_type.common import TradeType +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage, OrderBookMessageType + + +class WooXOrderBook(OrderBook): + @classmethod + def snapshot_message_from_exchange( + cls, + msg: Dict[str, any], + timestamp: float, + metadata: Optional[Dict] = None + ) -> OrderBookMessage: + """ + Creates a snapshot message with the order book snapshot message + :param msg: the response from the exchange when requesting the order book snapshot + :param timestamp: the snapshot timestamp + :param metadata: a dictionary with extra information to add to the snapshot data + :return: a snapshot message with the snapshot information received from the exchange + """ + + if metadata: + msg.update(metadata) + + return OrderBookMessage(OrderBookMessageType.SNAPSHOT, { + "update_id": timestamp, + "bids": [[entry['price'], entry['quantity']] for entry in msg["bids"]], + "asks": [[entry['price'], entry['quantity']] for entry in msg["asks"]], + }, timestamp=timestamp) + + @classmethod + def diff_message_from_exchange( + cls, + msg: Dict[str, any], + timestamp: Optional[float] = None, + metadata: Optional[Dict] = None + ) -> OrderBookMessage: + """ + Creates a diff message with the changes in the order book received from the exchange + :param msg: the changes in the order book + :param timestamp: the timestamp of the difference + :param metadata: a dictionary with extra information to add to the difference data + :return: a diff message with the changes in the order book notified by the exchange + """ + if metadata: + msg.update(metadata) + + return OrderBookMessage(OrderBookMessageType.DIFF, { + "trading_pair": msg['trading_pair'], + "update_id": msg['ts'], + "bids": msg['data']['bids'], + "asks": msg['data']['asks'] + }, timestamp=msg['ts']) + + @classmethod + def trade_message_from_exchange( + cls, + msg: Dict[str, any], + metadata: Optional[Dict] = None + ): + """ + Creates a trade message with the information from the trade event sent by the exchange + :param msg: the trade event details sent by the exchange + :param metadata: a dictionary with extra information to add to trade message + :return: a trade message with the details of the trade as provided by the exchange + """ + if metadata: + msg.update(metadata) + + timestamp = msg['ts'] + + return OrderBookMessage(OrderBookMessageType.TRADE, { + "trading_pair": msg['trading_pair'], + "trade_type": TradeType[msg['data']["side"]].value, + "trade_id": timestamp, + "update_id": timestamp, + "price": msg['data']['price'], + "amount": msg['data']['size'] + }, timestamp=timestamp * 1e-3) diff --git a/hummingbot/connector/exchange/woo_x/woo_x_utils.py b/hummingbot/connector/exchange/woo_x/woo_x_utils.py new file mode 100644 index 0000000000..9cb3289814 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_utils.py @@ -0,0 +1,107 @@ +from decimal import Decimal +from typing import Any, Dict + +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData +from hummingbot.core.data_type.trade_fee import TradeFeeSchema + +CENTRALIZED = True + +EXAMPLE_PAIR = "BTC-USDT" + +DEFAULT_FEES = TradeFeeSchema( + maker_percent_fee_decimal=Decimal("0.0003"), + taker_percent_fee_decimal=Decimal("0.0003"), + buy_percent_fee_deducted_from_returns=True +) + + +def is_exchange_information_valid(exchange_info: Dict[str, Any]) -> bool: + """ + Verifies if a trading pair is enabled to operate with based on its exchange information + :param exchange_info: the exchange information for a trading pair + :return: True if the trading pair is enabled, False otherwise + """ + category, *rest = exchange_info['symbol'].split('_') + + return category == 'SPOT' + + +class WooXConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="woo_x", const=True, client_data=None) + public_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X public API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + secret_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X secret API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + application_id: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X application ID", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + class Config: + title = "woo_x" + + +KEYS = WooXConfigMap.construct() + +OTHER_DOMAINS = ["woo_x_testnet"] +OTHER_DOMAINS_PARAMETER = {"woo_x_testnet": "woo_x_testnet"} +OTHER_DOMAINS_EXAMPLE_PAIR = {"woo_x_testnet": "BTC-USDT"} +OTHER_DOMAINS_DEFAULT_FEES = {"woo_x_testnet": DEFAULT_FEES} + + +class WooXTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="woo_x_testnet", const=True, client_data=None) + public_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X public API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + secret_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X secret API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + application_id: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Woo X application ID", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + class Config: + title = "woo_x_testnet" + + +OTHER_DOMAINS_KEYS = {"woo_x_testnet": WooXTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/woo_x/woo_x_web_utils.py b/hummingbot/connector/exchange/woo_x/woo_x_web_utils.py new file mode 100644 index 0000000000..5d36a2d235 --- /dev/null +++ b/hummingbot/connector/exchange/woo_x/woo_x_web_utils.py @@ -0,0 +1,58 @@ +import time +from typing import Callable, Optional + +import hummingbot.connector.exchange.woo_x.woo_x_constants as CONSTANTS +from hummingbot.connector.time_synchronizer import TimeSynchronizer +from hummingbot.core.api_throttler.async_throttler import AsyncThrottler +from hummingbot.core.web_assistant.auth import AuthBase +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory + + +def public_rest_url(path_url: str, domain: str = CONSTANTS.DEFAULT_DOMAIN) -> str: + """ + Creates a full URL for provided public REST endpoint + :param path_url: a public REST endpoint + :param domain: the Woo X domain to connect to ("com" or "us"). The default value is "com" + :return: the full URL to the endpoint + """ + return CONSTANTS.REST_URLS[domain] + path_url + + +def private_rest_url(path_url: str, domain: str = CONSTANTS.DEFAULT_DOMAIN) -> str: + return public_rest_url(path_url, domain) + + +def wss_public_url(domain: str = CONSTANTS.DEFAULT_DOMAIN) -> str: + return CONSTANTS.WSS_PUBLIC_URLS[domain] + + +def wss_private_url(domain: str = CONSTANTS.DEFAULT_DOMAIN) -> str: + return CONSTANTS.WSS_PRIVATE_URLS[domain] + + +def build_api_factory( + throttler: Optional[AsyncThrottler] = None, + time_synchronizer: Optional[TimeSynchronizer] = None, + domain: str = CONSTANTS.DEFAULT_DOMAIN, + time_provider: Optional[Callable] = None, + auth: Optional[AuthBase] = None +) -> WebAssistantsFactory: + throttler = throttler or create_throttler() + + api_factory = WebAssistantsFactory( + throttler=throttler, + auth=auth + ) + + return api_factory + + +async def get_current_server_time( + throttler: Optional[AsyncThrottler] = None, + domain: str = CONSTANTS.DEFAULT_DOMAIN, +) -> float: + return time.time() * 1e3 + + +def create_throttler() -> AsyncThrottler: + return AsyncThrottler(CONSTANTS.RATE_LIMITS) diff --git a/test/hummingbot/connector/exchange/woo_x/__init__.py b/test/hummingbot/connector/exchange/woo_x/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_order_book_data_source.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_order_book_data_source.py new file mode 100644 index 0000000000..797376993d --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_order_book_data_source.py @@ -0,0 +1,464 @@ +import asyncio +import json +import re +import unittest +from typing import Awaitable +from unittest.mock import AsyncMock, MagicMock, patch + +from aioresponses.core import aioresponses +from bidict import bidict + +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_web_utils as web_utils +from hummingbot.connector.exchange.woo_x.woo_x_api_order_book_data_source import WooXAPIOrderBookDataSource +from hummingbot.connector.exchange.woo_x.woo_x_exchange import WooXExchange +from hummingbot.connector.test_support.network_mocking_assistant import NetworkMockingAssistant +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessage + + +class WooXAPIOrderBookDataSourceUnitTests(unittest.TestCase): + # logging.Level required to receive logs from the data source logger + level = 0 + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() + cls.base_asset = "BTC" + cls.quote_asset = "USDT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + cls.ex_trading_pair = f"SPOT_{cls.base_asset}_{cls.quote_asset}" + cls.domain = "woo_x" + + def setUp(self) -> None: + super().setUp() + + self.log_records = [] + + self.listening_task = None + + self.mocking_assistant = NetworkMockingAssistant() + + client_config_map = ClientConfigAdapter(ClientConfigMap()) + + self.connector = WooXExchange( + client_config_map=client_config_map, + public_api_key="", + secret_api_key="", + application_id="", + trading_pairs=[], + trading_required=False, + domain=self.domain + ) + + self.data_source = WooXAPIOrderBookDataSource( + trading_pairs=[self.trading_pair], + connector=self.connector, + api_factory=self.connector._web_assistants_factory, + domain=self.domain + ) + + self.data_source.logger().setLevel(1) + + self.data_source.logger().addHandler(self) + + self._original_full_order_book_reset_time = self.data_source.FULL_ORDER_BOOK_RESET_DELTA_SECONDS + + self.data_source.FULL_ORDER_BOOK_RESET_DELTA_SECONDS = -1 + + self.resume_test_event = asyncio.Event() + + self.connector._set_trading_pair_symbol_map(bidict({self.ex_trading_pair: self.trading_pair})) + + def tearDown(self) -> None: + self.listening_task and self.listening_task.cancel() + + self.data_source.FULL_ORDER_BOOK_RESET_DELTA_SECONDS = self._original_full_order_book_reset_time + + super().tearDown() + + def handle(self, record): + self.log_records.append(record) + + def _is_logged(self, log_level: str, message: str) -> bool: + return any(record.levelname == log_level and record.getMessage() == message + for record in self.log_records) + + def _create_exception_and_unlock_test_with_event(self, exception): + self.resume_test_event.set() + raise exception + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + + def _successfully_subscribed_event(self): + resp = { + "result": None, + "id": 1 + } + return resp + + def _trade_update_event(self): + resp = { + "topic": "SPOT_BTC_USDT@trade", + "ts": 1618820361552, + "data": { + "symbol": "SPOT_BTC_USDT", + "price": 56749.15, + "size": 3.92864, + "side": "BUY", + "source": 0 + } + } + + return resp + + def _order_diff_event(self): + resp = { + "topic": "SPOT_BTC_USDT@orderbookupdate", + "ts": 1618826337580, + "data": { + "symbol": "SPOT_BTC_USDT", + "prevTs": 1618826337380, + "asks": [ + [ + 56749.15, + 3.92864 + ], + [ + 56749.8, + 0 + ], + ], + "bids": [ + [ + 56745.2, + 1.03895025 + ], + [ + 56744.6, + 1.0807 + ], + ] + } + } + + return resp + + def _snapshot_response(self): + return { + "success": True, + "bids": [ + { + "price": 4, + "quantity": 431 + } + ], + "asks": [ + { + "price": 4.000002, + "quantity": 12 + } + ], + "timestamp": 1686211049066 + } + + @aioresponses() + def test_get_new_order_book_successful(self, mock_api): + url = web_utils.public_rest_url(path_url=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, domain=self.domain) + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + resp = self._snapshot_response() + + mock_api.get(regex_url, body=json.dumps(resp)) + + order_book: OrderBook = self.async_run_with_timeout( + self.data_source.get_new_order_book(self.trading_pair) + ) + + expected_update_id = resp["timestamp"] + + self.assertEqual(expected_update_id, order_book.snapshot_uid) + bids = list(order_book.bid_entries()) + asks = list(order_book.ask_entries()) + self.assertEqual(1, len(bids)) + self.assertEqual(4, bids[0].price) + self.assertEqual(431, bids[0].amount) + self.assertEqual(expected_update_id, bids[0].update_id) + self.assertEqual(1, len(asks)) + self.assertEqual(4.000002, asks[0].price) + self.assertEqual(12, asks[0].amount) + self.assertEqual(expected_update_id, asks[0].update_id) + + @aioresponses() + def test_get_new_order_book_raises_exception(self, mock_api): + url = web_utils.public_rest_url(path_url=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, domain=self.domain) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + mock_api.get(regex_url, status=400) + + with self.assertRaises(IOError): + self.async_run_with_timeout( + self.data_source.get_new_order_book(self.trading_pair) + ) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listen_for_subscriptions_subscribes_to_trades_and_order_diffs(self, ws_connect_mock): + ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock() + + result_subscribe_trades = { + "id": "0", + "event": "subscribe", + "success": True, + "ts": 1609924478533 + } + + result_subscribe_diffs = { + "id": "1", + "event": "subscribe", + "success": True, + "ts": 1609924478533 + } + + self.mocking_assistant.add_websocket_aiohttp_message( + websocket_mock=ws_connect_mock.return_value, + message=json.dumps(result_subscribe_trades) + ) + + self.mocking_assistant.add_websocket_aiohttp_message( + websocket_mock=ws_connect_mock.return_value, + message=json.dumps(result_subscribe_diffs) + ) + + self.listening_task = self.ev_loop.create_task(self.data_source.listen_for_subscriptions()) + + self.mocking_assistant.run_until_all_aiohttp_messages_delivered(ws_connect_mock.return_value) + + sent_subscription_messages = self.mocking_assistant.json_messages_sent_through_websocket( + websocket_mock=ws_connect_mock.return_value + ) + + self.assertEqual(2, len(sent_subscription_messages)) + + expected_trade_subscription = { + "id": "0", + "topic": f"{self.ex_trading_pair}@trade", + "event": "subscribe", + } + + self.assertEqual(expected_trade_subscription, sent_subscription_messages[0]) + + expected_diff_subscription = { + "id": "1", + "topic": f"{self.ex_trading_pair}@orderbookupdate", + "event": "subscribe", + } + + self.assertEqual(expected_diff_subscription, sent_subscription_messages[1]) + + self.assertTrue(self._is_logged( + "INFO", + "Subscribed to public order book and trade channels..." + )) + + @patch("hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep") + @patch("aiohttp.ClientSession.ws_connect") + def test_listen_for_subscriptions_raises_cancel_exception(self, mock_ws, _: AsyncMock): + mock_ws.side_effect = asyncio.CancelledError + + with self.assertRaises(asyncio.CancelledError): + self.listening_task = self.ev_loop.create_task(self.data_source.listen_for_subscriptions()) + self.async_run_with_timeout(self.listening_task) + + @patch("hummingbot.core.data_type.order_book_tracker_data_source.OrderBookTrackerDataSource._sleep") + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listen_for_subscriptions_logs_exception_details(self, mock_ws, sleep_mock): + mock_ws.side_effect = Exception("TEST ERROR.") + sleep_mock.side_effect = lambda _: self._create_exception_and_unlock_test_with_event(asyncio.CancelledError()) + + self.listening_task = self.ev_loop.create_task(self.data_source.listen_for_subscriptions()) + + self.async_run_with_timeout(self.resume_test_event.wait()) + + self.assertTrue( + self._is_logged( + "ERROR", + "Unexpected error occurred when listening to order book streams. Retrying in 5 seconds...")) + + def test_subscribe_channels_raises_cancel_exception(self): + mock_ws = MagicMock() + mock_ws.send.side_effect = asyncio.CancelledError + + with self.assertRaises(asyncio.CancelledError): + self.listening_task = self.ev_loop.create_task(self.data_source._subscribe_channels(mock_ws)) + self.async_run_with_timeout(self.listening_task) + + def test_subscribe_channels_raises_exception_and_logs_error(self): + mock_ws = MagicMock() + mock_ws.send.side_effect = Exception("Test Error") + + with self.assertRaises(Exception): + self.listening_task = self.ev_loop.create_task(self.data_source._subscribe_channels(mock_ws)) + self.async_run_with_timeout(self.listening_task) + + self.assertTrue( + self._is_logged("ERROR", "Unexpected error occurred subscribing to order book trading and delta streams...") + ) + + def test_listen_for_trades_cancelled_when_listening(self): + mock_queue = MagicMock() + mock_queue.get.side_effect = asyncio.CancelledError() + self.data_source._message_queue[CONSTANTS.TRADE_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + with self.assertRaises(asyncio.CancelledError): + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_trades(self.ev_loop, msg_queue) + ) + self.async_run_with_timeout(self.listening_task) + + def test_listen_for_trades_logs_exception(self): + incomplete_resp = { + "m": 1, + "i": 2, + } + + mock_queue = AsyncMock() + mock_queue.get.side_effect = [incomplete_resp, asyncio.CancelledError()] + self.data_source._message_queue[CONSTANTS.TRADE_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_trades(self.ev_loop, msg_queue) + ) + + try: + self.async_run_with_timeout(self.listening_task) + except asyncio.CancelledError: + pass + + self.assertTrue( + self._is_logged("ERROR", "Unexpected error when processing public trade updates from exchange")) + + def test_listen_for_trades_successful(self): + mock_queue = AsyncMock() + mock_queue.get.side_effect = [self._trade_update_event(), asyncio.CancelledError()] + self.data_source._message_queue[CONSTANTS.TRADE_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_trades(self.ev_loop, msg_queue)) + + msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get()) + + self.assertEqual(1618820361552, msg.trade_id) + + def test_listen_for_order_book_diffs_cancelled(self): + mock_queue = AsyncMock() + mock_queue.get.side_effect = asyncio.CancelledError() + self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + with self.assertRaises(asyncio.CancelledError): + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_order_book_diffs(self.ev_loop, msg_queue) + ) + self.async_run_with_timeout(self.listening_task) + + def test_listen_for_order_book_diffs_logs_exception(self): + incomplete_resp = { + "m": 1, + "i": 2, + } + + mock_queue = AsyncMock() + mock_queue.get.side_effect = [incomplete_resp, asyncio.CancelledError()] + self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_order_book_diffs(self.ev_loop, msg_queue) + ) + + try: + self.async_run_with_timeout(self.listening_task) + except asyncio.CancelledError: + pass + + self.assertTrue( + self._is_logged("ERROR", "Unexpected error when processing public order book updates from exchange")) + + def test_listen_for_order_book_diffs_successful(self): + mock_queue = AsyncMock() + diff_event = self._order_diff_event() + mock_queue.get.side_effect = [diff_event, asyncio.CancelledError()] + self.data_source._message_queue[CONSTANTS.DIFF_EVENT_TYPE] = mock_queue + + msg_queue: asyncio.Queue = asyncio.Queue() + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_order_book_diffs(self.ev_loop, msg_queue)) + + msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get()) + + self.assertEqual(diff_event["ts"], msg.update_id) + + @aioresponses() + def test_listen_for_order_book_snapshots_cancelled_when_fetching_snapshot(self, mock_api): + url = web_utils.public_rest_url(path_url=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, domain=self.domain) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + mock_api.get(regex_url, exception=asyncio.CancelledError, repeat=True) + + with self.assertRaises(asyncio.CancelledError): + self.async_run_with_timeout( + self.data_source.listen_for_order_book_snapshots(self.ev_loop, asyncio.Queue()) + ) + + @aioresponses() + @patch("hummingbot.connector.exchange.woo_x.woo_x_api_order_book_data_source" + ".WooXAPIOrderBookDataSource._sleep") + def test_listen_for_order_book_snapshots_log_exception(self, mock_api, sleep_mock): + msg_queue: asyncio.Queue = asyncio.Queue() + sleep_mock.side_effect = lambda _: self._create_exception_and_unlock_test_with_event(asyncio.CancelledError()) + + url = web_utils.public_rest_url(path_url=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, domain=self.domain) + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + mock_api.get(regex_url, exception=Exception, repeat=True) + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_order_book_snapshots(self.ev_loop, msg_queue) + ) + self.async_run_with_timeout(self.resume_test_event.wait()) + + self.assertTrue( + self._is_logged("ERROR", f"Unexpected error fetching order book snapshot for {self.trading_pair}.")) + + @aioresponses() + def test_listen_for_order_book_snapshots_successful(self, mock_api, ): + msg_queue: asyncio.Queue = asyncio.Queue() + + url = web_utils.public_rest_url(path_url=CONSTANTS.ORDERBOOK_SNAPSHOT_PATH_URL, domain=self.domain) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + mock_api.get(regex_url, body=json.dumps(self._snapshot_response())) + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_order_book_snapshots(self.ev_loop, msg_queue) + ) + + msg: OrderBookMessage = self.async_run_with_timeout(msg_queue.get()) + + self.assertEqual(1686211049066, msg.update_id) diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_user_stream_data_source.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_user_stream_data_source.py new file mode 100644 index 0000000000..4308017ec8 --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_api_user_stream_data_source.py @@ -0,0 +1,273 @@ +import asyncio +import json +import unittest +from typing import Any, Awaitable, Dict, Optional +from unittest.mock import AsyncMock, MagicMock, patch + +from bidict import bidict + +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS +from hummingbot.connector.exchange.woo_x.woo_x_api_user_stream_data_source import WooXAPIUserStreamDataSource +from hummingbot.connector.exchange.woo_x.woo_x_auth import WooXAuth +from hummingbot.connector.exchange.woo_x.woo_x_exchange import WooXExchange +from hummingbot.connector.test_support.network_mocking_assistant import NetworkMockingAssistant +from hummingbot.connector.time_synchronizer import TimeSynchronizer +from hummingbot.core.api_throttler.async_throttler import AsyncThrottler + + +class WooXUserStreamDataSourceUnitTests(unittest.TestCase): + # the level is required to receive logs from the data source logger + level = 0 + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + cls.ex_trading_pair = cls.base_asset + cls.quote_asset + cls.domain = "woo_x" + + cls.listen_key = "TEST_LISTEN_KEY" + + def setUp(self) -> None: + super().setUp() + self.log_records = [] + self.listening_task: Optional[asyncio.Task] = None + self.mocking_assistant = NetworkMockingAssistant() + + self.throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self.mock_time_provider = MagicMock() + self.mock_time_provider.time.return_value = 1000 + self.auth = WooXAuth(api_key="TEST_API_KEY", secret_key="TEST_SECRET", time_provider=self.mock_time_provider) + self.time_synchronizer = TimeSynchronizer() + self.time_synchronizer.add_time_offset_ms_sample(0) + + client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.connector = WooXExchange( + client_config_map=client_config_map, + public_api_key="", + secret_api_key="", + application_id="", + trading_pairs=[], + trading_required=False, + domain=self.domain) + self.connector._web_assistants_factory._auth = self.auth + + self.data_source = WooXAPIUserStreamDataSource( + auth=self.auth, + trading_pairs=[self.trading_pair], + connector=self.connector, + api_factory=self.connector._web_assistants_factory, + domain=self.domain + ) + + self.data_source.logger().setLevel(1) + self.data_source.logger().addHandler(self) + + self.resume_test_event = asyncio.Event() + + self.connector._set_trading_pair_symbol_map(bidict({self.ex_trading_pair: self.trading_pair})) + + def tearDown(self) -> None: + self.listening_task and self.listening_task.cancel() + super().tearDown() + + def handle(self, record): + self.log_records.append(record) + + def _is_logged(self, log_level: str, message: str) -> bool: + return any(record.levelname == log_level and record.getMessage() == message + for record in self.log_records) + + def _raise_exception(self, exception_class): + raise exception_class + + def _create_exception_and_unlock_test_with_event(self, exception): + self.resume_test_event.set() + raise exception + + def _create_return_value_and_unlock_test_with_event(self, value): + self.resume_test_event.set() + return value + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + + def _error_response(self) -> Dict[str, Any]: + resp = { + "code": "ERROR CODE", + "msg": "ERROR MESSAGE" + } + + return resp + + def _user_update_event(self): + # Balance Update + resp = { + "e": "balanceUpdate", + "E": 1573200697110, + "a": "BTC", + "d": "100.00000000", + "T": 1573200697068 + } + return json.dumps(resp) + + def _successfully_subscribed_event(self): + resp = { + "result": None, + "id": 1 + } + return resp + + def _authentication_response(self, success: bool) -> str: + return json.dumps({ + "id": "auth", + "event": "auth", + "success": success, + "ts": 1686526749230, + **({} if success else {"errorMsg": "sample error message"}) + }) + + def _subscription_response(self, success: bool, channel: str) -> str: + return json.dumps({ + 'id': channel, + 'event': 'subscribe', + 'success': success, + 'ts': 1686527628871 + }) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listening_process_authenticates_and_subscribes_to_events(self, ws_connect_mock): + messages = asyncio.Queue() + + ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock() + + initial_last_recv_time = self.data_source.last_recv_time + + # Add the authentication response for the websocket + self.mocking_assistant.add_websocket_aiohttp_message( + ws_connect_mock.return_value, + self._authentication_response(True) + ) + + self.mocking_assistant.add_websocket_aiohttp_message( + ws_connect_mock.return_value, + self._subscription_response( + True, + 'executionreport' + ) + ) + + self.mocking_assistant.add_websocket_aiohttp_message( + ws_connect_mock.return_value, + self._subscription_response( + True, + 'balance' + ) + ) + + self.listening_task = asyncio.get_event_loop().create_task( + self.data_source.listen_for_user_stream(messages) + ) + + self.mocking_assistant.run_until_all_aiohttp_messages_delivered(ws_connect_mock.return_value) + + self.assertTrue( + self._is_logged("INFO", "Subscribed to private account and orders channels...") + ) + + sent_messages = self.mocking_assistant.json_messages_sent_through_websocket(ws_connect_mock.return_value) + + self.assertEqual(3, len(sent_messages)) + + for n, id in enumerate(['auth', 'executionreport', 'balance']): + self.assertEqual(sent_messages[n]['id'], id) + + self.assertGreater(self.data_source.last_recv_time, initial_last_recv_time) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listen_for_user_stream_authentication_failure(self, ws_connect_mock): + messages = asyncio.Queue() + + ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock() + + self.mocking_assistant.add_websocket_aiohttp_message( + ws_connect_mock.return_value, + self._authentication_response(False) + ) + + self.listening_task = asyncio.get_event_loop().create_task( + self.data_source.listen_for_user_stream(messages) + ) + + self.mocking_assistant.run_until_all_aiohttp_messages_delivered(ws_connect_mock.return_value) + + self.assertTrue( + self._is_logged( + "ERROR", + f"Error authenticating the private websocket connection: {self._authentication_response(False)}" + ) + ) + + self.assertTrue( + self._is_logged( + "ERROR", + "Unexpected error while listening to user stream. Retrying after 5 seconds..." + ) + ) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listen_for_user_stream_does_not_queue_empty_payload(self, mock_ws): + mock_ws.return_value = self.mocking_assistant.create_websocket_mock() + + self.mocking_assistant.add_websocket_aiohttp_message( + mock_ws.return_value, "" + ) + + msg_queue = asyncio.Queue() + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_user_stream(msg_queue) + ) + + self.mocking_assistant.run_until_all_aiohttp_messages_delivered(mock_ws.return_value) + + self.assertEqual(0, msg_queue.qsize()) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listen_for_user_stream_connection_failed(self, mock_ws): + mock_ws.side_effect = lambda *arg, **kwars: self._create_exception_and_unlock_test_with_event( + Exception("TEST ERROR.") + ) + + msg_queue = asyncio.Queue() + + self.listening_task = self.ev_loop.create_task( + self.data_source.listen_for_user_stream(msg_queue) + ) + + self.async_run_with_timeout(self.resume_test_event.wait()) + + self.assertTrue( + self._is_logged( + "ERROR", + "Unexpected error while listening to user stream. Retrying after 5 seconds..." + ) + ) + + @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock) + def test_listening_process_canceled_on_cancel_exception(self, mock_ws): + messages = asyncio.Queue() + + mock_ws.side_effect = asyncio.CancelledError + + with self.assertRaises(asyncio.CancelledError): + self.listening_task = asyncio.get_event_loop().create_task( + self.data_source.listen_for_user_stream(messages) + ) + + self.async_run_with_timeout(self.listening_task) diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_auth.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_auth.py new file mode 100644 index 0000000000..26bbc8c51e --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_auth.py @@ -0,0 +1,67 @@ +import asyncio +import hashlib +import hmac +import json +from unittest import TestCase +from unittest.mock import MagicMock + +from typing_extensions import Awaitable + +from hummingbot.connector.exchange.woo_x.woo_x_auth import WooXAuth +from hummingbot.core.web_assistant.connections.data_types import RESTMethod, RESTRequest + + +class WooXAuthTests(TestCase): + def setUp(self) -> None: + self._api_key = "testApiKey" + self._secret = "testSecret" + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): + ret = asyncio.get_event_loop().run_until_complete(asyncio.wait_for(coroutine, timeout)) + + return ret + + def test_rest_authenticate(self): + mock_time_provider = MagicMock() + + mock_time_provider.time.return_value = 1686452155.0 + + data = { + "symbol": "SPOT_BTC_USDT", + "order_type": "LIMIT", + "side": "BUY", + "order_price": 20000, + "order_quantity": 1, + } + + timestamp = str(int(mock_time_provider.time.return_value * 1e3)) + + auth = WooXAuth(api_key=self._api_key, secret_key=self._secret, time_provider=mock_time_provider) + + request = RESTRequest(method=RESTMethod.POST, data=json.dumps(data), is_auth_required=True) + + configured_request = self.async_run_with_timeout(auth.rest_authenticate(request)) + + signable = '&'.join([f"{key}={value}" for key, value in sorted(data.items())]) + f"|{timestamp}" + + signature = ( + hmac.new( + bytes(self._secret, "utf-8"), + bytes(signable, "utf-8"), + hashlib.sha256 + ).hexdigest().upper() + ) + + headers = { + 'x-api-key': self._api_key, + 'x-api-signature': signature, + 'x-api-timestamp': timestamp, + 'Content-Type': 'application/x-www-form-urlencoded', + 'Cache-Control': 'no-cache', + } + + self.assertEqual(timestamp, configured_request.headers['x-api-timestamp']) + + self.assertEqual(signature, configured_request.headers['x-api-signature']) + + self.assertEqual(headers, configured_request.headers) diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_exchange.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_exchange.py new file mode 100644 index 0000000000..64852d7a4a --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_exchange.py @@ -0,0 +1,920 @@ +import json +import logging +import re +import secrets +from decimal import Decimal +from typing import Any, Callable, List, Optional, Tuple +from unittest.mock import patch + +from aioresponses import aioresponses +from aioresponses.core import RequestCall + +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_web_utils as web_utils +from hummingbot.connector.exchange.woo_x.woo_x_exchange import WooXExchange +from hummingbot.connector.test_support.exchange_connector_test import AbstractExchangeConnectorTests +from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.common import OrderType +from hummingbot.core.data_type.in_flight_order import InFlightOrder +from hummingbot.core.data_type.trade_fee import DeductedFromReturnsTradeFee, TokenAmount, TradeFeeBase + + +class WooXExchangeTests(AbstractExchangeConnectorTests.ExchangeConnectorTests): + @property + def all_symbols_url(self): + return web_utils.public_rest_url(path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL, domain=self.exchange._domain) + + @property + def latest_prices_url(self): + params = { + 'symbol': self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset) + } + + query = ('?' + '&'.join([f"{key}={value}" for key, value in sorted(params.items())])) if len( + params) != 0 else '' + + url = web_utils.public_rest_url(path_url=CONSTANTS.MARKET_TRADES_PATH, domain=self.exchange._domain) + query + + return url + + @property + def network_status_url(self): + raise NotImplementedError + + @property + def trading_rules_url(self): + return web_utils.public_rest_url(CONSTANTS.EXCHANGE_INFO_PATH_URL, domain=self.exchange._domain) + + @property + def order_creation_url(self): + return web_utils.public_rest_url(CONSTANTS.ORDER_PATH_URL, domain=self.exchange._domain) + + @property + def balance_url(self): + return web_utils.private_rest_url(CONSTANTS.ACCOUNTS_PATH_URL, domain=self.exchange._domain) + + @property + def all_symbols_request_mock_response(self): + return { + "rows": [ + { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "quote_min": 0, + "quote_max": 200000, + "quote_tick": 0.01, + "base_min": 0.00001, + "base_max": 300, + "base_tick": 0.00000001, + "min_notional": 1, + "price_range": 0.1, + "price_scope": None, + "created_time": "1571824137.000", + "updated_time": "1686530374.000", + "is_stable": 0, + "precisions": [ + 1, + 10, + 100, + 500, + 1000, + 10000 + ] + } + ], + "success": True + } + + @property + def latest_prices_request_mock_response(self): + return { + "success": True, + "rows": [ + { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "side": "BUY", + "source": 0, + "executed_price": self.expected_latest_price, + "executed_quantity": 0.00025, + "executed_timestamp": "1567411795.000" + } + ] + } + + @property + def all_symbols_including_invalid_pair_mock_response(self) -> Tuple[str, Any]: + mock_response = self.all_symbols_request_mock_response + + return None, mock_response + + @property + def network_status_request_successful_mock_response(self): + return {} + + @property + def trading_rules_request_mock_response(self): + return { + "rows": [ + { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "quote_min": 0, + "quote_max": 200000, + "quote_tick": 0.01, + "base_min": 0.00001, + "base_max": 300, + "base_tick": 0.00000001, + "min_notional": 1, + "price_range": 0.1, + "price_scope": None, + "created_time": "1571824137.000", + "updated_time": "1686530374.000", + "is_stable": 0, + "precisions": [ + 1, + 10, + 100, + 500, + 1000, + 10000 + ] + } + ], + "success": None + } + + @property + def trading_rules_request_erroneous_mock_response(self): + return { + "rows": [ + { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "min_notional": 1, + "price_range": 0.1, + "price_scope": None, + "created_time": "1571824137.000", + "updated_time": "1686530374.000", + "is_stable": 0, + "precisions": [ + 1, + 10, + 100, + 500, + 1000, + 10000 + ] + } + ], + "success": None + } + + @property + def order_creation_request_successful_mock_response(self): + return { + "success": True, + "timestamp": "1686537643.701", + "order_id": self.expected_exchange_order_id, + "order_type": "LIMIT", + "order_price": 20000, + "order_quantity": 0.001, + "order_amount": None, + "client_order_id": 0 + } + + @property + def balance_request_mock_response_for_base_and_quote(self): + return { + "holding": [{ + "token": self.base_asset, + "holding": 10, + "frozen": 5, + "interest": 0.0, + "outstanding_holding": -0.00080, + "pending_exposure": 0.0, + "opening_cost": -126.36839957, + "holding_cost": -125.69703515, + "realised_pnl": 73572.86125165, + "settled_pnl": 73573.5326161, + "fee_24_h": 0.01432411, + "settled_pnl_24_h": 0.67528081, + "updated_time": "1675220398" + }, { + "token": self.quote_asset, + "holding": 2000, + "frozen": 0, + "interest": 0.0, + "outstanding_holding": -0.00080, + "pending_exposure": 0.0, + "opening_cost": -126.36839957, + "holding_cost": -125.69703515, + "realised_pnl": 73572.86125165, + "settled_pnl": 73573.5326161, + "fee_24_h": 0.01432411, + "settled_pnl_24_h": 0.67528081, + "updated_time": "1675220398" + }], + "success": True + } + + @property + def balance_request_mock_response_only_base(self): + return { + "holding": [{ + "token": self.base_asset, + "holding": 10, + "frozen": 5, + "interest": 0.0, + "outstanding_holding": -0.00080, + "pending_exposure": 0.0, + "opening_cost": -126.36839957, + "holding_cost": -125.69703515, + "realised_pnl": 73572.86125165, + "settled_pnl": 73573.5326161, + "fee_24_h": 0.01432411, + "settled_pnl_24_h": 0.67528081, + "updated_time": "1675220398" + }], + "success": True + } + + @property + def balance_event_websocket_update(self): + return { + "topic": "balance", + "ts": 1686539285351, + "data": { + "balances": { + self.base_asset: { + "holding": 10, + "frozen": 5, + "interest": 0.0, + "pendingShortQty": 0.0, + "pendingExposure": 0.0, + "pendingLongQty": 0.004, + "pendingLongExposure": 0.0, + "version": 9, + "staked": 0.0, + "unbonding": 0.0, + "vault": 0.0, + "averageOpenPrice": 0.0, + "pnl24H": 0.0, + "fee24H": 0.00773214, + "markPrice": 25772.05, + "pnl24HPercentage": 0.0 + } + } + } + } + + @property + def expected_latest_price(self): + return 9999.9 + + @property + def expected_supported_order_types(self): + return [OrderType.LIMIT, OrderType.LIMIT_MAKER, OrderType.MARKET] + + @property + def expected_trading_rule(self): + return TradingRule( + trading_pair=self.trading_pair, + min_order_size=Decimal(str(self.trading_rules_request_mock_response["rows"][0]["base_min"])), + min_price_increment=Decimal(str(self.trading_rules_request_mock_response["rows"][0]["quote_tick"])), + min_base_amount_increment=Decimal(str(self.trading_rules_request_mock_response["rows"][0]['base_tick'])), + min_notional_size=Decimal(str(self.trading_rules_request_mock_response["rows"][0]["min_notional"])) + ) + + @property + def expected_logged_error_for_erroneous_trading_rule(self): + erroneous_rule = self.trading_rules_request_erroneous_mock_response["rows"][0] + return f"Error parsing the trading pair rule {erroneous_rule}. Skipping." + + @property + def expected_exchange_order_id(self): + return 28 + + @property + def is_order_fill_http_update_included_in_status_update(self) -> bool: + return True + + @property + def is_order_fill_http_update_executed_during_websocket_order_event_processing(self) -> bool: + return False + + @property + def expected_partial_fill_price(self) -> Decimal: + return Decimal(10500) + + @property + def expected_partial_fill_amount(self) -> Decimal: + return Decimal("0.5") + + @property + def expected_fill_fee(self) -> TradeFeeBase: + return DeductedFromReturnsTradeFee( + percent_token=self.quote_asset, + flat_fees=[TokenAmount(token=self.quote_asset, amount=Decimal("30"))] + ) + + @property + def expected_fill_trade_id(self) -> str: + return str(30000) + + def exchange_symbol_for_tokens(self, base_token: str, quote_token: str) -> str: + return f"SPOT_{base_token}_{quote_token}" + + def create_exchange_instance(self): + client_config_map = ClientConfigAdapter(ClientConfigMap()) + + return WooXExchange( + client_config_map=client_config_map, + public_api_key="testAPIKey", + secret_api_key="testSecret", + application_id="applicationId", + trading_pairs=[self.trading_pair], + ) + + def validate_auth_credentials_present(self, request_call: RequestCall): + self._validate_auth_credentials_taking_parameters_from_argument(request_call) + + def validate_order_creation_request(self, order: InFlightOrder, request_call: RequestCall): + request_data = dict(request_call.kwargs["data"]) + self.assertEqual(self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), request_data["symbol"]) + self.assertEqual(order.trade_type.name.upper(), request_data["side"]) + self.assertEqual(WooXExchange.woo_x_order_type(OrderType.LIMIT), request_data["order_type"]) + self.assertEqual(Decimal("100"), Decimal(request_data["order_quantity"])) + self.assertEqual(Decimal("10000"), Decimal(request_data["order_price"])) + self.assertEqual(order.client_order_id, request_data["client_order_id"]) + + def validate_order_cancelation_request(self, order: InFlightOrder, request_call: RequestCall): + request_data = dict(request_call.kwargs["params"]) + + self.assertEqual( + self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + request_data["symbol"] + ) + + self.assertEqual(order.client_order_id, request_data["client_order_id"]) + + def validate_order_status_request(self, order: InFlightOrder, request_call: RequestCall): + return True + # request_params = request_call.kwargs["params"] + # + # + # logging.info(f"request params: {request_params}") + # logging.info(f"request: {request_call}") + # + # self.assertEqual(order.exchange_order_id, request_params["order_id"]) + + def validate_trades_request(self, order: InFlightOrder, request_call: RequestCall): + return True + + def configure_successful_cancelation_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + params = { + "client_order_id": order.client_order_id, + 'symbol': self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset) + } + + query = ('?' + '&'.join([f"{key}={value}" for key, value in sorted(params.items())])) if len( + params) != 0 else '' + + url = web_utils.public_rest_url(CONSTANTS.CANCEL_ORDER_PATH_URL) + query + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + response = self._order_cancelation_request_successful_mock_response(order=order) + + mock_api.delete(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_erroneous_cancelation_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + params = { + "client_order_id": order.client_order_id, + 'symbol': self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset) + } + + query = ('?' + '&'.join([f"{key}={value}" for key, value in sorted(params.items())])) if len( + params) != 0 else '' + + url = web_utils.public_rest_url(CONSTANTS.CANCEL_ORDER_PATH_URL) + query + + response = {"status": "CANCEL_FAILED"} + + mock_api.delete(url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_order_not_found_error_cancelation_response( + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url(CONSTANTS.ORDER_PATH_URL) + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + response = {"code": -2011, "msg": "Unknown order sent."} + mock_api.delete(regex_url, status=400, body=json.dumps(response), callback=callback) + return url + + def configure_one_successful_one_erroneous_cancel_all_response( + self, + successful_order: InFlightOrder, + erroneous_order: InFlightOrder, + mock_api: aioresponses) -> List[str]: + """ + :return: a list of all configured URLs for the cancelations + """ + all_urls = [] + url = self.configure_successful_cancelation_response(order=successful_order, mock_api=mock_api) + all_urls.append(url) + url = self.configure_erroneous_cancelation_response(order=erroneous_order, mock_api=mock_api) + all_urls.append(url) + return all_urls + + def configure_completely_filled_order_status_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None) -> str: + url = web_utils.public_rest_url(CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + response = self._order_status_request_completely_filled_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_canceled_order_status_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url(CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + response = self._order_status_request_canceled_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_erroneous_http_fill_trade_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(url + r"\?.*") + + mock_api.get(regex_url, status=400, callback=callback) + + return url + + def configure_open_order_status_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None) -> str: + """ + :return: the URL configured + """ + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + response = self._order_status_request_open_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_http_error_order_status_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None) -> str: + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + mock_api.get(regex_url, status=401, callback=callback, repeat=True) + return url + + def configure_partially_filled_order_status_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + + response = self._order_status_request_partially_filled_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def configure_order_not_found_error_order_status_response( + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> List[str]: + url = web_utils.public_rest_url(CONSTANTS.ORDER_PATH_URL) + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + response = {"code": -2013, "msg": "Order does not exist."} + mock_api.get(regex_url, body=json.dumps(response), status=400, callback=callback) + return [url] + + def configure_partial_fill_trade_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(url + r"\?.*") + + response = self._order_fills_request_partial_fill_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback) + + return url + + def configure_full_fill_trade_response( + self, + order: InFlightOrder, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None + ) -> str: + url = web_utils.public_rest_url( + path_url=CONSTANTS.GET_ORDER_BY_CLIENT_ORDER_ID_PATH.format(order.client_order_id)) + + regex_url = re.compile(url + r"\?.*") + + response = self._order_fills_request_full_fill_mock_response(order=order) + + mock_api.get(regex_url, body=json.dumps(response), callback=callback, repeat=True) + + return url + + def order_event_for_new_order_websocket_update(self, order: InFlightOrder): + return { + "topic": "executionreport", + "ts": 1686588154387, + "data": { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "clientOrderId": int(order.client_order_id), + "orderId": int(order.exchange_order_id), + "type": order.order_type.name.upper(), + "side": order.trade_type.name.upper(), + "quantity": float(order.amount), + "price": float(order.price), + "tradeId": 0, + "executedPrice": 0.0, + "executedQuantity": 0.0, + "fee": 0.0, + "feeAsset": "BTC", + "totalExecutedQuantity": 0.0, + "status": "NEW", + "reason": "", + "orderTag": "default", + "totalFee": 0.0, + "visible": 0.001, + "timestamp": 1686588154387, + "reduceOnly": False, + "maker": False + } + } + + def order_event_for_canceled_order_websocket_update(self, order: InFlightOrder): + return { + "topic": "executionreport", + "ts": 1686588270140, + "data": { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "clientOrderId": int(order.client_order_id), + "orderId": int(order.exchange_order_id), + "type": order.order_type.name.upper(), + "side": order.trade_type.name.upper(), + "quantity": float(order.amount), + "price": float(order.price), + "tradeId": 0, + "executedPrice": 0.0, + "executedQuantity": 0.0, + "fee": 0.0, + "feeAsset": "BTC", + "totalExecutedQuantity": 0.0, + "status": "CANCELLED", + "reason": "", + "orderTag": "default", + "totalFee": 0.0, + "visible": 0.001, + "timestamp": 1686588270140, + "reduceOnly": False, + "maker": False + } + } + + def order_event_for_full_fill_websocket_update(self, order: InFlightOrder): + return { + "topic": "executionreport", + "ts": 1686588450683, + "data": { + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "clientOrderId": int(order.client_order_id), + "orderId": 199270655, + "type": order.order_type.name.upper(), + "side": order.trade_type.name.upper(), + "quantity": float(order.amount), + "price": float(order.price), + "tradeId": 250106703, + "executedPrice": float(order.price), + "executedQuantity": float(order.amount), + "fee": float(self.expected_fill_fee.flat_fees[0].amount), + "feeAsset": self.expected_fill_fee.flat_fees[0].token, + "totalExecutedQuantity": float(order.amount), + "avgPrice": float(order.price), + "status": "FILLED", + "reason": "", + "orderTag": "default", + "totalFee": 0.00000030, + "visible": 0.001, + "timestamp": 1686588450683, + "reduceOnly": False, + "maker": True + } + } + + def trade_event_for_full_fill_websocket_update(self, order: InFlightOrder): + return None + + @patch("secrets.randbelow") + def test_client_order_id_on_order(self, mocked_secret): + mocked_secret.return_value = 10 + + result = self.exchange.buy( + trading_pair=self.trading_pair, + amount=Decimal("1"), + order_type=OrderType.LIMIT, + price=Decimal("2"), + ) + + expected_client_order_id = str(secrets.randbelow(9223372036854775807)) + + logging.error(expected_client_order_id) + + self.assertEqual(result, expected_client_order_id) + + mocked_secret.return_value = 20 + + expected_client_order_id = str(secrets.randbelow(9223372036854775807)) + + result = self.exchange.sell( + trading_pair=self.trading_pair, + amount=Decimal("1"), + order_type=OrderType.LIMIT, + price=Decimal("2"), + ) + + self.assertEqual(result, expected_client_order_id) + + @aioresponses() + def test_cancel_order_not_found_in_the_exchange(self, mock_api): + # Disabling this test because the connector has not been updated yet to validate + # order not found during cancellation (check _is_order_not_found_during_cancelation_error) + pass + + @aioresponses() + def test_lost_order_removed_if_not_found_during_order_status_update(self, mock_api): + # Disabling this test because the connector has not been updated yet to validate + # order not found during status update (check _is_order_not_found_during_status_update_error) + pass + + @aioresponses() + def test_check_network_failure(self, mock_api): + # Disabling this test because Woo X does not have an endpoint to check health. + pass + + @aioresponses() + def test_check_network_raises_cancel_exception(self, mock_api): + # Disabling this test because Woo X does not have an endpoint to check health. + pass + + @aioresponses() + def test_check_network_success(self, mock_api): + # Disabling this test because Woo X does not have an endpoint to check health. + pass + + @aioresponses() + def test_update_order_status_when_filled_correctly_processed_even_when_trade_fill_update_fails(self, mock_api): + pass + + def _validate_auth_credentials_taking_parameters_from_argument(self, request_call: RequestCall): + headers = request_call.kwargs["headers"] + + self.assertIn("x-api-key", headers) + self.assertIn("x-api-signature", headers) + self.assertIn("x-api-timestamp", headers) + + self.assertEqual("testAPIKey", headers["x-api-key"]) + + def _order_cancelation_request_successful_mock_response(self, order: InFlightOrder) -> Any: + return { + "success": True, + "status": "CANCEL_SENT" + } + + def _order_status_request_completely_filled_mock_response(self, order: InFlightOrder) -> Any: + return { + "success": True, + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "status": "FILLED", + "side": "BUY", + "created_time": "1686558570.495", + "order_id": int(order.exchange_order_id), + "order_tag": "default", + "price": float(order.price), + "type": "LIMIT", + "quantity": float(order.amount), + "amount": None, + "visible": float(order.amount), + "executed": float(order.amount), + "total_fee": 3e-07, + "fee_asset": "BTC", + "client_order_id": int(order.client_order_id), + "reduce_only": False, + "realized_pnl": None, + "average_executed_price": 10500, + "Transactions": [ + { + "id": self.expected_fill_trade_id, + "symbol": self.exchange_symbol_for_tokens(order.base_asset, order.quote_asset), + "order_id": int(order.exchange_order_id), + "fee": float(self.expected_fill_fee.flat_fees[0].amount), + "side": "BUY", + "executed_timestamp": "1686558583.434", + "executed_price": float(order.price), + "executed_quantity": float(order.amount), + "fee_asset": self.expected_fill_fee.flat_fees[0].token, + "is_maker": 1, + "realized_pnl": None + } + ] + } + + def _order_status_request_canceled_mock_response(self, order: InFlightOrder) -> Any: + return { + "success": True, + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "status": "CANCELLED", + "side": order.trade_type.name.upper(), + "created_time": "1686558863.782", + "order_id": int(order.exchange_order_id), + "order_tag": "default", + "price": float(order.price), + "type": order.order_type.name.upper(), + "quantity": float(order.amount), + "amount": None, + "visible": float(order.amount), + "executed": 0, + "total_fee": 0, + "fee_asset": "BTC", + "client_order_id": int(order.client_order_id), + "reduce_only": False, + "realized_pnl": None, + "average_executed_price": None, + "Transactions": [] + } + + def _order_status_request_open_mock_response(self, order: InFlightOrder) -> Any: + return { + "success": True, + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "status": "NEW", + "side": order.trade_type.name.upper(), + "created_time": "1686559699.983", + "order_id": int(order.exchange_order_id), + "order_tag": "default", + "price": float(order.price), + "type": order.order_type.name.upper(), + "quantity": float(order.amount), + "amount": None, + "visible": float(order.amount), + "executed": 0, + "total_fee": 0, + "fee_asset": "BTC", + "client_order_id": int(order.client_order_id), + "reduce_only": False, + "realized_pnl": None, + "average_executed_price": None, + "Transactions": [] + } + + def _order_status_request_partially_filled_mock_response(self, order: InFlightOrder) -> Any: + return { + "success": True, + "symbol": self.exchange_symbol_for_tokens(self.base_asset, self.quote_asset), + "status": "PARTIAL_FILLED", + "side": "BUY", + "created_time": "1686558570.495", + "order_id": order.exchange_order_id, + "order_tag": "default", + "price": float(order.price), + "type": "LIMIT", + "quantity": float(order.amount), + "amount": None, + "visible": float(order.amount), + "executed": float(order.amount), + "total_fee": 3e-07, + "fee_asset": "BTC", + "client_order_id": order.client_order_id, + "reduce_only": False, + "realized_pnl": None, + "average_executed_price": 10500, + "Transactions": [ + { + "id": self.expected_fill_trade_id, + "symbol": self.exchange_symbol_for_tokens(order.base_asset, order.quote_asset), + "order_id": int(order.exchange_order_id), + "fee": float(self.expected_fill_fee.flat_fees[0].amount), + "side": "BUY", + "executed_timestamp": "1686558583.434", + "executed_price": float(self.expected_partial_fill_price), + "executed_quantity": float(self.expected_partial_fill_amount), + "fee_asset": self.expected_fill_fee.flat_fees[0].token, + "is_maker": 1, + "realized_pnl": None + } + ] + } + + def _order_fills_request_partial_fill_mock_response(self, order: InFlightOrder): + return { + "success": True, + "meta": { + "total": 65, + "records_per_page": 100, + "current_page": 1 + }, + "rows": [ + { + "id": self.expected_fill_trade_id, + "symbol": self.exchange_symbol_for_tokens(order.base_asset, order.quote_asset), + "fee": float(self.expected_fill_fee.flat_fees[0].amount), + "side": "BUY", + "executed_timestamp": "1686585723.908", + "order_id": int(order.exchange_order_id), + "order_tag": "default", + "executed_price": float(self.expected_partial_fill_price), + "executed_quantity": float(self.expected_partial_fill_amount), + "fee_asset": self.expected_fill_fee.flat_fees[0].token, + "is_maker": 0, + "realized_pnl": None + } + ] + } + + def _order_fills_request_full_fill_mock_response(self, order: InFlightOrder): + return { + "success": True, + "meta": { + "total": 65, + "records_per_page": 100, + "current_page": 1 + }, + "rows": [ + { + "id": self.expected_fill_trade_id, + "symbol": self.exchange_symbol_for_tokens(order.base_asset, order.quote_asset), + "fee": float(self.expected_fill_fee.flat_fees[0].amount), + "side": "BUY", + "executed_timestamp": "1686585723.908", + "order_id": int(order.exchange_order_id), + "order_tag": "default", + "executed_price": float(order.price), + "executed_quantity": float(order.amount), + "fee_asset": self.expected_fill_fee.flat_fees[0].token, + "is_maker": 0, + "realized_pnl": None + } + ] + } diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_order_book.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_order_book.py new file mode 100644 index 0000000000..0d61e320a9 --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_order_book.py @@ -0,0 +1,105 @@ +from unittest import TestCase + +from hummingbot.connector.exchange.woo_x.woo_x_order_book import WooXOrderBook +from hummingbot.core.data_type.order_book_message import OrderBookMessageType + + +class WooXOrderBookTests(TestCase): + + def test_snapshot_message_from_exchange(self): + snapshot_message = WooXOrderBook.snapshot_message_from_exchange( + msg={ + "success": True, + "asks": [ + { + "price": 10669.4, + "quantity": 1.56263218 + }, + ], + "bids": [ + { + "price": 10669.3, + "quantity": 0.88159988 + }, + ], + "timestamp": 1564710591905 + }, + timestamp=1564710591905, + metadata={"trading_pair": "COINALPHA-HBOT"} + ) + + self.assertEqual(OrderBookMessageType.SNAPSHOT, snapshot_message.type) + self.assertEqual(1564710591905, snapshot_message.timestamp) + self.assertEqual(1564710591905, snapshot_message.update_id) + self.assertEqual(-1, snapshot_message.trade_id) + self.assertEqual(1, len(snapshot_message.bids)) + self.assertEqual(10669.3, snapshot_message.bids[0].price) + self.assertEqual(0.88159988, snapshot_message.bids[0].amount) + self.assertEqual(1564710591905, snapshot_message.bids[0].update_id) + self.assertEqual(1, len(snapshot_message.asks)) + self.assertEqual(10669.4, snapshot_message.asks[0].price) + self.assertEqual(1.56263218, snapshot_message.asks[0].amount) + self.assertEqual(1564710591905, snapshot_message.asks[0].update_id) + + def test_diff_message_from_exchange(self): + diff_msg = WooXOrderBook.diff_message_from_exchange( + msg={ + "topic": "SPOT_BTC_USDT@orderbookupdate", + "ts": 1618826337580, + "data": { + "symbol": "SPOT_BTC_USDT", + "prevTs": 1618826337380, + "asks": [ + [ + 56749.15, + 3.92864 + ], + ], + "bids": [ + [ + 56745.2, + 1.03895025 + ], + ] + } + }, + metadata={"trading_pair": "BTC-USDT"} + ) + + self.assertEqual(1618826337580, diff_msg.timestamp) + self.assertEqual(1618826337580, diff_msg.update_id) + self.assertEqual(1618826337580, diff_msg.first_update_id) + self.assertEqual(-1, diff_msg.trade_id) + self.assertEqual(1, len(diff_msg.bids)) + self.assertEqual(56745.2, diff_msg.bids[0].price) + self.assertEqual(1.03895025, diff_msg.bids[0].amount) + self.assertEqual(1618826337580, diff_msg.bids[0].update_id) + self.assertEqual(1, len(diff_msg.asks)) + self.assertEqual(56749.15, diff_msg.asks[0].price) + self.assertEqual(3.92864, diff_msg.asks[0].amount) + self.assertEqual(1618826337580, diff_msg.asks[0].update_id) + + def test_trade_message_from_exchange(self): + trade_update = { + "topic": "SPOT_ADA_USDT@trade", + "ts": 1618820361552, + "data": { + "symbol": "SPOT_ADA_USDT", + "price": 1.27988, + "size": 300, + "side": "BUY", + "source": 0 + } + } + + trade_message = WooXOrderBook.trade_message_from_exchange( + msg=trade_update, + metadata={"trading_pair": "ADA-USDT"} + ) + + self.assertEqual("ADA-USDT", trade_message.trading_pair) + self.assertEqual(OrderBookMessageType.TRADE, trade_message.type) + self.assertEqual(1618820361.552, trade_message.timestamp) + self.assertEqual(-1, trade_message.update_id) + self.assertEqual(-1, trade_message.first_update_id) + self.assertEqual(1618820361552, trade_message.trade_id) diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_utils.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_utils.py new file mode 100644 index 0000000000..b658fbf8b5 --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_utils.py @@ -0,0 +1,40 @@ +import unittest + +from hummingbot.connector.exchange.woo_x import woo_x_utils as utils + + +class WooXUtilTestCases(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.base_asset = "BTC" + cls.quote_asset = "USDT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + cls.hb_trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + cls.ex_trading_pair = f"{cls.base_asset}{cls.quote_asset}" + + def test_is_exchange_information_valid(self): + invalid_info_1 = { + "symbol": "MARGIN_BTC_USDT", + } + + self.assertFalse(utils.is_exchange_information_valid(invalid_info_1)) + + invalid_info_2 = { + "symbol": "PERP_BTC_ETH", + } + + self.assertFalse(utils.is_exchange_information_valid(invalid_info_2)) + + invalid_info_3 = { + "symbol": "BTC-USDT", + } + + self.assertFalse(utils.is_exchange_information_valid(invalid_info_3)) + + valid_info_4 = { + "symbol": f"SPOT_{self.base_asset}_{self.quote_asset}", + } + + self.assertTrue(utils.is_exchange_information_valid(valid_info_4)) diff --git a/test/hummingbot/connector/exchange/woo_x/test_woo_x_web_utils.py b/test/hummingbot/connector/exchange/woo_x/test_woo_x_web_utils.py new file mode 100644 index 0000000000..4f067178d2 --- /dev/null +++ b/test/hummingbot/connector/exchange/woo_x/test_woo_x_web_utils.py @@ -0,0 +1,11 @@ +from unittest import TestCase + +from hummingbot.connector.exchange.woo_x import woo_x_constants as CONSTANTS, woo_x_web_utils as web_utils + + +class WebUtilsTests(TestCase): + def test_rest_url(self): + url = web_utils.public_rest_url(path_url=CONSTANTS.MARKET_TRADES_PATH, domain=CONSTANTS.DEFAULT_DOMAIN) + self.assertEqual('https://api.woo.org/v1/public/market_trades', url) + url = web_utils.public_rest_url(path_url=CONSTANTS.MARKET_TRADES_PATH, domain='woo_x_testnet') + self.assertEqual('https://api.staging.woo.org/v1/public/market_trades', url) From 01b2fcbd5a1c23b6b01e6ae444842d154716fe19 Mon Sep 17 00:00:00 2001 From: bczhang Date: Wed, 23 Aug 2023 17:13:14 +0800 Subject: [PATCH 02/31] fix dydx market order bug --- .../dydx_perpetual_derivative.py | 22 ++++++- .../test_dydx_perpetual_derivative.py | 57 +++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py index ca4e5e7d37..5b8f4e0db5 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py @@ -25,7 +25,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit -from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType +from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType, PriceType from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState, OrderUpdate, TradeUpdate from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount, TradeFeeBase from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource @@ -240,12 +240,29 @@ async def _place_order( # Increment number of currently undergoing requests self._current_place_order_requests += 1 + if order_type.is_limit_type(): + time_in_force = CONSTANTS.TIF_GOOD_TIL_TIME + else: + time_in_force = CONSTANTS.TIF_IMMEDIATE_OR_CANCEL + if trade_type.name.lower() == 'buy': + # The price needs to be relatively high before the transaction, whether the test will be cancelled + price =Decimal("1.5") * self.get_price_for_volume( + trading_pair, + True, + amount + ).result_price + else: + price = Decimal("0.75") * self.get_price_for_volume( + trading_pair, + False, + amount + ).result_price + notional_amount = amount * price if notional_amount not in self._order_notional_amounts.keys(): self._order_notional_amounts[notional_amount] = len(self._order_notional_amounts.keys()) # Set updated rate limits self._throttler.set_rate_limits(self.rate_limits_rules) - size = str(amount) price = str(price) side = "BUY" if trade_type == TradeType.BUY else "SELL" @@ -254,7 +271,6 @@ async def _place_order( reduce_only = False post_only = order_type is OrderType.LIMIT_MAKER - time_in_force = CONSTANTS.TIF_GOOD_TIL_TIME market = await self.exchange_symbol_associated_to_pair(trading_pair) signature = self._auth.get_order_signature( diff --git a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py index e261984cd8..0fb7785f5a 100644 --- a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py @@ -21,6 +21,8 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType from hummingbot.core.data_type.in_flight_order import InFlightOrder +from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount, TradeFeeBase from hummingbot.core.web_assistant.connections.data_types import RESTRequest @@ -490,27 +492,25 @@ def place_buy_order( self, amount: Decimal = Decimal("100"), price: Decimal = Decimal("10_000"), - order_type: OrderType = OrderType.LIMIT, position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) self.exchange._current_place_order_requests = 1 self.exchange._throttler.set_rate_limits(self.exchange.rate_limits_rules) - return super().place_buy_order(amount, price, order_type, position_action) + return super().place_buy_order(amount, price, position_action) def place_sell_order( self, amount: Decimal = Decimal("100"), price: Decimal = Decimal("10_000"), - order_type: OrderType = OrderType.LIMIT, position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) self.exchange._current_place_order_requests = 1 self.exchange._throttler.set_rate_limits(self.exchange.rate_limits_rules) - return super().place_sell_order(amount, price, order_type, position_action) + return super().place_sell_order(amount, price, position_action) def validate_auth_credentials_present(self, request_call: RequestCall): request_headers = request_call.kwargs["headers"] @@ -1215,3 +1215,52 @@ def test_lost_order_removed_if_not_found_during_order_status_update(self, mock_a # Disabling this test because the connector has not been updated yet to validate # order not found during status update (check _is_order_not_found_during_status_update_error) pass + def place_buy_market_order( + self, + amount: Decimal = Decimal("100"), + price: Decimal = Decimal("10_000"), + order_type: OrderType = OrderType.MARKET, + position_action: PositionAction = PositionAction.OPEN, + ): + order_book = OrderBook() + self.exchange.order_book_tracker._order_books[self.trading_pair] = order_book + order_book.apply_snapshot( + bids=[], + asks=[OrderBookRow(price=5.1, amount=2000, update_id=1)], + update_id=1, + ) + + notional_amount = amount * price + self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) + self.exchange._current_place_order_requests = 1 + self.exchange._throttler.set_rate_limits(self.exchange.rate_limits_rules) + order_id = self.exchange.buy( + trading_pair=self.trading_pair, + amount=amount, + order_type=order_type, + price=price, + position_action=position_action, + ) + return order_id + + @aioresponses() + def test_create_buy_market_order_successfully(self, mock_api): + self._simulate_trading_rules_initialized() + request_sent_event = asyncio.Event() + self.exchange._set_current_timestamp(1640780000) + + url = self.order_creation_url + + creation_response = self.order_creation_request_successful_mock_response + + mock_api.post(url, + body=json.dumps(creation_response), + callback=lambda *args, **kwargs: request_sent_event.set()) + + leverage = 2 + self.exchange._perpetual_trading.set_leverage(self.trading_pair, leverage) + order_id = self.place_buy_market_order() + self.async_run_with_timeout(request_sent_event.wait()) + order_request = self._all_executed_requests(mock_api, url)[0] + request_data = json.loads(order_request.kwargs["data"]) + self.assertEqual(Decimal("1.5") * Decimal("5.1"), Decimal(request_data["price"])) From 6b8bac62bcb9f532d560b52387e1e9d83f741155 Mon Sep 17 00:00:00 2001 From: bczhang Date: Wed, 23 Aug 2023 17:53:22 +0800 Subject: [PATCH 03/31] format code --- .../dydx_perpetual_derivative.py | 99 +++++++++---------- .../test_dydx_perpetual_derivative.py | 5 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py index 5b8f4e0db5..b4cf9257d6 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py @@ -25,7 +25,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit -from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType, PriceType +from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState, OrderUpdate, TradeUpdate from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount, TradeFeeBase from hummingbot.core.data_type.user_stream_tracker_data_source import UserStreamTrackerDataSource @@ -38,20 +38,19 @@ class DydxPerpetualDerivative(PerpetualDerivativePyBase): - web_utils = web_utils def __init__( - self, - client_config_map: "ClientConfigAdapter", - dydx_perpetual_api_key: str, - dydx_perpetual_api_secret: str, - dydx_perpetual_passphrase: str, - dydx_perpetual_ethereum_address: str, - dydx_perpetual_stark_private_key: str, - trading_pairs: Optional[List[str]] = None, - trading_required: bool = True, - domain: str = CONSTANTS.DEFAULT_DOMAIN, + self, + client_config_map: "ClientConfigAdapter", + dydx_perpetual_api_key: str, + dydx_perpetual_api_secret: str, + dydx_perpetual_passphrase: str, + dydx_perpetual_ethereum_address: str, + dydx_perpetual_stark_private_key: str, + trading_pairs: Optional[List[str]] = None, + trading_required: bool = True, + domain: str = CONSTANTS.DEFAULT_DOMAIN, ): self._dydx_perpetual_api_key = dydx_perpetual_api_key self._dydx_perpetual_api_secret = dydx_perpetual_api_secret @@ -223,15 +222,15 @@ async def _place_cancel(self, order_id: str, tracked_order: InFlightOrder): return True async def _place_order( - self, - order_id: str, - trading_pair: str, - amount: Decimal, - trade_type: TradeType, - order_type: OrderType, - price: Decimal, - position_action: PositionAction = PositionAction.NIL, - **kwargs, + self, + order_id: str, + trading_pair: str, + amount: Decimal, + trade_type: TradeType, + order_type: OrderType, + price: Decimal, + position_action: PositionAction = PositionAction.NIL, + **kwargs, ) -> Tuple[str, float]: if self._current_place_order_requests == 0: # No requests are under way, the dictionary can be cleaned @@ -246,11 +245,11 @@ async def _place_order( time_in_force = CONSTANTS.TIF_IMMEDIATE_OR_CANCEL if trade_type.name.lower() == 'buy': # The price needs to be relatively high before the transaction, whether the test will be cancelled - price =Decimal("1.5") * self.get_price_for_volume( - trading_pair, - True, - amount - ).result_price + price = Decimal("1.5") * self.get_price_for_volume( + trading_pair, + True, + amount + ).result_price else: price = Decimal("0.75") * self.get_price_for_volume( trading_pair, @@ -305,10 +304,10 @@ async def _place_order( data=data, is_auth_required=True, limit_id=CONSTANTS.LIMIT_ID_ORDER_PLACE - + "_" - + trading_pair - + "_" - + str(self._order_notional_amounts[notional_amount]), + + "_" + + trading_pair + + "_" + + str(self._order_notional_amounts[notional_amount]), ) except Exception: self._current_place_order_requests -= 1 @@ -323,15 +322,15 @@ async def _place_order( return str(resp["order"]["id"]), iso_to_epoch_seconds(resp["order"]["createdAt"]) def _get_fee( - self, - base_currency: str, - quote_currency: str, - order_type: OrderType, - order_side: TradeType, - position_action: PositionAction, - amount: Decimal, - price: Decimal = s_decimal_NaN, - is_maker: Optional[bool] = None, + self, + base_currency: str, + quote_currency: str, + order_type: OrderType, + order_side: TradeType, + position_action: PositionAction, + amount: Decimal, + price: Decimal = s_decimal_NaN, + is_maker: Optional[bool] = None, ) -> TradeFeeBase: is_maker = is_maker or False if CONSTANTS.FEES_KEY not in self._trading_fees.keys(): @@ -401,7 +400,7 @@ async def _user_stream_event_listener(self): self._account_balances[quote] = Decimal(data["account"]["equity"]) # freeCollateral is sent only on start self._account_available_balances[quote] = ( - Decimal(data["account"]["quoteBalance"]) - self._allocated_collateral_sum + Decimal(data["account"]["quoteBalance"]) - self._allocated_collateral_sum ) if "openPositions" in data["account"]: await self._process_open_positions(data["account"]["openPositions"]) @@ -411,7 +410,7 @@ async def _user_stream_event_listener(self): quote = "USD" # freeCollateral is sent only on start self._account_available_balances[quote] = ( - Decimal(account["quoteBalance"]) - self._allocated_collateral_sum + Decimal(account["quoteBalance"]) - self._allocated_collateral_sum ) if "orders" in data.keys() and len(data["orders"]) > 0: @@ -432,9 +431,9 @@ async def _user_stream_event_listener(self): # Processing all orders of the account, not just the client's if order["status"] in ["OPEN"]: initial_margin_requirement = ( - Decimal(order["price"]) - * Decimal(order["size"]) - * self._margin_fractions[trading_pair]["initial"] + Decimal(order["price"]) + * Decimal(order["size"]) + * self._margin_fractions[trading_pair]["initial"] ) initial_margin_requirement = abs(initial_margin_requirement) self._allocated_collateral[order["id"]] = initial_margin_requirement @@ -557,8 +556,8 @@ async def _process_funding_payments(self, funding_payments: List): if trading_pair not in prev_timestamps.keys(): prev_timestamps[trading_pair] = None if ( - prev_timestamps[trading_pair] is not None - and dateparse(funding_payment["effectiveAt"]).timestamp() <= prev_timestamps[trading_pair] + prev_timestamps[trading_pair] is not None + and dateparse(funding_payment["effectiveAt"]).timestamp() <= prev_timestamps[trading_pair] ): continue timestamp = dateparse(funding_payment["effectiveAt"]).timestamp() @@ -635,10 +634,10 @@ def _process_order_fills(self, fill_data: Dict, order: InFlightOrder) -> Optiona position_action = ( PositionAction.OPEN if ( - order.trade_type is TradeType.BUY - and position_side == "BUY" - or order.trade_type is TradeType.SELL - and position_side == "SELL" + order.trade_type is TradeType.BUY + and position_side == "BUY" + or order.trade_type is TradeType.SELL + and position_side == "SELL" ) else PositionAction.CLOSE ) diff --git a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py index 0fb7785f5a..92b9d88390 100644 --- a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py @@ -1215,6 +1215,7 @@ def test_lost_order_removed_if_not_found_during_order_status_update(self, mock_a # Disabling this test because the connector has not been updated yet to validate # order not found during status update (check _is_order_not_found_during_status_update_error) pass + def place_buy_market_order( self, amount: Decimal = Decimal("100"), @@ -1259,8 +1260,8 @@ def test_create_buy_market_order_successfully(self, mock_api): leverage = 2 self.exchange._perpetual_trading.set_leverage(self.trading_pair, leverage) - order_id = self.place_buy_market_order() + self.place_buy_market_order() self.async_run_with_timeout(request_sent_event.wait()) order_request = self._all_executed_requests(mock_api, url)[0] request_data = json.loads(order_request.kwargs["data"]) - self.assertEqual(Decimal("1.5") * Decimal("5.1"), Decimal(request_data["price"])) + self.assertEqual(Decimal("1.5") * Decimal("5.1"), Decimal(request_data["price"])) From 420e07449a5a84247d52e0fca648e6675a24794d Mon Sep 17 00:00:00 2001 From: bczhang Date: Wed, 23 Aug 2023 18:07:46 +0800 Subject: [PATCH 04/31] format code --- .../dydx_perpetual_derivative.py | 26 ++--- .../test_dydx_perpetual_derivative.py | 108 ++++++++++-------- 2 files changed, 72 insertions(+), 62 deletions(-) diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py index b4cf9257d6..b75c9581c1 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py @@ -304,10 +304,10 @@ async def _place_order( data=data, is_auth_required=True, limit_id=CONSTANTS.LIMIT_ID_ORDER_PLACE - + "_" - + trading_pair - + "_" - + str(self._order_notional_amounts[notional_amount]), + + "_" + + trading_pair + + "_" + + str(self._order_notional_amounts[notional_amount]), ) except Exception: self._current_place_order_requests -= 1 @@ -400,7 +400,7 @@ async def _user_stream_event_listener(self): self._account_balances[quote] = Decimal(data["account"]["equity"]) # freeCollateral is sent only on start self._account_available_balances[quote] = ( - Decimal(data["account"]["quoteBalance"]) - self._allocated_collateral_sum + Decimal(data["account"]["quoteBalance"]) - self._allocated_collateral_sum ) if "openPositions" in data["account"]: await self._process_open_positions(data["account"]["openPositions"]) @@ -410,7 +410,7 @@ async def _user_stream_event_listener(self): quote = "USD" # freeCollateral is sent only on start self._account_available_balances[quote] = ( - Decimal(account["quoteBalance"]) - self._allocated_collateral_sum + Decimal(account["quoteBalance"]) - self._allocated_collateral_sum ) if "orders" in data.keys() and len(data["orders"]) > 0: @@ -431,9 +431,9 @@ async def _user_stream_event_listener(self): # Processing all orders of the account, not just the client's if order["status"] in ["OPEN"]: initial_margin_requirement = ( - Decimal(order["price"]) - * Decimal(order["size"]) - * self._margin_fractions[trading_pair]["initial"] + Decimal(order["price"]) + * Decimal(order["size"]) + * self._margin_fractions[trading_pair]["initial"] ) initial_margin_requirement = abs(initial_margin_requirement) self._allocated_collateral[order["id"]] = initial_margin_requirement @@ -634,10 +634,10 @@ def _process_order_fills(self, fill_data: Dict, order: InFlightOrder) -> Optiona position_action = ( PositionAction.OPEN if ( - order.trade_type is TradeType.BUY - and position_side == "BUY" - or order.trade_type is TradeType.SELL - and position_side == "SELL" + order.trade_type is TradeType.BUY + and position_side == "BUY" + or order.trade_type is TradeType.SELL + and position_side == "SELL" ) else PositionAction.CLOSE ) diff --git a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py index 92b9d88390..c7b3889228 100644 --- a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py @@ -29,15 +29,15 @@ class DydxPerpetualAuthMock(DydxPerpetualAuth): def get_order_signature( - self, - position_id: str, - client_id: str, - market: str, - side: str, - size: str, - price: str, - limit_fee: str, - expiration_epoch_seconds: int, + self, + position_id: str, + client_id: str, + market: str, + side: str, + size: str, + price: str, + limit_fee: str, + expiration_epoch_seconds: int, ) -> str: return "0123456789" @@ -489,10 +489,10 @@ def create_exchange_instance(self): return exchange def place_buy_order( - self, - amount: Decimal = Decimal("100"), - price: Decimal = Decimal("10_000"), - position_action: PositionAction = PositionAction.OPEN, + self, + amount: Decimal = Decimal("100"), + price: Decimal = Decimal("10_000"), + position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) @@ -501,10 +501,10 @@ def place_buy_order( return super().place_buy_order(amount, price, position_action) def place_sell_order( - self, - amount: Decimal = Decimal("100"), - price: Decimal = Decimal("10_000"), - position_action: PositionAction = PositionAction.OPEN, + self, + amount: Decimal = Decimal("100"), + price: Decimal = Decimal("10_000"), + position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) @@ -550,7 +550,8 @@ def validate_trades_request(self, order: InFlightOrder, request_call: RequestCal self.assertEqual(CONSTANTS.LAST_FILLS_MAX, request_params["limit"]) def configure_successful_cancelation_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: """ :return: the URL configured for the cancelation @@ -563,7 +564,8 @@ def configure_successful_cancelation_response( return url def configure_erroneous_cancelation_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: """ :return: the URL configured for the cancelation @@ -576,7 +578,7 @@ def configure_erroneous_cancelation_response( return url def configure_one_successful_one_erroneous_cancel_all_response( - self, successful_order: InFlightOrder, erroneous_order: InFlightOrder, mock_api: aioresponses + self, successful_order: InFlightOrder, erroneous_order: InFlightOrder, mock_api: aioresponses ) -> List[str]: """ :return: a list of all configured URLs for the cancelations @@ -604,7 +606,8 @@ def configure_order_not_found_error_order_status_response( raise NotImplementedError def configure_completely_filled_order_status_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> List[str]: """ :return: the URL configured @@ -617,7 +620,8 @@ def configure_completely_filled_order_status_response( return [url_order_status] def configure_canceled_order_status_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> List[str]: """ :return: the URL configured @@ -635,7 +639,8 @@ def configure_canceled_order_status_response( return [url_fills, url_order_status] def configure_open_order_status_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> List[str]: """ :return: the URL configured @@ -648,7 +653,8 @@ def configure_open_order_status_response( return [url] def configure_http_error_order_status_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: """ :return: the URL configured @@ -660,19 +666,22 @@ def configure_http_error_order_status_response( return url def configure_partially_filled_order_status_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> List[str]: # Dydx has no partial fill status raise NotImplementedError def configure_partial_fill_trade_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: # Dydx has no partial fill status raise NotImplementedError def configure_erroneous_http_fill_trade_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: """ :return: the URL configured @@ -683,7 +692,8 @@ def configure_erroneous_http_fill_trade_response( return url def configure_full_fill_trade_response( - self, order: InFlightOrder, mock_api: aioresponses, callback: Optional[Callable] = lambda *args, **kwargs: None + self, order: InFlightOrder, mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None ) -> str: """ :return: the URL configured @@ -1067,28 +1077,28 @@ def position_event_for_full_fill_websocket_update(self, order: InFlightOrder, un } def configure_successful_set_position_mode( - self, - position_mode: PositionMode, - mock_api: aioresponses, - callback: Optional[Callable] = lambda *args, **kwargs: None, + self, + position_mode: PositionMode, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None, ): # There's only one way position mode pass def configure_failed_set_position_mode( - self, - position_mode: PositionMode, - mock_api: aioresponses, - callback: Optional[Callable] = lambda *args, **kwargs: None, + self, + position_mode: PositionMode, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None, ) -> Tuple[str, str]: # There's only one way position mode, this should never be called pass def configure_failed_set_leverage( - self, - leverage: int, - mock_api: aioresponses, - callback: Optional[Callable] = lambda *args, **kwargs: None, + self, + leverage: int, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None, ) -> Tuple[str, str]: url = web_utils.public_rest_url(CONSTANTS.PATH_MARKETS) regex_url = re.compile(f"^{url}") @@ -1100,10 +1110,10 @@ def configure_failed_set_leverage( return url, "Failed to obtain markets information." def configure_successful_set_leverage( - self, - leverage: int, - mock_api: aioresponses, - callback: Optional[Callable] = lambda *args, **kwargs: None, + self, + leverage: int, + mock_api: aioresponses, + callback: Optional[Callable] = lambda *args, **kwargs: None, ): url = web_utils.public_rest_url(CONSTANTS.PATH_MARKETS) regex_url = re.compile(f"^{url}") @@ -1217,11 +1227,11 @@ def test_lost_order_removed_if_not_found_during_order_status_update(self, mock_a pass def place_buy_market_order( - self, - amount: Decimal = Decimal("100"), - price: Decimal = Decimal("10_000"), - order_type: OrderType = OrderType.MARKET, - position_action: PositionAction = PositionAction.OPEN, + self, + amount: Decimal = Decimal("100"), + price: Decimal = Decimal("10_000"), + order_type: OrderType = OrderType.MARKET, + position_action: PositionAction = PositionAction.OPEN, ): order_book = OrderBook() self.exchange.order_book_tracker._order_books[self.trading_pair] = order_book From c7a781feaba552db39346eb7d7f6bf7dfbf03a84 Mon Sep 17 00:00:00 2001 From: bczhang Date: Thu, 24 Aug 2023 10:47:02 +0800 Subject: [PATCH 05/31] fix bug --- .../dydx_perpetual/test_dydx_perpetual_derivative.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py index c7b3889228..8848ad0dd8 100644 --- a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py @@ -492,25 +492,27 @@ def place_buy_order( self, amount: Decimal = Decimal("100"), price: Decimal = Decimal("10_000"), + order_type: OrderType = OrderType.LIMIT, position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) self.exchange._current_place_order_requests = 1 self.exchange._throttler.set_rate_limits(self.exchange.rate_limits_rules) - return super().place_buy_order(amount, price, position_action) + return super().place_buy_order(amount, price, order_type, position_action) def place_sell_order( self, amount: Decimal = Decimal("100"), price: Decimal = Decimal("10_000"), + order_type: OrderType = OrderType.LIMIT, position_action: PositionAction = PositionAction.OPEN, ): notional_amount = amount * price self.exchange._order_notional_amounts[notional_amount] = len(self.exchange._order_notional_amounts.keys()) self.exchange._current_place_order_requests = 1 self.exchange._throttler.set_rate_limits(self.exchange.rate_limits_rules) - return super().place_sell_order(amount, price, position_action) + return super().place_sell_order(amount, price, order_type, position_action) def validate_auth_credentials_present(self, request_call: RequestCall): request_headers = request_call.kwargs["headers"] From a2e3cd87a50cdeb21a1d3b8aef138fbbf77575d9 Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 01:11:14 -0300 Subject: [PATCH 06/31] (feat) Added the new IP rate limits for Injective V2 --- .../injective_constants.py | 2 - .../injective_v2_perpetual_derivative.py | 3 +- .../injective_v2_perpetual_utils.py | 4 +- .../data_sources/injective_data_source.py | 12 +- .../injective_grantee_data_source.py | 45 +++--- .../injective_read_only_data_source.py | 12 +- .../injective_vaults_data_source.py | 10 +- .../injective_v2/injective_constants.py | 133 +++++++++++++++--- .../injective_v2/injective_v2_exchange.py | 3 +- .../injective_v2/injective_v2_utils.py | 40 ++++-- setup.py | 3 + .../test_injective_data_source.py | 3 + ...v2_utils.py => test_injective_v2_utils.py} | 13 +- 13 files changed, 212 insertions(+), 71 deletions(-) rename test/hummingbot/connector/exchange/injective_v2/{tests_injective_v2_utils.py => test_injective_v2_utils.py} (92%) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py index 3a58fec5d8..e474272bf1 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py @@ -7,8 +7,6 @@ TRANSACTIONS_CHECK_INTERVAL = CONSTANTS.TRANSACTIONS_CHECK_INTERVAL -RATE_LIMITS = CONSTANTS.RATE_LIMITS - ORDER_STATE_MAP = CONSTANTS.ORDER_STATE_MAP ORDER_NOT_FOUND_ERROR_MESSAGE = CONSTANTS.ORDER_NOT_FOUND_ERROR_MESSAGE diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index 3c528c7fc5..abad110787 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -60,6 +60,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -85,7 +86,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py index 5e5533b5e9..da2c346da3 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py @@ -73,7 +73,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index 6a5623d01b..f56b213b74 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -739,7 +739,7 @@ def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError @abstractmethod - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -804,9 +804,10 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( - chain_price=Decimal(trades_response["trades"][0]["price"]["price"])) + chain_price=Decimal(trades[0]["price"]["price"])) else: market = await self.derivative_market_info_for_id(market_id=market_id) @@ -815,7 +816,8 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( chain_price=Decimal(trades_response["trades"][0]["positionDelta"]["executionPrice"])) @@ -829,7 +831,7 @@ async def _transaction_from_chain(self, tx_hash: str, retries: int) -> int: while executed_tries < retries and not found: executed_tries += 1 try: - async with self.throttler.execute_task(limit_id=CONSTANTS.SPOT_ORDERS_HISTORY_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_CHAIN_LIMIT_ID): block_height = await self.query_executor.get_tx_block_height(tx_hash=tx_hash) found = True except ValueError: diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py index d50bc7bc8e..e514d7e1dc 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( granter_address: str, granter_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -253,13 +253,14 @@ async def initialize_trading_account(self): await self._client.get_account(address=self.trading_account_injective_address) self._is_trading_account_initialized = True - def order_hash_manager(self) -> OrderHashManager: + async def order_hash_manager(self) -> OrderHashManager: if self._order_hash_manager is None: - self._order_hash_manager = OrderHashManager( - address=self._granter_address, - network=self._network, - subaccount_indexes=[self._granter_subaccount_index] - ) + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_SUBACCOUNT_LIMIT_ID): + self._order_hash_manager = OrderHashManager( + address=self._granter_address, + network=self._network, + subaccount_indexes=[self._granter_subaccount_index] + ) return self._order_hash_manager def supported_order_types(self) -> List[OrderType]: @@ -278,7 +279,11 @@ async def update_markets(self): for market_info in markets: try: - ticker_base, ticker_quote = market_info["ticker"].split("/") + if "/" in market_info["ticker"]: + ticker_base, ticker_quote = market_info["ticker"].split("/") + else: + ticker_base = market_info["ticker"] + ticker_quote = None base_token = self._token_from_market_info( denom=market_info["baseDenom"], token_meta=market_info["baseTokenMeta"], @@ -339,7 +344,7 @@ async def order_updates_for_transaction( transaction_spot_orders = [] transaction_derivative_orders = [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -433,12 +438,14 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: return self._granter_subaccount_index == CONSTANTS.DEFAULT_SUBACCOUNT_INDEX - def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candidate_symbol: str) -> InjectiveToken: + def _token_from_market_info( + self, denom: str, token_meta: Dict[str, Any], candidate_symbol: Optional[str] = None + ) -> InjectiveToken: token = self._tokens_map.get(denom) if token is None: unique_symbol = token_meta["symbol"] if unique_symbol in self._token_symbol_symbol_and_denom_map: - if candidate_symbol not in self._token_symbol_symbol_and_denom_map: + if candidate_symbol is not None and candidate_symbol not in self._token_symbol_symbol_and_denom_map: unique_symbol = candidate_symbol else: unique_symbol = token_meta["name"] @@ -455,7 +462,9 @@ def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candid return token def _parse_derivative_market_info(self, market_info: Dict[str, Any]) -> InjectiveDerivativeMarket: - _, ticker_quote = market_info["ticker"].split("/") + ticker_quote = None + if "/" in market_info["ticker"]: + _, ticker_quote = market_info["ticker"].split("/") quote_token = self._token_from_market_info( denom=market_info["quoteDenom"], token_meta=market_info["quoteTokenMeta"], @@ -475,7 +484,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -484,7 +493,7 @@ def _calculate_order_hashes( derivative_hashes = [] if len(spot_orders) > 0 or len(derivative_orders) > 0: - hash_manager = self.order_hash_manager() + hash_manager = await self.order_hash_manager() hash_manager_result = hash_manager.compute_order_hashes( spot_orders=spot_orders, derivative_orders=derivative_orders, @@ -545,11 +554,11 @@ async def _order_creation_messages( order_definition = await self._create_derivative_order_definition(order=order) derivative_order_definitions.append(order_definition) - market_spot_hashes, market_derivative_hashes = self._calculate_order_hashes( + market_spot_hashes, market_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_market_order_definitions, derivative_orders=derivative_market_order_definitions, ) - limit_spot_hashes, limit_derivative_hashes = self._calculate_order_hashes( + limit_spot_hashes, limit_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_order_definitions, derivative_orders=derivative_order_definitions, ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py index 1a56e03f87..7c0850b0ad 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py @@ -21,6 +21,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType from hummingbot.core.data_type.in_flight_order import OrderUpdate from hummingbot.core.pubsub import PubSub @@ -33,6 +34,7 @@ class InjectiveReadOnlyDataSource(InjectiveDataSource): def __init__( self, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -45,9 +47,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._markets_initialization_lock = asyncio.Lock() self._spot_market_info_map: Optional[Dict[str, InjectiveSpotMarket]] = None @@ -317,8 +317,10 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError - def _calculate_order_hashes(self, spot_orders: List[GatewayInFlightOrder], - derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: + async def _calculate_order_hashes( + self, + spot_orders: List[GatewayInFlightOrder], + derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: raise NotImplementedError def _reset_order_hash_manager(self): diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py index 80d8904d16..39cc1da564 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( vault_contract_address: str, vault_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -324,7 +324,7 @@ async def order_updates_for_transaction( spot_orders = spot_orders or [] perpetual_orders = perpetual_orders or [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -441,7 +441,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] diff --git a/hummingbot/connector/exchange/injective_v2/injective_constants.py b/hummingbot/connector/exchange/injective_v2/injective_constants.py index 75dac9bfc0..cfa4c6a03f 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_constants.py +++ b/hummingbot/connector/exchange/injective_v2/injective_constants.py @@ -1,6 +1,6 @@ import sys -from hummingbot.core.api_throttler.data_types import RateLimit +from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit from hummingbot.core.data_type.in_flight_order import OrderState EXCHANGE_NAME = "injective_v2" @@ -18,14 +18,14 @@ # Public limit ids SPOT_MARKETS_LIMIT_ID = "SpotMarkets" DERIVATIVE_MARKETS_LIMIT_ID = "DerivativeMarkets" -DERIVATIVE_MARKET_LIMIT_ID = "DerivativeMarket" SPOT_ORDERBOOK_LIMIT_ID = "SpotOrderBookSnapshot" DERIVATIVE_ORDERBOOK_LIMIT_ID = "DerivativeOrderBookSnapshot" -GET_TRANSACTION_LIMIT_ID = "GetTransaction" -GET_CHAIN_TRANSACTION_LIMIT_ID = "GetChainTransaction" +GET_TRANSACTION_INDEXER_LIMIT_ID = "GetTransactionIndexer" +GET_TRANSACTION_CHAIN_LIMIT_ID = "GetTransactionChain" FUNDING_RATES_LIMIT_ID = "FundingRates" ORACLE_PRICES_LIMIT_ID = "OraclePrices" FUNDING_PAYMENTS_LIMIT_ID = "FundingPayments" +GET_SUBACCOUNT_LIMIT_ID = "GetSubaccount" # Private limit ids PORTFOLIO_BALANCES_LIMIT_ID = "AccountPortfolio" @@ -37,29 +37,116 @@ SIMULATE_TRANSACTION_LIMIT_ID = "SimulateTransaction" SEND_TRANSACTION = "SendTransaction" +CHAIN_ENDPOINTS_GROUP_LIMIT_ID = "ChainGroupLimit" +INDEXER_ENDPOINTS_GROUP_LIMIT_ID = "IndexerGroupLimit" + NO_LIMIT = sys.maxsize ONE_SECOND = 1 -RATE_LIMITS = [ - RateLimit(limit_id=SPOT_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKET_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_CHAIN_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=PORTFOLIO_BALANCES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=POSITIONS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SIMULATE_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SEND_TRANSACTION, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_RATES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=ORACLE_PRICES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_PAYMENTS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), +ENDPOINTS_RATE_LIMITS = [ + RateLimit( + limit_id=GET_SUBACCOUNT_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_CHAIN_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SIMULATE_TRANSACTION_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SEND_TRANSACTION, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_INDEXER_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=PORTFOLIO_BALANCES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=POSITIONS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_RATES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=ORACLE_PRICES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_PAYMENTS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), +] + +PUBLIC_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=20, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=50, time_interval=ONE_SECOND), +] +PUBLIC_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) + +CUSTOM_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), ] +CUSTOM_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) ORDER_STATE_MAP = { "booked": OrderState.OPEN, diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index 6b5f67a2f5..537b512c35 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -59,6 +59,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -84,7 +85,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py index 139c01cd7c..4636bdfa8d 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py @@ -1,12 +1,13 @@ from abc import ABC, abstractmethod from decimal import Decimal -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Union from pydantic import Field, SecretStr from pydantic.class_validators import validator from pyinjective.constant import Network from hummingbot.client.config.config_data_types import BaseClientModel, BaseConnectorConfigMap, ClientFieldData +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -16,6 +17,7 @@ from hummingbot.connector.exchange.injective_v2.data_sources.injective_vaults_data_source import ( InjectiveVaultsDataSource, ) +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.trade_fee import TradeFeeSchema if TYPE_CHECKING: @@ -68,6 +70,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.node == "lb" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS + class InjectiveTestnetNetworkMode(InjectiveNetworkMode): testnet_node: str = Field( @@ -78,6 +83,9 @@ class InjectiveTestnetNetworkMode(InjectiveNetworkMode): ), ) + class Config: + title = "testnet_network" + @validator("testnet_node", pre=True) def validate_node(cls, v: str): if v not in TESTNET_NODES: @@ -90,8 +98,8 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return True - class Config: - title = "testnet_network" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS class InjectiveCustomNetworkMode(InjectiveNetworkMode): @@ -169,6 +177,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.secure_connection + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.CUSTOM_NODE_RATE_LIMITS + NETWORK_MODES = { InjectiveMainnetNetworkMode.Config.title: InjectiveMainnetNetworkMode, @@ -180,7 +191,9 @@ def use_secure_connection(self) -> bool: class InjectiveAccountMode(BaseClientModel, ABC): @abstractmethod - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": pass @@ -219,7 +232,9 @@ class InjectiveDelegatedAccountMode(InjectiveAccountMode): class Config: title = "delegate_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveGranteeDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -227,6 +242,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " granter_subaccount_index=self.granter_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -263,7 +279,9 @@ class InjectiveVaultAccountMode(InjectiveAccountMode): class Config: title = "vault_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveVaultsDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -271,6 +289,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " vault_subaccount_index=self.vault_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -279,10 +298,13 @@ class InjectiveReadOnlyAccountMode(InjectiveAccountMode): class Config: title = "read_only_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveReadOnlyDataSource( network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -344,7 +366,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/setup.py b/setup.py index 5419d5ea4f..8e84f108a4 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,9 @@ def main(): "eth-utils", "ethsnarks-loopring", "flake8", + "gql", + "grpcio", + "grpcio-tools" "hexbytes", "importlib-metadata", "injective-py" diff --git a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py index 013a412d32..b2f39b1f28 100644 --- a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py +++ b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py @@ -10,6 +10,7 @@ from pyinjective.constant import Network from pyinjective.wallet import Address, PrivateKey +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -42,6 +43,7 @@ def setUp(self, _) -> None: granter_address=Address(bytes.fromhex(granter_private_key.to_public_key().to_hex())).to_acc_bech32(), granter_subaccount_index=0, network=Network.testnet(node="sentry"), + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() @@ -383,6 +385,7 @@ def setUp(self, _) -> None: vault_subaccount_index=1, network=Network.testnet(node="sentry"), use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() diff --git a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py similarity index 92% rename from test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py rename to test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py index 60b2e2b84e..6b8063b6e1 100644 --- a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py +++ b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py @@ -4,6 +4,7 @@ from pyinjective.constant import Network import hummingbot.connector.exchange.injective_v2.injective_v2_utils as utils +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -113,7 +114,11 @@ def test_injective_delegate_account_config_creation(self): granter_subaccount_index=0, ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveGranteeDataSource, type(data_source)) @@ -127,7 +132,11 @@ def test_injective_vault_account_config_creation(self): bytes.fromhex(private_key.to_public_key().to_hex())).to_acc_bech32(), ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveVaultsDataSource, type(data_source)) From 7571e891c683a6e65139b6e36af53c23753ac37b Mon Sep 17 00:00:00 2001 From: bczhang Date: Tue, 29 Aug 2023 16:39:03 +0800 Subject: [PATCH 07/31] fix error --- .../gate_io_perpetual/gate_io_perpetual_derivative.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hummingbot/connector/derivative/gate_io_perpetual/gate_io_perpetual_derivative.py b/hummingbot/connector/derivative/gate_io_perpetual/gate_io_perpetual_derivative.py index 093504cf38..33d7af3bcf 100644 --- a/hummingbot/connector/derivative/gate_io_perpetual/gate_io_perpetual_derivative.py +++ b/hummingbot/connector/derivative/gate_io_perpetual/gate_io_perpetual_derivative.py @@ -703,9 +703,14 @@ async def _update_positions(self): hb_trading_pair = await self.trading_pair_associated_to_exchange_symbol(ex_trading_pair) amount = Decimal(position.get("size")) - position_side = PositionSide.LONG if Decimal(position.get("size")) > 0 else PositionSide.SHORT - - pos_key = self._perpetual_trading.position_key(hb_trading_pair, position_side) + ex_mode = position.get("mode") + if ex_mode == 'single': + mode = PositionMode.ONEWAY + position_side = PositionSide.LONG if Decimal(position.get("size")) > 0 else PositionSide.SHORT + else: + mode = PositionMode.HEDGE + position_side = PositionSide.LONG if ex_mode == "dual_long" else PositionSide.SHORT + pos_key = self._perpetual_trading.position_key(hb_trading_pair, position_side, mode) if amount != 0: trading_rule = self._trading_rules[hb_trading_pair] From 6b40dec953e66188a83d681de3d5d23346ee1a20 Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 01:11:14 -0300 Subject: [PATCH 08/31] (feat) Added the new IP rate limits for Injective V2 --- .../injective_constants.py | 2 - .../injective_v2_perpetual_derivative.py | 3 +- .../injective_v2_perpetual_utils.py | 4 +- .../data_sources/injective_data_source.py | 12 +- .../injective_grantee_data_source.py | 45 +++--- .../injective_read_only_data_source.py | 12 +- .../injective_vaults_data_source.py | 10 +- .../injective_v2/injective_constants.py | 133 +++++++++++++++--- .../injective_v2/injective_v2_exchange.py | 3 +- .../injective_v2/injective_v2_utils.py | 40 ++++-- setup.py | 3 + .../test_injective_data_source.py | 3 + ...v2_utils.py => test_injective_v2_utils.py} | 13 +- 13 files changed, 212 insertions(+), 71 deletions(-) rename test/hummingbot/connector/exchange/injective_v2/{tests_injective_v2_utils.py => test_injective_v2_utils.py} (92%) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py index 3a58fec5d8..e474272bf1 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py @@ -7,8 +7,6 @@ TRANSACTIONS_CHECK_INTERVAL = CONSTANTS.TRANSACTIONS_CHECK_INTERVAL -RATE_LIMITS = CONSTANTS.RATE_LIMITS - ORDER_STATE_MAP = CONSTANTS.ORDER_STATE_MAP ORDER_NOT_FOUND_ERROR_MESSAGE = CONSTANTS.ORDER_NOT_FOUND_ERROR_MESSAGE diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index 3c528c7fc5..abad110787 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -60,6 +60,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -85,7 +86,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py index 5e5533b5e9..da2c346da3 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py @@ -73,7 +73,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index 6a5623d01b..f56b213b74 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -739,7 +739,7 @@ def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError @abstractmethod - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -804,9 +804,10 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( - chain_price=Decimal(trades_response["trades"][0]["price"]["price"])) + chain_price=Decimal(trades[0]["price"]["price"])) else: market = await self.derivative_market_info_for_id(market_id=market_id) @@ -815,7 +816,8 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( chain_price=Decimal(trades_response["trades"][0]["positionDelta"]["executionPrice"])) @@ -829,7 +831,7 @@ async def _transaction_from_chain(self, tx_hash: str, retries: int) -> int: while executed_tries < retries and not found: executed_tries += 1 try: - async with self.throttler.execute_task(limit_id=CONSTANTS.SPOT_ORDERS_HISTORY_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_CHAIN_LIMIT_ID): block_height = await self.query_executor.get_tx_block_height(tx_hash=tx_hash) found = True except ValueError: diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py index d50bc7bc8e..e514d7e1dc 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( granter_address: str, granter_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -253,13 +253,14 @@ async def initialize_trading_account(self): await self._client.get_account(address=self.trading_account_injective_address) self._is_trading_account_initialized = True - def order_hash_manager(self) -> OrderHashManager: + async def order_hash_manager(self) -> OrderHashManager: if self._order_hash_manager is None: - self._order_hash_manager = OrderHashManager( - address=self._granter_address, - network=self._network, - subaccount_indexes=[self._granter_subaccount_index] - ) + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_SUBACCOUNT_LIMIT_ID): + self._order_hash_manager = OrderHashManager( + address=self._granter_address, + network=self._network, + subaccount_indexes=[self._granter_subaccount_index] + ) return self._order_hash_manager def supported_order_types(self) -> List[OrderType]: @@ -278,7 +279,11 @@ async def update_markets(self): for market_info in markets: try: - ticker_base, ticker_quote = market_info["ticker"].split("/") + if "/" in market_info["ticker"]: + ticker_base, ticker_quote = market_info["ticker"].split("/") + else: + ticker_base = market_info["ticker"] + ticker_quote = None base_token = self._token_from_market_info( denom=market_info["baseDenom"], token_meta=market_info["baseTokenMeta"], @@ -339,7 +344,7 @@ async def order_updates_for_transaction( transaction_spot_orders = [] transaction_derivative_orders = [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -433,12 +438,14 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: return self._granter_subaccount_index == CONSTANTS.DEFAULT_SUBACCOUNT_INDEX - def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candidate_symbol: str) -> InjectiveToken: + def _token_from_market_info( + self, denom: str, token_meta: Dict[str, Any], candidate_symbol: Optional[str] = None + ) -> InjectiveToken: token = self._tokens_map.get(denom) if token is None: unique_symbol = token_meta["symbol"] if unique_symbol in self._token_symbol_symbol_and_denom_map: - if candidate_symbol not in self._token_symbol_symbol_and_denom_map: + if candidate_symbol is not None and candidate_symbol not in self._token_symbol_symbol_and_denom_map: unique_symbol = candidate_symbol else: unique_symbol = token_meta["name"] @@ -455,7 +462,9 @@ def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candid return token def _parse_derivative_market_info(self, market_info: Dict[str, Any]) -> InjectiveDerivativeMarket: - _, ticker_quote = market_info["ticker"].split("/") + ticker_quote = None + if "/" in market_info["ticker"]: + _, ticker_quote = market_info["ticker"].split("/") quote_token = self._token_from_market_info( denom=market_info["quoteDenom"], token_meta=market_info["quoteTokenMeta"], @@ -475,7 +484,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -484,7 +493,7 @@ def _calculate_order_hashes( derivative_hashes = [] if len(spot_orders) > 0 or len(derivative_orders) > 0: - hash_manager = self.order_hash_manager() + hash_manager = await self.order_hash_manager() hash_manager_result = hash_manager.compute_order_hashes( spot_orders=spot_orders, derivative_orders=derivative_orders, @@ -545,11 +554,11 @@ async def _order_creation_messages( order_definition = await self._create_derivative_order_definition(order=order) derivative_order_definitions.append(order_definition) - market_spot_hashes, market_derivative_hashes = self._calculate_order_hashes( + market_spot_hashes, market_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_market_order_definitions, derivative_orders=derivative_market_order_definitions, ) - limit_spot_hashes, limit_derivative_hashes = self._calculate_order_hashes( + limit_spot_hashes, limit_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_order_definitions, derivative_orders=derivative_order_definitions, ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py index 1a56e03f87..7c0850b0ad 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py @@ -21,6 +21,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType from hummingbot.core.data_type.in_flight_order import OrderUpdate from hummingbot.core.pubsub import PubSub @@ -33,6 +34,7 @@ class InjectiveReadOnlyDataSource(InjectiveDataSource): def __init__( self, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -45,9 +47,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._markets_initialization_lock = asyncio.Lock() self._spot_market_info_map: Optional[Dict[str, InjectiveSpotMarket]] = None @@ -317,8 +317,10 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError - def _calculate_order_hashes(self, spot_orders: List[GatewayInFlightOrder], - derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: + async def _calculate_order_hashes( + self, + spot_orders: List[GatewayInFlightOrder], + derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: raise NotImplementedError def _reset_order_hash_manager(self): diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py index 80d8904d16..39cc1da564 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( vault_contract_address: str, vault_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -324,7 +324,7 @@ async def order_updates_for_transaction( spot_orders = spot_orders or [] perpetual_orders = perpetual_orders or [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -441,7 +441,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] diff --git a/hummingbot/connector/exchange/injective_v2/injective_constants.py b/hummingbot/connector/exchange/injective_v2/injective_constants.py index 75dac9bfc0..cfa4c6a03f 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_constants.py +++ b/hummingbot/connector/exchange/injective_v2/injective_constants.py @@ -1,6 +1,6 @@ import sys -from hummingbot.core.api_throttler.data_types import RateLimit +from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit from hummingbot.core.data_type.in_flight_order import OrderState EXCHANGE_NAME = "injective_v2" @@ -18,14 +18,14 @@ # Public limit ids SPOT_MARKETS_LIMIT_ID = "SpotMarkets" DERIVATIVE_MARKETS_LIMIT_ID = "DerivativeMarkets" -DERIVATIVE_MARKET_LIMIT_ID = "DerivativeMarket" SPOT_ORDERBOOK_LIMIT_ID = "SpotOrderBookSnapshot" DERIVATIVE_ORDERBOOK_LIMIT_ID = "DerivativeOrderBookSnapshot" -GET_TRANSACTION_LIMIT_ID = "GetTransaction" -GET_CHAIN_TRANSACTION_LIMIT_ID = "GetChainTransaction" +GET_TRANSACTION_INDEXER_LIMIT_ID = "GetTransactionIndexer" +GET_TRANSACTION_CHAIN_LIMIT_ID = "GetTransactionChain" FUNDING_RATES_LIMIT_ID = "FundingRates" ORACLE_PRICES_LIMIT_ID = "OraclePrices" FUNDING_PAYMENTS_LIMIT_ID = "FundingPayments" +GET_SUBACCOUNT_LIMIT_ID = "GetSubaccount" # Private limit ids PORTFOLIO_BALANCES_LIMIT_ID = "AccountPortfolio" @@ -37,29 +37,116 @@ SIMULATE_TRANSACTION_LIMIT_ID = "SimulateTransaction" SEND_TRANSACTION = "SendTransaction" +CHAIN_ENDPOINTS_GROUP_LIMIT_ID = "ChainGroupLimit" +INDEXER_ENDPOINTS_GROUP_LIMIT_ID = "IndexerGroupLimit" + NO_LIMIT = sys.maxsize ONE_SECOND = 1 -RATE_LIMITS = [ - RateLimit(limit_id=SPOT_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKET_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_CHAIN_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=PORTFOLIO_BALANCES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=POSITIONS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SIMULATE_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SEND_TRANSACTION, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_RATES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=ORACLE_PRICES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_PAYMENTS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), +ENDPOINTS_RATE_LIMITS = [ + RateLimit( + limit_id=GET_SUBACCOUNT_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_CHAIN_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SIMULATE_TRANSACTION_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SEND_TRANSACTION, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_INDEXER_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=PORTFOLIO_BALANCES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=POSITIONS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_RATES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=ORACLE_PRICES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_PAYMENTS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), +] + +PUBLIC_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=20, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=50, time_interval=ONE_SECOND), +] +PUBLIC_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) + +CUSTOM_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), ] +CUSTOM_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) ORDER_STATE_MAP = { "booked": OrderState.OPEN, diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index 6b5f67a2f5..537b512c35 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -59,6 +59,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -84,7 +85,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py index 139c01cd7c..4636bdfa8d 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py @@ -1,12 +1,13 @@ from abc import ABC, abstractmethod from decimal import Decimal -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Union from pydantic import Field, SecretStr from pydantic.class_validators import validator from pyinjective.constant import Network from hummingbot.client.config.config_data_types import BaseClientModel, BaseConnectorConfigMap, ClientFieldData +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -16,6 +17,7 @@ from hummingbot.connector.exchange.injective_v2.data_sources.injective_vaults_data_source import ( InjectiveVaultsDataSource, ) +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.trade_fee import TradeFeeSchema if TYPE_CHECKING: @@ -68,6 +70,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.node == "lb" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS + class InjectiveTestnetNetworkMode(InjectiveNetworkMode): testnet_node: str = Field( @@ -78,6 +83,9 @@ class InjectiveTestnetNetworkMode(InjectiveNetworkMode): ), ) + class Config: + title = "testnet_network" + @validator("testnet_node", pre=True) def validate_node(cls, v: str): if v not in TESTNET_NODES: @@ -90,8 +98,8 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return True - class Config: - title = "testnet_network" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS class InjectiveCustomNetworkMode(InjectiveNetworkMode): @@ -169,6 +177,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.secure_connection + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.CUSTOM_NODE_RATE_LIMITS + NETWORK_MODES = { InjectiveMainnetNetworkMode.Config.title: InjectiveMainnetNetworkMode, @@ -180,7 +191,9 @@ def use_secure_connection(self) -> bool: class InjectiveAccountMode(BaseClientModel, ABC): @abstractmethod - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": pass @@ -219,7 +232,9 @@ class InjectiveDelegatedAccountMode(InjectiveAccountMode): class Config: title = "delegate_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveGranteeDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -227,6 +242,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " granter_subaccount_index=self.granter_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -263,7 +279,9 @@ class InjectiveVaultAccountMode(InjectiveAccountMode): class Config: title = "vault_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveVaultsDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -271,6 +289,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " vault_subaccount_index=self.vault_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -279,10 +298,13 @@ class InjectiveReadOnlyAccountMode(InjectiveAccountMode): class Config: title = "read_only_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveReadOnlyDataSource( network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -344,7 +366,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/setup.py b/setup.py index 5419d5ea4f..8e84f108a4 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,9 @@ def main(): "eth-utils", "ethsnarks-loopring", "flake8", + "gql", + "grpcio", + "grpcio-tools" "hexbytes", "importlib-metadata", "injective-py" diff --git a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py index 013a412d32..b2f39b1f28 100644 --- a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py +++ b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py @@ -10,6 +10,7 @@ from pyinjective.constant import Network from pyinjective.wallet import Address, PrivateKey +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -42,6 +43,7 @@ def setUp(self, _) -> None: granter_address=Address(bytes.fromhex(granter_private_key.to_public_key().to_hex())).to_acc_bech32(), granter_subaccount_index=0, network=Network.testnet(node="sentry"), + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() @@ -383,6 +385,7 @@ def setUp(self, _) -> None: vault_subaccount_index=1, network=Network.testnet(node="sentry"), use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() diff --git a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py similarity index 92% rename from test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py rename to test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py index 60b2e2b84e..6b8063b6e1 100644 --- a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py +++ b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py @@ -4,6 +4,7 @@ from pyinjective.constant import Network import hummingbot.connector.exchange.injective_v2.injective_v2_utils as utils +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -113,7 +114,11 @@ def test_injective_delegate_account_config_creation(self): granter_subaccount_index=0, ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveGranteeDataSource, type(data_source)) @@ -127,7 +132,11 @@ def test_injective_vault_account_config_creation(self): bytes.fromhex(private_key.to_public_key().to_hex())).to_acc_bech32(), ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveVaultsDataSource, type(data_source)) From 2a72765954ec8508f86c15fd2dada4192dd46244 Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 11:32:17 -0300 Subject: [PATCH 09/31] (fix) Relaxed version requirement for diff-cover tool --- setup/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/environment.yml b/setup/environment.yml index 8bc59cbe42..c17d8e98a0 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -37,7 +37,7 @@ dependencies: - commlib-py==0.10.6 - cryptography==3.4.7 - cython==3.0.0a10 - - diff-cover==5.1.2 + - diff-cover - docker==5.0.3 - eip712-structs==1.1.0 - ethsnarks-loopring==0.1.5 From 73978355818cf1636ac2549ad3b0d1480055a2ca Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 12:47:06 -0300 Subject: [PATCH 10/31] (fix) Solve issue in test_logger_mixin_for_tests.py that was causing random issues in other tests by not cleaning up the async loop correctly --- test/test_logger_mixin_for_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_logger_mixin_for_test.py b/test/test_logger_mixin_for_test.py index a94d81cfb5..6e076c28a3 100644 --- a/test/test_logger_mixin_for_test.py +++ b/test/test_logger_mixin_for_test.py @@ -8,7 +8,17 @@ class TestTestLoggerMixin(unittest.TestCase): def setUp(self): + super().setUp() self.logger = LoggerMixinForTest() + self._original_async_loop = asyncio.get_event_loop() + self.async_loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.async_loop) + + def tearDown(self) -> None: + super().tearDown() + self.async_loop.stop() + self.async_loop.close() + asyncio.set_event_loop(self._original_async_loop) def test_handle(self): self.logger.log_records = [] From c852538422fc4c43c3480431b538fedc60618db0 Mon Sep 17 00:00:00 2001 From: abel Date: Wed, 30 Aug 2023 01:06:03 -0300 Subject: [PATCH 11/31] (fix) Changed logic in status polling loop in ExchangePyBase to ensure no error interrupts the update process --- .../injective_v2_perpetual_derivative.py | 1 + .../data_sources/injective_data_source.py | 3 +++ .../injective_v2/injective_v2_exchange.py | 1 + hummingbot/connector/exchange_py_base.py | 18 +++++++++++++----- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index abad110787..9556b72b8a 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -544,6 +544,7 @@ def _update_order_after_creation_success( new_state=order.current_state, misc_updates=misc_updates, ) + self.logger().debug(f"\nCreated order {order.client_order_id} ({exchange_order_id}) with TX {misc_updates}") self._order_tracker.process_order_update(order_update) def _on_order_creation_failure( diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index f56b213b74..0f86b37c69 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -452,6 +452,8 @@ async def create_orders( except asyncio.CancelledError: raise except Exception as ex: + self.logger().debug( + f"Error broadcasting transaction to create orders (message: {order_creation_messages})") results = self._place_order_results( orders_to_create=spot_orders + perpetual_orders, order_hashes=spot_order_hashes + derivative_order_hashes, @@ -522,6 +524,7 @@ async def cancel_orders( except asyncio.CancelledError: raise except Exception as ex: + self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") results.extend([ CancelOrderResult( client_order_id=order.client_order_id, diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index 537b512c35..dd80a2667d 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -487,6 +487,7 @@ def _update_order_after_creation_success( new_state=order.current_state, misc_updates=misc_updates, ) + self.logger().debug(f"\nCreated order {order.client_order_id} ({exchange_order_id}) with TX {misc_updates}") self._order_tracker.process_order_update(order_update) def _on_order_creation_failure( diff --git a/hummingbot/connector/exchange_py_base.py b/hummingbot/connector/exchange_py_base.py index 48404d0e46..d07ac722da 100644 --- a/hummingbot/connector/exchange_py_base.py +++ b/hummingbot/connector/exchange_py_base.py @@ -934,11 +934,19 @@ async def _status_polling_loop_fetch_updates(self): ) async def _update_all_balances(self): - await self._update_balances() - if not self.real_time_balance_update: - # This is only required for exchanges that do not provide balance update notifications through websocket - self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self.in_flight_orders.items()} - self._in_flight_orders_snapshot_timestamp = self.current_timestamp + try: + await self._update_balances() + if not self.real_time_balance_update: + # This is only required for exchanges that do not provide balance update notifications through websocket + self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self.in_flight_orders.items()} + self._in_flight_orders_snapshot_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as request_error: + self.logger().warning( + f"Failed to update balances. Error: {request_error}", + exc_info=request_error, + ) async def _update_orders_fills(self, orders: List[InFlightOrder]): for order in orders: From 494a2b8ff4871bf7645414f121b7f612bee3f319 Mon Sep 17 00:00:00 2001 From: abel Date: Wed, 30 Aug 2023 13:07:09 -0300 Subject: [PATCH 12/31] (fix) Added logic to prevent the order creation and cancelation process from being canceled in the middle in Injective connectors --- .../injective_v2_perpetual_derivative.py | 10 ++++- .../data_sources/injective_data_source.py | 45 ++++++++++--------- .../injective_v2/injective_v2_exchange.py | 10 ++++- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index 9556b72b8a..00f3798738 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -1012,7 +1012,10 @@ def _process_transaction_event(self, transaction_event: Dict[str, Any]): async def _check_orders_transactions(self): while True: try: - await self._check_orders_creation_transactions() + # Executing the process shielded from this async task to isolate it from network disconnections + # (network disconnections cancel this task) + task = asyncio.create_task(self._check_orders_creation_transactions()) + await asyncio.shield(task) await self._sleep(CONSTANTS.TRANSACTIONS_CHECK_INTERVAL) except NotImplementedError: raise @@ -1088,7 +1091,10 @@ async def _check_created_orders_status_for_transaction(self, transaction_hash: s async def _process_queued_orders(self): while True: try: - await self._cancel_and_create_queued_orders() + # Executing the batch cancelation and creation process shielded from this async task to isolate the + # creation/cancelation process from network disconnections (network disconnections cancel this task) + task = asyncio.create_task(self._cancel_and_create_queued_orders()) + await asyncio.shield(task) sleep_time = (self.clock.tick_size * 0.5 if self.clock is not None else self._orders_processing_delta_time) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index 0f86b37c69..956eebb910 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -503,35 +503,36 @@ async def cancel_orders( derivative_orders_data.append(order_data) orders_with_hash.append(order) - delegated_message = self._order_cancel_message( - spot_orders_to_cancel=spot_orders_data, - derivative_orders_to_cancel=derivative_orders_data, - ) + if len(orders_with_hash) > 0: + delegated_message = self._order_cancel_message( + spot_orders_to_cancel=spot_orders_data, + derivative_orders_to_cancel=derivative_orders_data, + ) - try: - result = await self._send_in_transaction(messages=[delegated_message]) - if result["rawLog"] != "[]": - raise ValueError(f"Error sending the order cancel transaction ({result['rawLog']})") - else: - cancel_transaction_hash = result.get("txhash", "") + try: + result = await self._send_in_transaction(messages=[delegated_message]) + if result["rawLog"] != "[]": + raise ValueError(f"Error sending the order cancel transaction ({result['rawLog']})") + else: + cancel_transaction_hash = result.get("txhash", "") + results.extend([ + CancelOrderResult( + client_order_id=order.client_order_id, + trading_pair=order.trading_pair, + misc_updates={"cancelation_transaction_hash": cancel_transaction_hash}, + ) for order in orders_with_hash + ]) + except asyncio.CancelledError: + raise + except Exception as ex: + self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") results.extend([ CancelOrderResult( client_order_id=order.client_order_id, trading_pair=order.trading_pair, - misc_updates={"cancelation_transaction_hash": cancel_transaction_hash}, + exception=ex, ) for order in orders_with_hash ]) - except asyncio.CancelledError: - raise - except Exception as ex: - self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") - results.extend([ - CancelOrderResult( - client_order_id=order.client_order_id, - trading_pair=order.trading_pair, - exception=ex, - ) for order in orders_with_hash - ]) return results diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index dd80a2667d..125fe4caa8 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -898,7 +898,10 @@ def _process_transaction_event(self, transaction_event: Dict[str, Any]): async def _check_orders_transactions(self): while True: try: - await self._check_orders_creation_transactions() + # Executing the process shielded from this async task to isolate it from network disconnections + # (network disconnections cancel this task) + task = asyncio.create_task(self._check_orders_creation_transactions()) + await asyncio.shield(task) await self._sleep(CONSTANTS.TRANSACTIONS_CHECK_INTERVAL) except NotImplementedError: raise @@ -974,7 +977,10 @@ async def _check_created_orders_status_for_transaction(self, transaction_hash: s async def _process_queued_orders(self): while True: try: - await self._cancel_and_create_queued_orders() + # Executing the batch cancelation and creation process shielded from this async task to isolate the + # creation/cancelation process from network disconnections (network disconnections cancel this task) + task = asyncio.create_task(self._cancel_and_create_queued_orders()) + await asyncio.shield(task) sleep_time = (self.clock.tick_size * 0.5 if self.clock is not None else self._orders_processing_delta_time) From 5de82f82547feab22e3b8a821b695c49a5108ffc Mon Sep 17 00:00:00 2001 From: bczhang Date: Thu, 31 Aug 2023 11:05:02 +0800 Subject: [PATCH 13/31] add unittest --- .../gate_io_perpetual/test_gate_io_perpetual_derivative.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py index 81c9b74f06..26111933fc 100644 --- a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py @@ -843,6 +843,7 @@ def configure_successful_set_position_mode( "user": 1666, "currency": "USDT", "total": "9707.803567115145", + "size": "9707.803567115145", "unrealised_pnl": "3371.248828", "position_margin": "38.712189181", "order_margin": "0", @@ -850,6 +851,7 @@ def configure_successful_set_position_mode( "point": "0", "bonus": "0", "in_dual_mode": True if position_mode is PositionMode.HEDGE else False, + "mode": "single" if position_mode is PositionMode.ONEWAY else "dual_long", "history": { "dnw": "10000", "pnl": "68.3685", From 985256ec75f814702ca34c684aa52d17217c90f5 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 18:11:36 -0300 Subject: [PATCH 14/31] (fix) Changes in environment.yml to solve issues with grpc libraries binaries for Mac incorrectly compiles (for M1 and M2) --- .../exchange/bitfinex/bitfinex_exchange.pyx | 2 +- setup/environment.yml | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx index d349a911c8..1c945c296e 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx @@ -475,7 +475,7 @@ cdef class BitfinexExchange(ExchangeBase): http_method: str, url, headers, - data_str: Optional[str, list] = None) -> list: + data_str = None) -> list: """ A wrapper for submitting API requests to Bitfinex :returns: json data from the endpoints diff --git a/setup/environment.yml b/setup/environment.yml index c17d8e98a0..6b1310fad6 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -1,27 +1,28 @@ name: hummingbot channels: + - conda-forge - defaults dependencies: - bidict - - coverage=5.5 + - coverage + - cython=3.0 + - grpcio-tools - nomkl - nose=1.3.7 - nose-exclude - numpy=1.23.5 - numpy-base=1.23.5 - pandas=1.5.3 - - pip=23.1.2 + - pip - prompt_toolkit=3.0.20 - - pydantic=1.9.* + - pydantic=1.10 - pytest - - python=3.10.12 - - pytables=3.8.0 + - python=3.10 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 - - typing-extensions<4.6.0 - ujson - - zlib=1.2.13 + - zlib - pip: - aiohttp==3.* - aioprocessing==2.0 @@ -34,23 +35,19 @@ dependencies: - appnope==0.1.3 - base58==2.1.1 - cachetools==4.0.0 - - commlib-py==0.10.6 + - commlib-py==0.10 - cryptography==3.4.7 - - cython==3.0.0a10 - diff-cover - docker==5.0.3 - eip712-structs==1.1.0 - ethsnarks-loopring==0.1.5 - flake8==3.7.9 - gql - - grpcio - - grpcio-tools - importlib-metadata==0.23 - injective-py==0.7.* - mypy-extensions==0.4.3 - pandas_ta==0.3.14b - pre-commit==2.18.1 - - protobuf>=4 - psutil==5.7.2 - ptpython==3.0.20 - pyjwt==1.7.1 From b051273249d3a62c425dfa5ca6f5771f7f7ae5e9 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 18:35:21 -0300 Subject: [PATCH 15/31] (fix) Solve dependencies issues for Mac M1 and M2 --- setup/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/environment.yml b/setup/environment.yml index 6b1310fad6..0abf03ce3e 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -5,7 +5,7 @@ channels: dependencies: - bidict - coverage - - cython=3.0 + - cython=3.0.0 - grpcio-tools - nomkl - nose=1.3.7 @@ -17,7 +17,7 @@ dependencies: - prompt_toolkit=3.0.20 - pydantic=1.10 - pytest - - python=3.10 + - python=3.10.12 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 From fc3bebaa7c56a40761ca95413c022109d5665345 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 19:27:44 -0300 Subject: [PATCH 16/31] (fix) Solve errors tryin to upgrade cython version --- setup/environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/environment.yml b/setup/environment.yml index 0abf03ce3e..b6b775b8a4 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -5,7 +5,6 @@ channels: dependencies: - bidict - coverage - - cython=3.0.0 - grpcio-tools - nomkl - nose=1.3.7 @@ -17,7 +16,7 @@ dependencies: - prompt_toolkit=3.0.20 - pydantic=1.10 - pytest - - python=3.10.12 + - python=3.10 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 @@ -35,8 +34,9 @@ dependencies: - appnope==0.1.3 - base58==2.1.1 - cachetools==4.0.0 - - commlib-py==0.10 + - commlib-py==0.10.6 - cryptography==3.4.7 + - cython==3.0.0a10 - diff-cover - docker==5.0.3 - eip712-structs==1.1.0 From 9de5ab59ba7f72a40de18fdf875f25e0b13edb0e Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 1 Sep 2023 00:14:59 -0300 Subject: [PATCH 17/31] (fix) Added logic in `install` script to reinstall grpcio with pip for Mac Intel because the conda grpcio binary for Mac Intel is broken --- install | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install b/install index 4a42eb6b4c..a49b9791c0 100755 --- a/install +++ b/install @@ -35,3 +35,11 @@ conda develop . pip install objgraph pre-commit install + +OS=`uname` +ARCH=`uname -m` + +if [[ "$OS" = "Darwin" && "$ARCH" = "x86_64" ]]; then + pip install grpcio --ignore-installed +fi + From b6cc57c2ec2be74cf123daf1cff226f0bc0bd66b Mon Sep 17 00:00:00 2001 From: bczhang Date: Fri, 1 Sep 2023 12:37:59 +0800 Subject: [PATCH 18/31] update unittest --- .../test_gate_io_perpetual_derivative.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py index 26111933fc..8ec2d041c2 100644 --- a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py @@ -1712,3 +1712,97 @@ def test_create_buy_limit_maker_order_successfully(self, mock_api): f"{Decimal('100.000000')} to {PositionAction.OPEN.name} a {self.trading_pair} position." ) ) + + @aioresponses() + def test_update_position_mode( + self, + mock_api: aioresponses, + ): + self._simulate_trading_rules_initialized() + get_position_url = web_utils.public_rest_url( + endpoint=CONSTANTS.POSITION_INFORMATION_URL + ) + regex_get_position_url = re.compile(f"^{get_position_url}") + response = [ + { + "user": 10000, + "contract": "BTC_USDT", + "size": 9440, + "leverage": "0", + "risk_limit": "100", + "leverage_max": "100", + "maintenance_rate": "0.005", + "value": "2.497143098997", + "margin": "4.431548146258", + "entry_price": "3779.55", + "liq_price": "99999999", + "mark_price": "3780.32", + "unrealised_pnl": "-0.000507486844", + "realised_pnl": "0.045543982432", + "history_pnl": "0", + "last_close_pnl": "0", + "realised_point": "0", + "history_point": "0", + "adl_ranking": 5, + "pending_orders": 16, + "close_order": { + "id": 232323, + "price": "3779", + "is_liq": False + }, + "mode": "single", + "update_time": 1684994406, + "cross_leverage_limit": "0" + } + ] + mock_api.get(regex_get_position_url, body=json.dumps(response)) + self.async_run_with_timeout(self.exchange._update_positions()) + + + position: Position = self.exchange.account_positions[self.trading_pair] + self.assertEqual(self.trading_pair, position.trading_pair) + self.assertEqual(PositionSide.LONG, position.position_side) + + get_position_url = web_utils.public_rest_url( + endpoint=CONSTANTS.POSITION_INFORMATION_URL + ) + regex_get_position_url = re.compile(f"^{get_position_url}") + response = [ + { + "user": 10000, + "contract": "BTC_USDT", + "size": 9440, + "leverage": "0", + "risk_limit": "100", + "leverage_max": "100", + "maintenance_rate": "0.005", + "value": "2.497143098997", + "margin": "4.431548146258", + "entry_price": "3779.55", + "liq_price": "99999999", + "mark_price": "3780.32", + "unrealised_pnl": "-0.000507486844", + "realised_pnl": "0.045543982432", + "history_pnl": "0", + "last_close_pnl": "0", + "realised_point": "0", + "history_point": "0", + "adl_ranking": 5, + "pending_orders": 16, + "close_order": { + "id": 232323, + "price": "3779", + "is_liq": False + }, + "mode": "dual_long", + "update_time": 1684994406, + "cross_leverage_limit": "0" + } + ] + mock_api.get(regex_get_position_url, body=json.dumps(response)) + self.async_run_with_timeout(self.exchange._update_positions()) + + + position: Position = self.exchange.account_positions[f"{self.trading_pair}LONG"] + self.assertEqual(self.trading_pair, position.trading_pair) + self.assertEqual(PositionSide.LONG, position.position_side) From a450862456841c04acb9cd768c032c75347e48de Mon Sep 17 00:00:00 2001 From: bczhang Date: Fri, 1 Sep 2023 13:11:44 +0800 Subject: [PATCH 19/31] update unittest --- .../test_gate_io_perpetual_derivative.py | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py index 8ec2d041c2..70e46be7a3 100644 --- a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py @@ -1724,41 +1724,40 @@ def test_update_position_mode( ) regex_get_position_url = re.compile(f"^{get_position_url}") response = [ - { - "user": 10000, - "contract": "BTC_USDT", - "size": 9440, - "leverage": "0", - "risk_limit": "100", - "leverage_max": "100", - "maintenance_rate": "0.005", - "value": "2.497143098997", - "margin": "4.431548146258", - "entry_price": "3779.55", - "liq_price": "99999999", - "mark_price": "3780.32", - "unrealised_pnl": "-0.000507486844", - "realised_pnl": "0.045543982432", - "history_pnl": "0", - "last_close_pnl": "0", - "realised_point": "0", - "history_point": "0", - "adl_ranking": 5, - "pending_orders": 16, - "close_order": { - "id": 232323, - "price": "3779", - "is_liq": False - }, - "mode": "single", - "update_time": 1684994406, - "cross_leverage_limit": "0" - } - ] + { + "user": 10000, + "contract": "BTC_USDT", + "size": 9440, + "leverage": "0", + "risk_limit": "100", + "leverage_max": "100", + "maintenance_rate": "0.005", + "value": "2.497143098997", + "margin": "4.431548146258", + "entry_price": "3779.55", + "liq_price": "99999999", + "mark_price": "3780.32", + "unrealised_pnl": "-0.000507486844", + "realised_pnl": "0.045543982432", + "history_pnl": "0", + "last_close_pnl": "0", + "realised_point": "0", + "history_point": "0", + "adl_ranking": 5, + "pending_orders": 16, + "close_order": { + "id": 232323, + "price": "3779", + "is_liq": False + }, + "mode": "single", + "update_time": 1684994406, + "cross_leverage_limit": "0" + } + ] mock_api.get(regex_get_position_url, body=json.dumps(response)) self.async_run_with_timeout(self.exchange._update_positions()) - position: Position = self.exchange.account_positions[self.trading_pair] self.assertEqual(self.trading_pair, position.trading_pair) self.assertEqual(PositionSide.LONG, position.position_side) From 5278a18de7e6a86710747deb20cc66935521f034 Mon Sep 17 00:00:00 2001 From: bczhang Date: Fri, 1 Sep 2023 13:26:20 +0800 Subject: [PATCH 20/31] update unittest --- .../gate_io_perpetual/test_gate_io_perpetual_derivative.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py index 70e46be7a3..3d486e99dd 100644 --- a/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/gate_io_perpetual/test_gate_io_perpetual_derivative.py @@ -1800,8 +1800,6 @@ def test_update_position_mode( ] mock_api.get(regex_get_position_url, body=json.dumps(response)) self.async_run_with_timeout(self.exchange._update_positions()) - - position: Position = self.exchange.account_positions[f"{self.trading_pair}LONG"] self.assertEqual(self.trading_pair, position.trading_pair) self.assertEqual(PositionSide.LONG, position.position_side) From 4553553338604776efb2e00ed90f09104f023594 Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 1 Sep 2023 09:33:03 -0300 Subject: [PATCH 21/31] (fix) Added comment to explain the change in `install` script --- install | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install b/install index a49b9791c0..123662a956 100755 --- a/install +++ b/install @@ -36,6 +36,10 @@ pip install objgraph pre-commit install +# The following logic is required to replace the grpcio package installed from conda binaries in Mac Intel +# for binaries from Pypi. We need to do this because the conda binaries fro Mac Intel are broken. +# We can't use the Pypi binaries universally because they are broken for Mac ARM (M1 and M2). +# This logic can be removed once the grpcio conda binaries for Mac Intel are fixed OS=`uname` ARCH=`uname -m` From 08ef60c9e9fe281ece1d70b82a8f7c791a0723ff Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 01:11:14 -0300 Subject: [PATCH 22/31] (feat) Added the new IP rate limits for Injective V2 --- .../injective_constants.py | 2 - .../injective_v2_perpetual_derivative.py | 3 +- .../injective_v2_perpetual_utils.py | 4 +- .../data_sources/injective_data_source.py | 12 +- .../injective_grantee_data_source.py | 45 +++--- .../injective_read_only_data_source.py | 12 +- .../injective_vaults_data_source.py | 10 +- .../injective_v2/injective_constants.py | 133 +++++++++++++++--- .../injective_v2/injective_v2_exchange.py | 3 +- .../injective_v2/injective_v2_utils.py | 40 ++++-- setup.py | 3 + .../test_injective_data_source.py | 3 + ...v2_utils.py => test_injective_v2_utils.py} | 13 +- 13 files changed, 212 insertions(+), 71 deletions(-) rename test/hummingbot/connector/exchange/injective_v2/{tests_injective_v2_utils.py => test_injective_v2_utils.py} (92%) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py index 3a58fec5d8..e474272bf1 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_constants.py @@ -7,8 +7,6 @@ TRANSACTIONS_CHECK_INTERVAL = CONSTANTS.TRANSACTIONS_CHECK_INTERVAL -RATE_LIMITS = CONSTANTS.RATE_LIMITS - ORDER_STATE_MAP = CONSTANTS.ORDER_STATE_MAP ORDER_NOT_FOUND_ERROR_MESSAGE = CONSTANTS.ORDER_NOT_FOUND_ERROR_MESSAGE diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index 3c528c7fc5..abad110787 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -60,6 +60,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -85,7 +86,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py index 5e5533b5e9..da2c346da3 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_utils.py @@ -73,7 +73,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index 6a5623d01b..f56b213b74 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -739,7 +739,7 @@ def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError @abstractmethod - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -804,9 +804,10 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( - chain_price=Decimal(trades_response["trades"][0]["price"]["price"])) + chain_price=Decimal(trades[0]["price"]["price"])) else: market = await self.derivative_market_info_for_id(market_id=market_id) @@ -815,7 +816,8 @@ async def _last_traded_price(self, market_id: str) -> Decimal: market_ids=[market_id], limit=1, ) - if len(trades_response["trades"]) > 0: + trades = trades_response.get("trades", []) + if len(trades) > 0: price = market.price_from_chain_format( chain_price=Decimal(trades_response["trades"][0]["positionDelta"]["executionPrice"])) @@ -829,7 +831,7 @@ async def _transaction_from_chain(self, tx_hash: str, retries: int) -> int: while executed_tries < retries and not found: executed_tries += 1 try: - async with self.throttler.execute_task(limit_id=CONSTANTS.SPOT_ORDERS_HISTORY_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_CHAIN_LIMIT_ID): block_height = await self.query_executor.get_tx_block_height(tx_hash=tx_hash) found = True except ValueError: diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py index d50bc7bc8e..e514d7e1dc 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_grantee_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( granter_address: str, granter_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -253,13 +253,14 @@ async def initialize_trading_account(self): await self._client.get_account(address=self.trading_account_injective_address) self._is_trading_account_initialized = True - def order_hash_manager(self) -> OrderHashManager: + async def order_hash_manager(self) -> OrderHashManager: if self._order_hash_manager is None: - self._order_hash_manager = OrderHashManager( - address=self._granter_address, - network=self._network, - subaccount_indexes=[self._granter_subaccount_index] - ) + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_SUBACCOUNT_LIMIT_ID): + self._order_hash_manager = OrderHashManager( + address=self._granter_address, + network=self._network, + subaccount_indexes=[self._granter_subaccount_index] + ) return self._order_hash_manager def supported_order_types(self) -> List[OrderType]: @@ -278,7 +279,11 @@ async def update_markets(self): for market_info in markets: try: - ticker_base, ticker_quote = market_info["ticker"].split("/") + if "/" in market_info["ticker"]: + ticker_base, ticker_quote = market_info["ticker"].split("/") + else: + ticker_base = market_info["ticker"] + ticker_quote = None base_token = self._token_from_market_info( denom=market_info["baseDenom"], token_meta=market_info["baseTokenMeta"], @@ -339,7 +344,7 @@ async def order_updates_for_transaction( transaction_spot_orders = [] transaction_derivative_orders = [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -433,12 +438,14 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: return self._granter_subaccount_index == CONSTANTS.DEFAULT_SUBACCOUNT_INDEX - def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candidate_symbol: str) -> InjectiveToken: + def _token_from_market_info( + self, denom: str, token_meta: Dict[str, Any], candidate_symbol: Optional[str] = None + ) -> InjectiveToken: token = self._tokens_map.get(denom) if token is None: unique_symbol = token_meta["symbol"] if unique_symbol in self._token_symbol_symbol_and_denom_map: - if candidate_symbol not in self._token_symbol_symbol_and_denom_map: + if candidate_symbol is not None and candidate_symbol not in self._token_symbol_symbol_and_denom_map: unique_symbol = candidate_symbol else: unique_symbol = token_meta["name"] @@ -455,7 +462,9 @@ def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candid return token def _parse_derivative_market_info(self, market_info: Dict[str, Any]) -> InjectiveDerivativeMarket: - _, ticker_quote = market_info["ticker"].split("/") + ticker_quote = None + if "/" in market_info["ticker"]: + _, ticker_quote = market_info["ticker"].split("/") quote_token = self._token_from_market_info( denom=market_info["quoteDenom"], token_meta=market_info["quoteTokenMeta"], @@ -475,7 +484,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] @@ -484,7 +493,7 @@ def _calculate_order_hashes( derivative_hashes = [] if len(spot_orders) > 0 or len(derivative_orders) > 0: - hash_manager = self.order_hash_manager() + hash_manager = await self.order_hash_manager() hash_manager_result = hash_manager.compute_order_hashes( spot_orders=spot_orders, derivative_orders=derivative_orders, @@ -545,11 +554,11 @@ async def _order_creation_messages( order_definition = await self._create_derivative_order_definition(order=order) derivative_order_definitions.append(order_definition) - market_spot_hashes, market_derivative_hashes = self._calculate_order_hashes( + market_spot_hashes, market_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_market_order_definitions, derivative_orders=derivative_market_order_definitions, ) - limit_spot_hashes, limit_derivative_hashes = self._calculate_order_hashes( + limit_spot_hashes, limit_derivative_hashes = await self._calculate_order_hashes( spot_orders=spot_order_definitions, derivative_orders=derivative_order_definitions, ) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py index 1a56e03f87..7c0850b0ad 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py @@ -21,6 +21,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType from hummingbot.core.data_type.in_flight_order import OrderUpdate from hummingbot.core.pubsub import PubSub @@ -33,6 +34,7 @@ class InjectiveReadOnlyDataSource(InjectiveDataSource): def __init__( self, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -45,9 +47,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._markets_initialization_lock = asyncio.Lock() self._spot_market_info_map: Optional[Dict[str, InjectiveSpotMarket]] = None @@ -317,8 +317,10 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: raise NotImplementedError - def _calculate_order_hashes(self, spot_orders: List[GatewayInFlightOrder], - derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: + async def _calculate_order_hashes( + self, + spot_orders: List[GatewayInFlightOrder], + derivative_orders: [GatewayPerpetualInFlightOrder]) -> Tuple[List[str], List[str]]: raise NotImplementedError def _reset_order_hash_manager(self): diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py index 80d8904d16..39cc1da564 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py @@ -27,6 +27,7 @@ from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.api_throttler.async_throttler_base import AsyncThrottlerBase +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.in_flight_order import OrderState, OrderUpdate from hummingbot.core.pubsub import PubSub @@ -43,6 +44,7 @@ def __init__( vault_contract_address: str, vault_subaccount_index: int, network: Network, + rate_limits: List[RateLimit], use_secure_connection: bool = True): self._network = network self._client = AsyncClient( @@ -75,9 +77,7 @@ def __init__( self._publisher = PubSub() self._last_received_message_time = 0 self._order_creation_lock = asyncio.Lock() - # We create a throttler instance here just to have a fully valid instance from the first moment. - # The connector using this data source should replace the throttler with the one used by the connector. - self._throttler = AsyncThrottler(rate_limits=CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(rate_limits=rate_limits) self._is_timeout_height_initialized = False self._is_trading_account_initialized = False @@ -324,7 +324,7 @@ async def order_updates_for_transaction( spot_orders = spot_orders or [] perpetual_orders = perpetual_orders or [] - async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_LIMIT_ID): + async with self.throttler.execute_task(limit_id=CONSTANTS.GET_TRANSACTION_INDEXER_LIMIT_ID): transaction_info = await self.query_executor.get_tx_by_hash(tx_hash=transaction_hash) transaction_messages = json.loads(base64.b64decode(transaction_info["data"]["messages"]).decode()) @@ -441,7 +441,7 @@ async def _updated_derivative_market_info_for_id(self, market_id: str) -> Inject market = self._parse_derivative_market_info(market_info=market_info) return market - def _calculate_order_hashes( + async def _calculate_order_hashes( self, spot_orders: List[GatewayInFlightOrder], derivative_orders: [GatewayPerpetualInFlightOrder] diff --git a/hummingbot/connector/exchange/injective_v2/injective_constants.py b/hummingbot/connector/exchange/injective_v2/injective_constants.py index 75dac9bfc0..cfa4c6a03f 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_constants.py +++ b/hummingbot/connector/exchange/injective_v2/injective_constants.py @@ -1,6 +1,6 @@ import sys -from hummingbot.core.api_throttler.data_types import RateLimit +from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit from hummingbot.core.data_type.in_flight_order import OrderState EXCHANGE_NAME = "injective_v2" @@ -18,14 +18,14 @@ # Public limit ids SPOT_MARKETS_LIMIT_ID = "SpotMarkets" DERIVATIVE_MARKETS_LIMIT_ID = "DerivativeMarkets" -DERIVATIVE_MARKET_LIMIT_ID = "DerivativeMarket" SPOT_ORDERBOOK_LIMIT_ID = "SpotOrderBookSnapshot" DERIVATIVE_ORDERBOOK_LIMIT_ID = "DerivativeOrderBookSnapshot" -GET_TRANSACTION_LIMIT_ID = "GetTransaction" -GET_CHAIN_TRANSACTION_LIMIT_ID = "GetChainTransaction" +GET_TRANSACTION_INDEXER_LIMIT_ID = "GetTransactionIndexer" +GET_TRANSACTION_CHAIN_LIMIT_ID = "GetTransactionChain" FUNDING_RATES_LIMIT_ID = "FundingRates" ORACLE_PRICES_LIMIT_ID = "OraclePrices" FUNDING_PAYMENTS_LIMIT_ID = "FundingPayments" +GET_SUBACCOUNT_LIMIT_ID = "GetSubaccount" # Private limit ids PORTFOLIO_BALANCES_LIMIT_ID = "AccountPortfolio" @@ -37,29 +37,116 @@ SIMULATE_TRANSACTION_LIMIT_ID = "SimulateTransaction" SEND_TRANSACTION = "SendTransaction" +CHAIN_ENDPOINTS_GROUP_LIMIT_ID = "ChainGroupLimit" +INDEXER_ENDPOINTS_GROUP_LIMIT_ID = "IndexerGroupLimit" + NO_LIMIT = sys.maxsize ONE_SECOND = 1 -RATE_LIMITS = [ - RateLimit(limit_id=SPOT_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKETS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_MARKET_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=GET_CHAIN_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=PORTFOLIO_BALANCES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=POSITIONS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SPOT_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=DERIVATIVE_TRADES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SIMULATE_TRANSACTION_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=SEND_TRANSACTION, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_RATES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=ORACLE_PRICES_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), - RateLimit(limit_id=FUNDING_PAYMENTS_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), +ENDPOINTS_RATE_LIMITS = [ + RateLimit( + limit_id=GET_SUBACCOUNT_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_CHAIN_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SIMULATE_TRANSACTION_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SEND_TRANSACTION, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(CHAIN_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_MARKETS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERBOOK_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=GET_TRANSACTION_INDEXER_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=PORTFOLIO_BALANCES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=POSITIONS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_ORDERS_HISTORY_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=SPOT_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=DERIVATIVE_TRADES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_RATES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=ORACLE_PRICES_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), + RateLimit( + limit_id=FUNDING_PAYMENTS_LIMIT_ID, + limit=NO_LIMIT, + time_interval=ONE_SECOND, + linked_limits=[LinkedLimitWeightPair(INDEXER_ENDPOINTS_GROUP_LIMIT_ID)]), +] + +PUBLIC_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=20, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=50, time_interval=ONE_SECOND), +] +PUBLIC_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) + +CUSTOM_NODE_RATE_LIMITS = [ + RateLimit(limit_id=CHAIN_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), + RateLimit(limit_id=INDEXER_ENDPOINTS_GROUP_LIMIT_ID, limit=NO_LIMIT, time_interval=ONE_SECOND), ] +CUSTOM_NODE_RATE_LIMITS.extend(ENDPOINTS_RATE_LIMITS) ORDER_STATE_MAP = { "booked": OrderState.OPEN, diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index 6b5f67a2f5..537b512c35 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -59,6 +59,7 @@ def __init__( self._trading_required = trading_required self._trading_pairs = trading_pairs self._data_source = connector_configuration.create_data_source() + self._rate_limits = connector_configuration.network.rate_limits() super().__init__(client_config_map=client_config_map) self._data_source.configure_throttler(throttler=self._throttler) @@ -84,7 +85,7 @@ def authenticator(self) -> AuthBase: @property def rate_limits_rules(self) -> List[RateLimit]: - return CONSTANTS.RATE_LIMITS + return self._rate_limits @property def domain(self) -> str: diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py index 139c01cd7c..4636bdfa8d 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_utils.py @@ -1,12 +1,13 @@ from abc import ABC, abstractmethod from decimal import Decimal -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Union from pydantic import Field, SecretStr from pydantic.class_validators import validator from pyinjective.constant import Network from hummingbot.client.config.config_data_types import BaseClientModel, BaseConnectorConfigMap, ClientFieldData +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -16,6 +17,7 @@ from hummingbot.connector.exchange.injective_v2.data_sources.injective_vaults_data_source import ( InjectiveVaultsDataSource, ) +from hummingbot.core.api_throttler.data_types import RateLimit from hummingbot.core.data_type.trade_fee import TradeFeeSchema if TYPE_CHECKING: @@ -68,6 +70,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.node == "lb" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS + class InjectiveTestnetNetworkMode(InjectiveNetworkMode): testnet_node: str = Field( @@ -78,6 +83,9 @@ class InjectiveTestnetNetworkMode(InjectiveNetworkMode): ), ) + class Config: + title = "testnet_network" + @validator("testnet_node", pre=True) def validate_node(cls, v: str): if v not in TESTNET_NODES: @@ -90,8 +98,8 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return True - class Config: - title = "testnet_network" + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.PUBLIC_NODE_RATE_LIMITS class InjectiveCustomNetworkMode(InjectiveNetworkMode): @@ -169,6 +177,9 @@ def network(self) -> Network: def use_secure_connection(self) -> bool: return self.secure_connection + def rate_limits(self) -> List[RateLimit]: + return CONSTANTS.CUSTOM_NODE_RATE_LIMITS + NETWORK_MODES = { InjectiveMainnetNetworkMode.Config.title: InjectiveMainnetNetworkMode, @@ -180,7 +191,9 @@ def use_secure_connection(self) -> bool: class InjectiveAccountMode(BaseClientModel, ABC): @abstractmethod - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": pass @@ -219,7 +232,9 @@ class InjectiveDelegatedAccountMode(InjectiveAccountMode): class Config: title = "delegate_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveGranteeDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -227,6 +242,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " granter_subaccount_index=self.granter_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -263,7 +279,9 @@ class InjectiveVaultAccountMode(InjectiveAccountMode): class Config: title = "vault_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveVaultsDataSource( private_key=self.private_key.get_secret_value(), subaccount_index=self.subaccount_index, @@ -271,6 +289,7 @@ def create_data_source(self, network: Network, use_secure_connection: bool) -> " vault_subaccount_index=self.vault_subaccount_index, network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -279,10 +298,13 @@ class InjectiveReadOnlyAccountMode(InjectiveAccountMode): class Config: title = "read_only_account" - def create_data_source(self, network: Network, use_secure_connection: bool) -> "InjectiveDataSource": + def create_data_source( + self, network: Network, use_secure_connection: bool, rate_limits: List[RateLimit], + ) -> "InjectiveDataSource": return InjectiveReadOnlyDataSource( network=network, use_secure_connection=use_secure_connection, + rate_limits=rate_limits, ) @@ -344,7 +366,9 @@ def validate_account_type(cls, v: Union[(str, Dict) + tuple(ACCOUNT_MODES.values def create_data_source(self): return self.account_type.create_data_source( - network=self.network.network(), use_secure_connection=self.network.use_secure_connection() + network=self.network.network(), + use_secure_connection=self.network.use_secure_connection(), + rate_limits=self.network.rate_limits(), ) diff --git a/setup.py b/setup.py index eaeb67fcd9..54c37abd41 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,9 @@ def main(): "eth-utils", "ethsnarks-loopring", "flake8", + "gql", + "grpcio", + "grpcio-tools" "hexbytes", "importlib-metadata", "injective-py" diff --git a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py index 013a412d32..b2f39b1f28 100644 --- a/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py +++ b/test/hummingbot/connector/exchange/injective_v2/data_sources/test_injective_data_source.py @@ -10,6 +10,7 @@ from pyinjective.constant import Network from pyinjective.wallet import Address, PrivateKey +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -42,6 +43,7 @@ def setUp(self, _) -> None: granter_address=Address(bytes.fromhex(granter_private_key.to_public_key().to_hex())).to_acc_bech32(), granter_subaccount_index=0, network=Network.testnet(node="sentry"), + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() @@ -383,6 +385,7 @@ def setUp(self, _) -> None: vault_subaccount_index=1, network=Network.testnet(node="sentry"), use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, ) self.query_executor = ProgrammableQueryExecutor() diff --git a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py similarity index 92% rename from test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py rename to test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py index 60b2e2b84e..6b8063b6e1 100644 --- a/test/hummingbot/connector/exchange/injective_v2/tests_injective_v2_utils.py +++ b/test/hummingbot/connector/exchange/injective_v2/test_injective_v2_utils.py @@ -4,6 +4,7 @@ from pyinjective.constant import Network import hummingbot.connector.exchange.injective_v2.injective_v2_utils as utils +from hummingbot.connector.exchange.injective_v2 import injective_constants as CONSTANTS from hummingbot.connector.exchange.injective_v2.data_sources.injective_grantee_data_source import ( InjectiveGranteeDataSource, ) @@ -113,7 +114,11 @@ def test_injective_delegate_account_config_creation(self): granter_subaccount_index=0, ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveGranteeDataSource, type(data_source)) @@ -127,7 +132,11 @@ def test_injective_vault_account_config_creation(self): bytes.fromhex(private_key.to_public_key().to_hex())).to_acc_bech32(), ) - data_source = config.create_data_source(network=Network.testnet(node="sentry"), use_secure_connection=True) + data_source = config.create_data_source( + network=Network.testnet(node="sentry"), + use_secure_connection=True, + rate_limits=CONSTANTS.PUBLIC_NODE_RATE_LIMITS, + ) self.assertEqual(InjectiveVaultsDataSource, type(data_source)) From 0b55470f7698508277943ee23101ffd16ff956f7 Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 29 Aug 2023 12:47:06 -0300 Subject: [PATCH 23/31] (fix) Solve issue in test_logger_mixin_for_tests.py that was causing random issues in other tests by not cleaning up the async loop correctly --- test/test_logger_mixin_for_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_logger_mixin_for_test.py b/test/test_logger_mixin_for_test.py index a94d81cfb5..6e076c28a3 100644 --- a/test/test_logger_mixin_for_test.py +++ b/test/test_logger_mixin_for_test.py @@ -8,7 +8,17 @@ class TestTestLoggerMixin(unittest.TestCase): def setUp(self): + super().setUp() self.logger = LoggerMixinForTest() + self._original_async_loop = asyncio.get_event_loop() + self.async_loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.async_loop) + + def tearDown(self) -> None: + super().tearDown() + self.async_loop.stop() + self.async_loop.close() + asyncio.set_event_loop(self._original_async_loop) def test_handle(self): self.logger.log_records = [] From 25e6eae088f6dfb61c228635cadf6df9ae580f3e Mon Sep 17 00:00:00 2001 From: abel Date: Wed, 30 Aug 2023 01:06:03 -0300 Subject: [PATCH 24/31] (fix) Changed logic in status polling loop in ExchangePyBase to ensure no error interrupts the update process --- .../injective_v2_perpetual_derivative.py | 1 + .../data_sources/injective_data_source.py | 3 +++ .../injective_v2/injective_v2_exchange.py | 1 + hummingbot/connector/exchange_py_base.py | 18 +++++++++++++----- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index abad110787..9556b72b8a 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -544,6 +544,7 @@ def _update_order_after_creation_success( new_state=order.current_state, misc_updates=misc_updates, ) + self.logger().debug(f"\nCreated order {order.client_order_id} ({exchange_order_id}) with TX {misc_updates}") self._order_tracker.process_order_update(order_update) def _on_order_creation_failure( diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index f56b213b74..0f86b37c69 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -452,6 +452,8 @@ async def create_orders( except asyncio.CancelledError: raise except Exception as ex: + self.logger().debug( + f"Error broadcasting transaction to create orders (message: {order_creation_messages})") results = self._place_order_results( orders_to_create=spot_orders + perpetual_orders, order_hashes=spot_order_hashes + derivative_order_hashes, @@ -522,6 +524,7 @@ async def cancel_orders( except asyncio.CancelledError: raise except Exception as ex: + self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") results.extend([ CancelOrderResult( client_order_id=order.client_order_id, diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index 537b512c35..dd80a2667d 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -487,6 +487,7 @@ def _update_order_after_creation_success( new_state=order.current_state, misc_updates=misc_updates, ) + self.logger().debug(f"\nCreated order {order.client_order_id} ({exchange_order_id}) with TX {misc_updates}") self._order_tracker.process_order_update(order_update) def _on_order_creation_failure( diff --git a/hummingbot/connector/exchange_py_base.py b/hummingbot/connector/exchange_py_base.py index 48404d0e46..d07ac722da 100644 --- a/hummingbot/connector/exchange_py_base.py +++ b/hummingbot/connector/exchange_py_base.py @@ -934,11 +934,19 @@ async def _status_polling_loop_fetch_updates(self): ) async def _update_all_balances(self): - await self._update_balances() - if not self.real_time_balance_update: - # This is only required for exchanges that do not provide balance update notifications through websocket - self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self.in_flight_orders.items()} - self._in_flight_orders_snapshot_timestamp = self.current_timestamp + try: + await self._update_balances() + if not self.real_time_balance_update: + # This is only required for exchanges that do not provide balance update notifications through websocket + self._in_flight_orders_snapshot = {k: copy.copy(v) for k, v in self.in_flight_orders.items()} + self._in_flight_orders_snapshot_timestamp = self.current_timestamp + except asyncio.CancelledError: + raise + except Exception as request_error: + self.logger().warning( + f"Failed to update balances. Error: {request_error}", + exc_info=request_error, + ) async def _update_orders_fills(self, orders: List[InFlightOrder]): for order in orders: From 7477f9eff292a989f3ac792ee5eb0b3931042a3d Mon Sep 17 00:00:00 2001 From: abel Date: Wed, 30 Aug 2023 13:07:09 -0300 Subject: [PATCH 25/31] (fix) Added logic to prevent the order creation and cancelation process from being canceled in the middle in Injective connectors --- .../injective_v2_perpetual_derivative.py | 10 ++++- .../data_sources/injective_data_source.py | 45 ++++++++++--------- .../injective_v2/injective_v2_exchange.py | 10 ++++- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py index 9556b72b8a..00f3798738 100644 --- a/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py +++ b/hummingbot/connector/derivative/injective_v2_perpetual/injective_v2_perpetual_derivative.py @@ -1012,7 +1012,10 @@ def _process_transaction_event(self, transaction_event: Dict[str, Any]): async def _check_orders_transactions(self): while True: try: - await self._check_orders_creation_transactions() + # Executing the process shielded from this async task to isolate it from network disconnections + # (network disconnections cancel this task) + task = asyncio.create_task(self._check_orders_creation_transactions()) + await asyncio.shield(task) await self._sleep(CONSTANTS.TRANSACTIONS_CHECK_INTERVAL) except NotImplementedError: raise @@ -1088,7 +1091,10 @@ async def _check_created_orders_status_for_transaction(self, transaction_hash: s async def _process_queued_orders(self): while True: try: - await self._cancel_and_create_queued_orders() + # Executing the batch cancelation and creation process shielded from this async task to isolate the + # creation/cancelation process from network disconnections (network disconnections cancel this task) + task = asyncio.create_task(self._cancel_and_create_queued_orders()) + await asyncio.shield(task) sleep_time = (self.clock.tick_size * 0.5 if self.clock is not None else self._orders_processing_delta_time) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py index 0f86b37c69..956eebb910 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_data_source.py @@ -503,35 +503,36 @@ async def cancel_orders( derivative_orders_data.append(order_data) orders_with_hash.append(order) - delegated_message = self._order_cancel_message( - spot_orders_to_cancel=spot_orders_data, - derivative_orders_to_cancel=derivative_orders_data, - ) + if len(orders_with_hash) > 0: + delegated_message = self._order_cancel_message( + spot_orders_to_cancel=spot_orders_data, + derivative_orders_to_cancel=derivative_orders_data, + ) - try: - result = await self._send_in_transaction(messages=[delegated_message]) - if result["rawLog"] != "[]": - raise ValueError(f"Error sending the order cancel transaction ({result['rawLog']})") - else: - cancel_transaction_hash = result.get("txhash", "") + try: + result = await self._send_in_transaction(messages=[delegated_message]) + if result["rawLog"] != "[]": + raise ValueError(f"Error sending the order cancel transaction ({result['rawLog']})") + else: + cancel_transaction_hash = result.get("txhash", "") + results.extend([ + CancelOrderResult( + client_order_id=order.client_order_id, + trading_pair=order.trading_pair, + misc_updates={"cancelation_transaction_hash": cancel_transaction_hash}, + ) for order in orders_with_hash + ]) + except asyncio.CancelledError: + raise + except Exception as ex: + self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") results.extend([ CancelOrderResult( client_order_id=order.client_order_id, trading_pair=order.trading_pair, - misc_updates={"cancelation_transaction_hash": cancel_transaction_hash}, + exception=ex, ) for order in orders_with_hash ]) - except asyncio.CancelledError: - raise - except Exception as ex: - self.logger().debug(f"Error broadcasting transaction to cancel orders (message: {delegated_message})") - results.extend([ - CancelOrderResult( - client_order_id=order.client_order_id, - trading_pair=order.trading_pair, - exception=ex, - ) for order in orders_with_hash - ]) return results diff --git a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py index dd80a2667d..125fe4caa8 100644 --- a/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py +++ b/hummingbot/connector/exchange/injective_v2/injective_v2_exchange.py @@ -898,7 +898,10 @@ def _process_transaction_event(self, transaction_event: Dict[str, Any]): async def _check_orders_transactions(self): while True: try: - await self._check_orders_creation_transactions() + # Executing the process shielded from this async task to isolate it from network disconnections + # (network disconnections cancel this task) + task = asyncio.create_task(self._check_orders_creation_transactions()) + await asyncio.shield(task) await self._sleep(CONSTANTS.TRANSACTIONS_CHECK_INTERVAL) except NotImplementedError: raise @@ -974,7 +977,10 @@ async def _check_created_orders_status_for_transaction(self, transaction_hash: s async def _process_queued_orders(self): while True: try: - await self._cancel_and_create_queued_orders() + # Executing the batch cancelation and creation process shielded from this async task to isolate the + # creation/cancelation process from network disconnections (network disconnections cancel this task) + task = asyncio.create_task(self._cancel_and_create_queued_orders()) + await asyncio.shield(task) sleep_time = (self.clock.tick_size * 0.5 if self.clock is not None else self._orders_processing_delta_time) From d0ad63274c0fbe009441cf652329d76fdeb32755 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 18:11:36 -0300 Subject: [PATCH 26/31] (fix) Changes in environment.yml to solve issues with grpc libraries binaries for Mac incorrectly compiles (for M1 and M2) --- .../exchange/bitfinex/bitfinex_exchange.pyx | 2 +- setup/environment.yml | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx index d349a911c8..1c945c296e 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx @@ -475,7 +475,7 @@ cdef class BitfinexExchange(ExchangeBase): http_method: str, url, headers, - data_str: Optional[str, list] = None) -> list: + data_str = None) -> list: """ A wrapper for submitting API requests to Bitfinex :returns: json data from the endpoints diff --git a/setup/environment.yml b/setup/environment.yml index c17d8e98a0..6b1310fad6 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -1,27 +1,28 @@ name: hummingbot channels: + - conda-forge - defaults dependencies: - bidict - - coverage=5.5 + - coverage + - cython=3.0 + - grpcio-tools - nomkl - nose=1.3.7 - nose-exclude - numpy=1.23.5 - numpy-base=1.23.5 - pandas=1.5.3 - - pip=23.1.2 + - pip - prompt_toolkit=3.0.20 - - pydantic=1.9.* + - pydantic=1.10 - pytest - - python=3.10.12 - - pytables=3.8.0 + - python=3.10 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 - - typing-extensions<4.6.0 - ujson - - zlib=1.2.13 + - zlib - pip: - aiohttp==3.* - aioprocessing==2.0 @@ -34,23 +35,19 @@ dependencies: - appnope==0.1.3 - base58==2.1.1 - cachetools==4.0.0 - - commlib-py==0.10.6 + - commlib-py==0.10 - cryptography==3.4.7 - - cython==3.0.0a10 - diff-cover - docker==5.0.3 - eip712-structs==1.1.0 - ethsnarks-loopring==0.1.5 - flake8==3.7.9 - gql - - grpcio - - grpcio-tools - importlib-metadata==0.23 - injective-py==0.7.* - mypy-extensions==0.4.3 - pandas_ta==0.3.14b - pre-commit==2.18.1 - - protobuf>=4 - psutil==5.7.2 - ptpython==3.0.20 - pyjwt==1.7.1 From 543779db97dc73312eab09efe6ae85f15895d534 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 18:35:21 -0300 Subject: [PATCH 27/31] (fix) Solve dependencies issues for Mac M1 and M2 --- setup/environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/environment.yml b/setup/environment.yml index 6b1310fad6..0abf03ce3e 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -5,7 +5,7 @@ channels: dependencies: - bidict - coverage - - cython=3.0 + - cython=3.0.0 - grpcio-tools - nomkl - nose=1.3.7 @@ -17,7 +17,7 @@ dependencies: - prompt_toolkit=3.0.20 - pydantic=1.10 - pytest - - python=3.10 + - python=3.10.12 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 From 486346eea9895ef851d512bc3c0c272e4390d365 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 31 Aug 2023 19:27:44 -0300 Subject: [PATCH 28/31] (fix) Solve errors tryin to upgrade cython version --- setup/environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/environment.yml b/setup/environment.yml index 0abf03ce3e..b6b775b8a4 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -5,7 +5,6 @@ channels: dependencies: - bidict - coverage - - cython=3.0.0 - grpcio-tools - nomkl - nose=1.3.7 @@ -17,7 +16,7 @@ dependencies: - prompt_toolkit=3.0.20 - pydantic=1.10 - pytest - - python=3.10.12 + - python=3.10 - scipy=1.10.1 - sqlalchemy=1.4 - tabulate==0.8.9 @@ -35,8 +34,9 @@ dependencies: - appnope==0.1.3 - base58==2.1.1 - cachetools==4.0.0 - - commlib-py==0.10 + - commlib-py==0.10.6 - cryptography==3.4.7 + - cython==3.0.0a10 - diff-cover - docker==5.0.3 - eip712-structs==1.1.0 From f4de33f8bd0656f8d940cb580e04fe0c15aac2ac Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 1 Sep 2023 00:14:59 -0300 Subject: [PATCH 29/31] (fix) Added logic in `install` script to reinstall grpcio with pip for Mac Intel because the conda grpcio binary for Mac Intel is broken --- install | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install b/install index 4a42eb6b4c..a49b9791c0 100755 --- a/install +++ b/install @@ -35,3 +35,11 @@ conda develop . pip install objgraph pre-commit install + +OS=`uname` +ARCH=`uname -m` + +if [[ "$OS" = "Darwin" && "$ARCH" = "x86_64" ]]; then + pip install grpcio --ignore-installed +fi + From db9d03c02d7a32ab7963d0cc7736990e30834a4d Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 1 Sep 2023 09:33:03 -0300 Subject: [PATCH 30/31] (fix) Added comment to explain the change in `install` script --- install | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install b/install index a49b9791c0..123662a956 100755 --- a/install +++ b/install @@ -36,6 +36,10 @@ pip install objgraph pre-commit install +# The following logic is required to replace the grpcio package installed from conda binaries in Mac Intel +# for binaries from Pypi. We need to do this because the conda binaries fro Mac Intel are broken. +# We can't use the Pypi binaries universally because they are broken for Mac ARM (M1 and M2). +# This logic can be removed once the grpcio conda binaries for Mac Intel are fixed OS=`uname` ARCH=`uname -m` From 00db7ba8fb26c083165bbaf5e4dd3c00d4df9d99 Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 1 Sep 2023 11:19:06 -0300 Subject: [PATCH 31/31] (fix) Applied fix to support malformed ticker names to the vaults and read only data sources --- .../injective_read_only_data_source.py | 16 ++++++++++++---- .../data_sources/injective_vaults_data_source.py | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py index 7c0850b0ad..6e23aa9cf7 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_read_only_data_source.py @@ -223,7 +223,11 @@ async def update_markets(self): for market_info in markets: try: - ticker_base, ticker_quote = market_info["ticker"].split("/") + if "/" in market_info["ticker"]: + ticker_base, ticker_quote = market_info["ticker"].split("/") + else: + ticker_base = market_info["ticker"] + ticker_quote = None base_token = self._token_from_market_info( denom=market_info["baseDenom"], token_meta=market_info["baseTokenMeta"], @@ -367,12 +371,14 @@ def _place_order_results( ) -> List[PlaceOrderResult]: raise NotImplementedError - def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candidate_symbol: str) -> InjectiveToken: + def _token_from_market_info( + self, denom: str, token_meta: Dict[str, Any], candidate_symbol: Optional[str] = None + ) -> InjectiveToken: token = self._tokens_map.get(denom) if token is None: unique_symbol = token_meta["symbol"] if unique_symbol in self._token_symbol_symbol_and_denom_map: - if candidate_symbol not in self._token_symbol_symbol_and_denom_map: + if candidate_symbol is not None and candidate_symbol not in self._token_symbol_symbol_and_denom_map: unique_symbol = candidate_symbol else: unique_symbol = token_meta["name"] @@ -389,7 +395,9 @@ def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candid return token def _parse_derivative_market_info(self, market_info: Dict[str, Any]) -> InjectiveDerivativeMarket: - _, ticker_quote = market_info["ticker"].split("/") + ticker_quote = None + if "/" in market_info["ticker"]: + _, ticker_quote = market_info["ticker"].split("/") quote_token = self._token_from_market_info( denom=market_info["quoteDenom"], token_meta=market_info["quoteTokenMeta"], diff --git a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py index 39cc1da564..d7ad51e264 100644 --- a/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py +++ b/hummingbot/connector/exchange/injective_v2/data_sources/injective_vaults_data_source.py @@ -269,7 +269,11 @@ async def update_markets(self): for market_info in markets: try: - ticker_base, ticker_quote = market_info["ticker"].split("/") + if "/" in market_info["ticker"]: + ticker_base, ticker_quote = market_info["ticker"].split("/") + else: + ticker_base = market_info["ticker"] + ticker_quote = None base_token = self._token_from_market_info( denom=market_info["baseDenom"], token_meta=market_info["baseTokenMeta"], @@ -399,12 +403,14 @@ def _sign_and_encode(self, transaction: Transaction) -> bytes: def _uses_default_portfolio_subaccount(self) -> bool: return self._vault_subaccount_index == CONSTANTS.DEFAULT_SUBACCOUNT_INDEX - def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candidate_symbol: str) -> InjectiveToken: + def _token_from_market_info( + self, denom: str, token_meta: Dict[str, Any], candidate_symbol: Optional[str] = None + ) -> InjectiveToken: token = self._tokens_map.get(denom) if token is None: unique_symbol = token_meta["symbol"] if unique_symbol in self._token_symbol_symbol_and_denom_map: - if candidate_symbol not in self._token_symbol_symbol_and_denom_map: + if candidate_symbol is not None and candidate_symbol not in self._token_symbol_symbol_and_denom_map: unique_symbol = candidate_symbol else: unique_symbol = token_meta["name"] @@ -421,7 +427,9 @@ def _token_from_market_info(self, denom: str, token_meta: Dict[str, Any], candid return token def _parse_derivative_market_info(self, market_info: Dict[str, Any]) -> InjectiveDerivativeMarket: - _, ticker_quote = market_info["ticker"].split("/") + ticker_quote = None + if "/" in market_info["ticker"]: + _, ticker_quote = market_info["ticker"].split("/") quote_token = self._token_from_market_info( denom=market_info["quoteDenom"], token_meta=market_info["quoteTokenMeta"],