Added YQ support #1
Conversation
| return params | ||
|
|
||
| def _compose_api_url(self, path: str) -> str: | ||
| return self.config.endpoint + path |
There was a problem hiding this comment.
да ну, специально его не взял. кучу ненужного делает
| return self.config.endpoint + path | ||
|
|
||
| def _compose_web_url(self, path: str) -> str: | ||
| return self.config.web_base_url + path |
| def wait_results(self, query_id: str) -> Any: | ||
| result_set_count = self.client.wait_query_to_succeed( | ||
| query_id, | ||
| execution_timeout=timedelta(minutes=30), |
There was a problem hiding this comment.
этот таймаут стоит или в аргументы конструктора вытащить или перестать задавать вообще
There was a problem hiding this comment.
окей, передвину в аргумент метода
| self.user_agent = user_agent | ||
|
|
||
| # urls should not contain trailing / | ||
| self.endpoint: str = "https://api.yandex-query.cloud.yandex.net" |
There was a problem hiding this comment.
стоило бы вынести в дефолтные аргументы конструктора
There was a problem hiding this comment.
это нужно для тестов, менять клиентам этот параметр надобности нет. только если появится YQ в KZ =))
| def close(self): | ||
| self.client.close() | ||
|
|
||
| def create_query(self, query_text: str|None, name: str|None=None, description: str | None = None, query_type: QueryType = QueryType.ANALYTICS) -> str: |
There was a problem hiding this comment.
я, кстати, вырезал тип streaming, так как подумал, что в airflow/jupyter стриминговые запросы явно лишние
| self.folder_id = folder_id | ||
| self.connection_id = connection_id | ||
| self.public_ssh_key = public_ssh_key | ||
| self.service_account_id = service_account_id |
There was a problem hiding this comment.
еще токен можно получить из-под текущего sa виртуальной машины, если он задан. удобно на виртуалках гонять. это http-запрос на адрес 169.xxx
| self.query_id: str | None | ||
|
|
||
| def execute(self, context: Context) -> Any: | ||
| self.hook = YQHook( |
There was a problem hiding this comment.
два execute подряд приведут к тому, что первый hook не будет остановлен
uzhastik
force-pushed
the
yq_support
branch
2 times, most recently
from
7472f4e to
69b5de1
Compare
|
This pull request has been automatically marked as stale because it has not had recent activity. It will be closed in 5 days if no further activity occurs. Thank you for your contributions. |
^ Add meaningful description above
Read the Pull Request Guidelines for more information.
In case of fundamental code changes, an Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in a newsfragment file, named
{pr_number}.significant.rstor{issue_number}.significant.rst, in newsfragments.