From 91565b063e1322776a09fda3634756a2951164b5 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 01/28] autogenerated docs workflow --- .github/workflows/docs.yml | 18 ++++++++++++++++++ docs/conf.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..97dc09d0 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,18 @@ +name: Deploy Sphinx documentation to Github Pages + +on: + push: + branches: [master, autogenetated_docs] # branch to trigger deployment + +jobs: + pages: + runs-on: ubuntu-20.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - id: deployment + uses: sphinx-notes/pages@v3 \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index f8cc5f6d..12812338 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'ydb' -copyright = '2021, yandex' +copyright = '2024, yandex' author = 'yandex' # The short X.Y version From 3e4036b7f6d4f8f31e5062d6fae38c0c617b559b Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 02/28] Add link to docs on README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index cfc57eb2..db7c3de2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ YDB Python SDK Officially supported Python client for YDB. +--- + +**Documentation**: https://ydb-platform.github.io/ydb-python-sdk + +--- + ## Quickstart ### Prerequisites From 617750e319093b426909a36002c51b906f039048 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 03/28] change docs theme --- docs/conf.py | 20 ++++++++++---------- docs/requirements.txt | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/conf.py b/docs/conf.py index 12812338..b48901a6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,11 +39,11 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.todo', - 'sphinx.ext.napoleon', - 'sphinx.ext.coverage', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinx.ext.napoleon', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. @@ -79,13 +79,13 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' html_theme_options = { - 'fixed_sidebar': True, - 'page_width': '1140px', - 'show_related': True, - 'show_powered_by': False + 'fixed_sidebar': True, + 'page_width': '1140px', + 'show_related': True, + 'show_powered_by': False } # Theme options are theme-specific and customize the look and feel of a theme diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..438b963c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +sphinx_rtd_theme==2.0.0 \ No newline at end of file From 6508d8829969cfc8365f33b16685b6cfbcffc263 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 04/28] try to add logo --- docs/_static/logo.svg | 4 ++++ docs/conf.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/_static/logo.svg diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 00000000..0a813215 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs/conf.py b/docs/conf.py index b48901a6..1e4f0a8b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,6 +88,8 @@ 'show_powered_by': False } +html_logo = '_static/logo.svg' + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -97,7 +99,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static'] +html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. From ecb2cd7cf9966df2a9c14c0848ca19292d04509c Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 05/28] add favicon --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 1e4f0a8b..0f6fc3fc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -89,6 +89,7 @@ } html_logo = '_static/logo.svg' +html_favicon = '_static/logo.svg' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 218562f844bc77ed13feeba0eefec1b3391535a6 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 06/28] overview doc added --- docs/index.rst | 7 +++---- docs/overview.rst | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 docs/overview.rst diff --git a/docs/index.rst b/docs/index.rst index 9a69b2bc..fd03336a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,18 +3,17 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to ydb's documentation! +YDB Python SDK =============================== .. toctree:: - :maxdepth: 10 + :maxdepth: 3 + overview ydb.rst examples.rst - - Indices and tables ================== diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 00000000..d6853e24 --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,50 @@ +Project Homepage +================ + +YDB Python SDK is hosted on GitHub at https://github.com/ydb-platform/ydb-python-sdk under the ydb-platform organization. + +Releases and project status are available on Pypi at https://pypi.org/project/ydb. + +The most recent published version of this documentation should be at https://ydb-platform.github.io/ydb-python-sdk. + +Installation +============ + +Prerequisites +------------- + +* Python 3.8 or higher; +* `pip` version 9.0.1 or higher; + +If necessary, upgrade your version of `pip`: + +:: + $ python -m pip install --upgrade pip + +If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv: + +:: + $ python -m pip install virtualenv + $ virtualenv venv + $ source venv/bin/activate + $ python -m pip install --upgrade pip + +Installation via Pypi +--------------------- + +To install YDB Python SDK through Pypi execute the following command: + +:: + $ pip install ydb + +Community +========= + +You can ask your questions in official Telegram chats: +**EN** `Official YDB chat `_. +**RU** `Official YDB chat `_. + + +Bugs and feature enhancements to YDB Python SDK should be reported on the `GitHub +issue tracker +`_. \ No newline at end of file From c9021029dfe972d5143352ee815a80f6d4a39cc4 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 07/28] format fixes --- docs/index.rst | 2 +- docs/overview.rst | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index fd03336a..5beb4e32 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ YDB Python SDK =============================== .. toctree:: - :maxdepth: 3 + :maxdepth: 4 overview ydb.rst diff --git a/docs/overview.rst b/docs/overview.rst index d6853e24..06227924 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -16,14 +16,12 @@ Prerequisites * Python 3.8 or higher; * `pip` version 9.0.1 or higher; -If necessary, upgrade your version of `pip`: +If necessary, upgrade your version of `pip`:: -:: $ python -m pip install --upgrade pip -If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv: +If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv:: -:: $ python -m pip install virtualenv $ virtualenv venv $ source venv/bin/activate @@ -32,17 +30,17 @@ If you cannot upgrade `pip` due to a system-owned installation, you can run the Installation via Pypi --------------------- -To install YDB Python SDK through Pypi execute the following command: - -:: +To install YDB Python SDK through Pypi execute the following command:: + $ pip install ydb Community ========= You can ask your questions in official Telegram chats: -**EN** `Official YDB chat `_. -**RU** `Official YDB chat `_. + +* **EN** `YDB chat `_. +* **RU** `YDB chat `_. Bugs and feature enhancements to YDB Python SDK should be reported on the `GitHub From a15c05f057066f7930745308fa988fb2656d4730 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 08/28] update overview doc --- docs/conf.py | 1 + docs/overview.rst | 18 +++++++++--------- docs/requirements.txt | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0f6fc3fc..761a37ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,6 +44,7 @@ 'sphinx.ext.todo', 'sphinx.ext.napoleon', 'sphinx.ext.coverage', + 'sphinx_copybutton', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/overview.rst b/docs/overview.rst index 06227924..aba9fc8f 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -14,25 +14,25 @@ Prerequisites ------------- * Python 3.8 or higher; -* `pip` version 9.0.1 or higher; +* ``pip`` version 9.0.1 or higher; -If necessary, upgrade your version of `pip`:: +If necessary, upgrade your version of ``pip``:: - $ python -m pip install --upgrade pip + python -m pip install --upgrade pip If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv:: - $ python -m pip install virtualenv - $ virtualenv venv - $ source venv/bin/activate - $ python -m pip install --upgrade pip + python -m pip install virtualenv + virtualenv venv + source venv/bin/activate + python -m pip install --upgrade pip Installation via Pypi --------------------- To install YDB Python SDK through Pypi execute the following command:: - - $ pip install ydb + + pip install ydb Community ========= diff --git a/docs/requirements.txt b/docs/requirements.txt index 438b963c..f8306480 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ -sphinx_rtd_theme==2.0.0 \ No newline at end of file +sphinx_rtd_theme==2.0.0 +sphinx-copybutton==0.5.2 From 1a7347a958ad86e7575f8258d479d8a5d284ffe2 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 09/28] restructure docs --- docs/{ydb.rst => apireference.rst} | 4 ++-- docs/index.rst | 6 +++--- docs/overview.rst | 13 ++++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) rename docs/{ydb.rst => apireference.rst} (97%) diff --git a/docs/ydb.rst b/docs/apireference.rst similarity index 97% rename from docs/ydb.rst rename to docs/apireference.rst index 53297005..0653d7bb 100644 --- a/docs/ydb.rst +++ b/docs/apireference.rst @@ -1,5 +1,5 @@ -ydb -=== +YDB API Reference +================= .. toctree:: :caption: Contents: diff --git a/docs/index.rst b/docs/index.rst index 5beb4e32..acd3ecf5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,11 +7,11 @@ YDB Python SDK =============================== .. toctree:: - :maxdepth: 4 + :maxdepth: 3 overview - ydb.rst - examples.rst + examples + apireference Indices and tables diff --git a/docs/overview.rst b/docs/overview.rst index aba9fc8f..56e2fa3f 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -1,5 +1,8 @@ +Overview +======== + Project Homepage -================ +---------------- YDB Python SDK is hosted on GitHub at https://github.com/ydb-platform/ydb-python-sdk under the ydb-platform organization. @@ -8,10 +11,10 @@ Releases and project status are available on Pypi at https://pypi.org/project/yd The most recent published version of this documentation should be at https://ydb-platform.github.io/ydb-python-sdk. Installation -============ +------------ Prerequisites -------------- +^^^^^^^^^^^^^ * Python 3.8 or higher; * ``pip`` version 9.0.1 or higher; @@ -28,14 +31,14 @@ If you cannot upgrade `pip` due to a system-owned installation, you can run the python -m pip install --upgrade pip Installation via Pypi ---------------------- +^^^^^^^^^^^^^^^^^^^^^ To install YDB Python SDK through Pypi execute the following command:: pip install ydb Community -========= +--------- You can ask your questions in official Telegram chats: From de09626dcd7544d3aa9ee5d6a803c9d3c7524c32 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 10/28] quick start docs --- docs/index.rst | 1 + docs/overview.rst | 26 ----------------------- docs/quickstart.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 26 deletions(-) create mode 100644 docs/quickstart.rst diff --git a/docs/index.rst b/docs/index.rst index acd3ecf5..184a35ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,6 +10,7 @@ YDB Python SDK :maxdepth: 3 overview + quickstart examples apireference diff --git a/docs/overview.rst b/docs/overview.rst index 56e2fa3f..4313b91c 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -10,32 +10,6 @@ Releases and project status are available on Pypi at https://pypi.org/project/yd The most recent published version of this documentation should be at https://ydb-platform.github.io/ydb-python-sdk. -Installation ------------- - -Prerequisites -^^^^^^^^^^^^^ - -* Python 3.8 or higher; -* ``pip`` version 9.0.1 or higher; - -If necessary, upgrade your version of ``pip``:: - - python -m pip install --upgrade pip - -If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv:: - - python -m pip install virtualenv - virtualenv venv - source venv/bin/activate - python -m pip install --upgrade pip - -Installation via Pypi -^^^^^^^^^^^^^^^^^^^^^ - -To install YDB Python SDK through Pypi execute the following command:: - - pip install ydb Community --------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 00000000..ab3bf04d --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,52 @@ +Quick Start +=========== + +Installation +------------ + +Prerequisites +^^^^^^^^^^^^^ + +* Python 3.8 or higher; +* ``pip`` version 9.0.1 or higher; + +If necessary, upgrade your version of ``pip``:: + + python -m pip install --upgrade pip + +If you cannot upgrade `pip` due to a system-owned installation, you can run the example in a virtualenv:: + + python -m pip install virtualenv + virtualenv venv + source venv/bin/activate + python -m pip install --upgrade pip + +Installation via Pypi +^^^^^^^^^^^^^^^^^^^^^ + +To install YDB Python SDK through Pypi execute the following command:: + + pip install ydb + +Usage +----- + +Import ydb package: + +.. code-block:: python + + import ydb + +Driver initialization: + +.. code-block:: python + + endpoint = "grpc://localhost:2136" # your ydb endpoint + database = "/local" # your ydb database + + with ydb.Driver( + endpoint=endpoint, + database=database, + credentials=ydb.credentials_from_env_variables(), + ) as driver: + driver.wait(timeout=5, fail_fast=True) From b9ff6a8da5a2f7d37ee3caaa91cc2cf955e9422b Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 11/28] temp --- docs/quickstart.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index ab3bf04d..f7b91a3a 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -50,3 +50,7 @@ Driver initialization: credentials=ydb.credentials_from_env_variables(), ) as driver: driver.wait(timeout=5, fail_fast=True) + + +>>> print("hello world") +hello world From 5f37bd04efe9a81c3f26642a2eab8762e09cae0d Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 12/28] usage docs --- docs/quickstart.rst | 69 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f7b91a3a..5bfc8b94 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -31,13 +31,15 @@ To install YDB Python SDK through Pypi execute the following command:: Usage ----- -Import ydb package: +Import Package +^^^^^^^^^^^^^^ .. code-block:: python import ydb -Driver initialization: +Driver Initialization +^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python @@ -51,6 +53,65 @@ Driver initialization: ) as driver: driver.wait(timeout=5, fail_fast=True) +SessionPool Initialization +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + with ydb.QuerySessionPool(driver) as pool: + pass + +Query Execution +^^^^^^^^^^^^^^^ + +Python SDK supports queries described by YQL syntax. +There are two primary methods for executing queries, each with different properties and use cases: + +* ``pool.execute_with_retries``: + * Buffers the entire result set in client memory. + * Automatically retries execution in case of retriable issues. + * Does not allow specifying a transaction execution mode. + * Recommended for one-off queries that are expected to produce small result sets. + +* ``tx.execute``: + * Returns an iterator over the query results, allowing processing of results that may not fit into client memory. + * Retries must be handled manually via `pool.retry_operation_sync`. + * Allows specifying a transaction execution mode. + * Recommended for scenarios where `pool.execute_with_retries` is insufficient. + + +Usage of ``pool.execute_with_retries()``: + +.. code-block:: python + + pool.execute_with_retries("DROP TABLE IF EXISTS example") + pool.execute_with_retries("CREATE TABLE example(key UInt64, value String, PRIMARY KEY (key))") + + pool.execute_with_retries("INSERT INTO example (key, value) VALUES (1, 'luffy')") + + res = pool.execute_with_retries("SELECT COUNT(*) AS rows_count FROM example") + +>>> res[0].rows_count +1 + +Example of ``tx.execute()``: + +.. code-block:: python + + def callee(session: ydb.QuerySessionSync): + with session.transaction() as tx: + with tx.execute( + "INSERT INTO example (key, value) VALUES (2, 'zoro')" + ): + pass + + with tx.execute( + "INSERT INTO example (key, value) VALUES (3, 'sanji')", + commit_tx=True, + ): + pass + + pool.retry_operation_sync(callee) + + ->>> print("hello world") -hello world From 91a14ab53399522f2d4c16b978aa0038cb66c80d Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 13/28] Update examples --- docs/examples.rst | 202 +++++++++++++++++++++++++++++++------------- docs/quickstart.rst | 2 + 2 files changed, 147 insertions(+), 57 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index 920d1292..4f8dee84 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -1,67 +1,155 @@ Examples =============== -ydb -^^^ +Basic example +^^^^^^^^^^^^^ + +All examples in this section are parts of `basic example `_. + +For deeper upderstanding it is better to read the whole example. Create table ------------ -:: - - ... create an instance of Driver ... - - description = ( - ydb.TableDescription() - .with_primary_keys('key1', 'key2') - .with_columns( - ydb.Column('key1', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ydb.Column('key2', ydb.OptionalType(ydb.PrimitiveType.Uint64)), - ydb.Column('value', ydb.OptionalType(ydb.PrimitiveType.Utf8)) - ) - .with_profile( - ydb.TableProfile() - .with_partitioning_policy( - ydb.PartitioningPolicy() - .with_explicit_partitions( - ydb.ExplicitPartitions( - ( - ydb.KeyBound((100, )), - ydb.KeyBound((300, 100)), - ydb.KeyBound((400, )), - ) - ) - ) - ) - ) - ) - - session = driver.table_client.session().create() - session.create_table('/my/table/', description) - - -Read table +.. code-block:: python + + def create_tables(pool: ydb.QuerySessionPool): + print("\nCreating table series...") + pool.execute_with_retries( + """ + CREATE table `series` ( + `series_id` Int64, + `title` Utf8, + `series_info` Utf8, + `release_date` Date, + PRIMARY KEY (`series_id`) + ) + """ + ) + + print("\nCreating table seasons...") + pool.execute_with_retries( + """ + CREATE table `seasons` ( + `series_id` Int64, + `season_id` Int64, + `title` Utf8, + `first_aired` Date, + `last_aired` Date, + PRIMARY KEY (`series_id`, `season_id`) + ) + """ + ) + + print("\nCreating table episodes...") + pool.execute_with_retries( + """ + CREATE table `episodes` ( + `series_id` Int64, + `season_id` Int64, + `episode_id` Int64, + `title` Utf8, + `air_date` Date, + PRIMARY KEY (`series_id`, `season_id`, `episode_id`) + ) + """ + ) + + +Upsert Simple +------------- + +.. code-block:: python + + def upsert_simple(pool: ydb.QuerySessionPool): + print("\nPerforming UPSERT into episodes...") + + pool.execute_with_retries( + """ + UPSERT INTO episodes (series_id, season_id, episode_id, title) VALUES (2, 6, 1, "TBD"); + """ + ) + + +Simple Select ---------- -:: - - .... initialize driver and session .... - - key_prefix_type = ydb.TupleType().add_element( - ydb.OptionalType(ydb.PrimitiveType.Uint64).add_element( - ydb.OptionalType(ydb.PrimitiveType.Utf8)) - async_table_iterator = session.read_table( - '/my/table', - columns=('KeyColumn0', 'KeyColumn1', 'ValueColumn'), - ydb.KeyRange( - ydb.KeyBound((100, 'hundred'), key_prefix_type) - ydb.KeyBound((200, 'two-hundreds'), key_prefix_type) - ) + +.. code-block:: python + + def select_simple(pool: ydb.QuerySessionPool): + print("\nCheck series table...") + result_sets = pool.execute_with_retries( + """ + SELECT + series_id, + title, + release_date + FROM series + WHERE series_id = 1; + """, + ) + first_set = result_sets[0] + for row in first_set.rows: + print( + "series, id: ", + row.series_id, + ", title: ", + row.title, + ", release date: ", + row.release_date, ) - while True: - try: - chunk_future = next(table_iterator) - chunk = chunk_future.result() # or any other way to await - ... additional data processing ... - except StopIteration: - break + return first_set + +Select With Parameters +---------------------- + +.. code-block:: python + + def select_with_parameters(pool: ydb.QuerySessionPool, series_id, season_id, episode_id): + result_sets = pool.execute_with_retries( + """ + DECLARE $seriesId AS Int64; + DECLARE $seasonId AS Int64; + DECLARE $episodeId AS Int64; + + SELECT + title, + air_date + FROM episodes + WHERE series_id = $seriesId AND season_id = $seasonId AND episode_id = $episodeId; + """, + { + "$seriesId": series_id, # could be defined implicit + "$seasonId": (season_id, ydb.PrimitiveType.Int64), # could be defined via tuple + "$episodeId": ydb.TypedValue(episode_id, ydb.PrimitiveType.Int64), # could be defined via special class + }, + ) + + print("\n> select_with_parameters:") + first_set = result_sets[0] + for row in first_set.rows: + print("episode title:", row.title, ", air date:", row.air_date) + + return first_set + +Huge Select +----------- + +.. code-block:: python + + def huge_select(pool: ydb.QuerySessionPool): + def callee(session: ydb.QuerySessionSync): + query = """SELECT * from episodes;""" + + with session.transaction().execute( + query, + commit_tx=True, + ) as result_sets: + print("\n> Huge SELECT call") + for result_set in result_sets: + for row in result_set.rows: + print("episode title:", row.title, ", air date:", row.air_date) + + return pool.retry_operation_sync(callee) + diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5bfc8b94..cb1d1062 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -68,12 +68,14 @@ Python SDK supports queries described by YQL syntax. There are two primary methods for executing queries, each with different properties and use cases: * ``pool.execute_with_retries``: + * Buffers the entire result set in client memory. * Automatically retries execution in case of retriable issues. * Does not allow specifying a transaction execution mode. * Recommended for one-off queries that are expected to produce small result sets. * ``tx.execute``: + * Returns an iterator over the query results, allowing processing of results that may not fit into client memory. * Retries must be handled manually via `pool.retry_operation_sync`. * Allows specifying a transaction execution mode. From 4709becae3fc14f3f97b76cb83fdf951f313ad83 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:35:41 +0300 Subject: [PATCH 14/28] add query service to api reference --- docs/apireference.rst | 69 ++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 0653d7bb..5b78525e 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -33,8 +33,38 @@ DriverConfig ------------------------ -Table -^^^^^ +Query Service +^^^^^^^^^^^^^ + +QuerySessionPool +~~~~~~~~~~~~~~~~ + +.. autoclass:: QuerySessionPool + :members: + :inherited-members: + :undoc-members: + +QuerySessionSync +~~~~~~~~~~~~~~~~ + +.. autoclass:: QuerySessionSync + :members: + :inherited-members: + :undoc-members: + +QueryTxContextSync +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: QueryTxContextSync + :members: + :inherited-members: + :undoc-members: + +------------------------ + +Table Service +^^^^^^^^^^^^^ + TableClient ~~~~~~~~~~~ .. autoclass:: TableClient @@ -50,6 +80,14 @@ TableClientSettings :inherited-members: :undoc-members: +Session Pool +~~~~~~~~~~~~ + +.. autoclass:: SessionPool + :members: + :inherited-members: + :undoc-members: + Session ~~~~~~~ @@ -66,6 +104,14 @@ Transaction Context :inherited-members: :undoc-members: +DataQuery +^^^^^^^^^ + +.. autoclass:: DataQuery + :members: + :inherited-members: + :undoc-members: + -------------------------- Scheme @@ -81,17 +127,6 @@ SchemeClient ------------------ -Session Pool -^^^^^^^^^^^^ - -.. autoclass:: SessionPool - :members: - :inherited-members: - :undoc-members: - ------------------------------ - - Result Sets ^^^^^^^^^^^ @@ -101,11 +136,3 @@ Result Sets :undoc-members: ----------------------------- - -DataQuery -^^^^^^^^^ - -.. autoclass:: DataQuery - :members: - :inherited-members: - :undoc-members: From 53874aa6e694b27c4337c41d7e4d96d87534b345 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:38:53 +0300 Subject: [PATCH 15/28] update autogenerated docs with aio reference --- docs/apireference.rst | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 5b78525e..7d0cf489 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -22,6 +22,15 @@ Driver object :undoc-members: +Driver object (AsyncIO) +~~~~~~~~~~~~~ + +.. autoclass:: ydb.aio.Driver + :members: + :inherited-members: + :undoc-members: + + DriverConfig ~~~~~~~~~~~~ @@ -44,18 +53,18 @@ QuerySessionPool :inherited-members: :undoc-members: -QuerySessionSync +QuerySession ~~~~~~~~~~~~~~~~ -.. autoclass:: QuerySessionSync +.. autoclass:: QuerySession :members: :inherited-members: :undoc-members: -QueryTxContextSync +QueryTxContext ~~~~~~~~~~~~~~~~~~ -.. autoclass:: QueryTxContextSync +.. autoclass:: QueryTxContext :members: :inherited-members: :undoc-members: From f601499ea96381808a15032ef4f0d1c711a2153c Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 13:50:03 +0300 Subject: [PATCH 16/28] extent public api with QueryTxContext --- docs/apireference.rst | 24 ++++++++++++------------ ydb/aio/query/__init__.py | 2 ++ ydb/query/__init__.py | 2 ++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 7d0cf489..118a03b9 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -16,7 +16,7 @@ Driver Driver object ~~~~~~~~~~~~~ -.. autoclass:: Driver +.. autoclass:: ydb.Driver :members: :inherited-members: :undoc-members: @@ -34,7 +34,7 @@ Driver object (AsyncIO) DriverConfig ~~~~~~~~~~~~ -.. autoclass:: DriverConfig +.. autoclass:: ydb.DriverConfig :members: :inherited-members: :undoc-members: @@ -48,7 +48,7 @@ Query Service QuerySessionPool ~~~~~~~~~~~~~~~~ -.. autoclass:: QuerySessionPool +.. autoclass:: ydb.QuerySessionPool :members: :inherited-members: :undoc-members: @@ -56,7 +56,7 @@ QuerySessionPool QuerySession ~~~~~~~~~~~~~~~~ -.. autoclass:: QuerySession +.. autoclass:: ydb.QuerySession :members: :inherited-members: :undoc-members: @@ -64,7 +64,7 @@ QuerySession QueryTxContext ~~~~~~~~~~~~~~~~~~ -.. autoclass:: QueryTxContext +.. autoclass:: ydb.QueryTxContext :members: :inherited-members: :undoc-members: @@ -76,7 +76,7 @@ Table Service TableClient ~~~~~~~~~~~ -.. autoclass:: TableClient +.. autoclass:: ydb.TableClient :members: :inherited-members: :undoc-members: @@ -84,7 +84,7 @@ TableClient TableClientSettings ~~~~~~~~~~~~~~~~~~~ -.. autoclass:: TableClientSettings +.. autoclass:: ydb.TableClientSettings :members: :inherited-members: :undoc-members: @@ -92,7 +92,7 @@ TableClientSettings Session Pool ~~~~~~~~~~~~ -.. autoclass:: SessionPool +.. autoclass:: ydb.SessionPool :members: :inherited-members: :undoc-members: @@ -100,7 +100,7 @@ Session Pool Session ~~~~~~~ -.. autoclass:: Session +.. autoclass:: ydb.Session :members: :inherited-members: :undoc-members: @@ -108,7 +108,7 @@ Session Transaction Context ~~~~~~~~~~~~~~~~~~~ -.. autoclass:: TxContext +.. autoclass:: ydb.TxContext :members: :inherited-members: :undoc-members: @@ -116,7 +116,7 @@ Transaction Context DataQuery ^^^^^^^^^ -.. autoclass:: DataQuery +.. autoclass:: ydb.DataQuery :members: :inherited-members: :undoc-members: @@ -129,7 +129,7 @@ Scheme SchemeClient ~~~~~~~~~~~~ -.. autoclass:: SchemeClient +.. autoclass:: ydb.SchemeClient :members: :inherited-members: :undoc-members: diff --git a/ydb/aio/query/__init__.py b/ydb/aio/query/__init__.py index ea5273d7..8e7dd4fd 100644 --- a/ydb/aio/query/__init__.py +++ b/ydb/aio/query/__init__.py @@ -1,7 +1,9 @@ __all__ = [ "QuerySessionPool", "QuerySession", + "QueryTxContext", ] from .pool import QuerySessionPool from .session import QuerySession +from .transaction import QueryTxContext diff --git a/ydb/query/__init__.py b/ydb/query/__init__.py index 1e950bb7..7431332e 100644 --- a/ydb/query/__init__.py +++ b/ydb/query/__init__.py @@ -6,6 +6,7 @@ "QuerySessionPool", "QueryClientSync", "QuerySession", + "QueryTxContext", ] import logging @@ -15,6 +16,7 @@ ) from .session import QuerySession +from .transaction import QueryTxContext from .._grpc.grpcwrapper import common_utils from .._grpc.grpcwrapper.ydb_query_public_types import ( From 1850b73112afc03ca83c165e7ed80ae618cc0fa3 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 15:58:22 +0300 Subject: [PATCH 17/28] change doc structure --- docs/apireference.rst | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 118a03b9..5e3ce93f 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -7,11 +7,8 @@ YDB API Reference .. module:: ydb -Module contents ---------------- - Driver -^^^^^^ +------ Driver object ~~~~~~~~~~~~~ @@ -43,7 +40,7 @@ DriverConfig ------------------------ Query Service -^^^^^^^^^^^^^ +------------- QuerySessionPool ~~~~~~~~~~~~~~~~ @@ -72,7 +69,7 @@ QueryTxContext ------------------------ Table Service -^^^^^^^^^^^^^ +------------- TableClient ~~~~~~~~~~~ @@ -114,7 +111,7 @@ Transaction Context :undoc-members: DataQuery -^^^^^^^^^ +--------- .. autoclass:: ydb.DataQuery :members: @@ -124,7 +121,7 @@ DataQuery -------------------------- Scheme -^^^^^^ +------ SchemeClient ~~~~~~~~~~~~ @@ -137,7 +134,7 @@ SchemeClient ------------------ Result Sets -^^^^^^^^^^^ +----------- .. autoclass:: ydb.convert._ResultSet :members: From e3d499e0d00c586cff898e2940d24319d99d137d Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:17:21 +0300 Subject: [PATCH 18/28] try to combine sync and async api ref --- docs/apireference.rst | 47 +++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 5e3ce93f..82960198 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -10,16 +10,20 @@ YDB API Reference Driver ------ -Driver object -~~~~~~~~~~~~~ - .. autoclass:: ydb.Driver :members: :inherited-members: :undoc-members: -Driver object (AsyncIO) +.. autoclass:: ydb.DriverConfig + :members: + :inherited-members: + :undoc-members: + :exclude-members: database, ca_cert, channel_options, secure_channel, endpoint, endpoints, credentials, use_all_nodes, root_certificates, certificate_chain, private_key, grpc_keep_alive_timeout, table_client_settings, primary_user_agent + + +AsyncIO ~~~~~~~~~~~~~ .. autoclass:: ydb.aio.Driver @@ -28,44 +32,57 @@ Driver object (AsyncIO) :undoc-members: -DriverConfig -~~~~~~~~~~~~ +------------------------ -.. autoclass:: ydb.DriverConfig +Common +------------- + +.. autoclass:: ydb.BaseRequestSettings :members: :inherited-members: :undoc-members: - :exclude-members: database, ca_cert, channel_options, secure_channel, endpoint, endpoints, credentials, use_all_nodes, root_certificates, certificate_chain, private_key, grpc_keep_alive_timeout, table_client_settings, primary_user_agent ------------------------ Query Service ------------- -QuerySessionPool -~~~~~~~~~~~~~~~~ - .. autoclass:: ydb.QuerySessionPool :members: :inherited-members: :undoc-members: -QuerySession -~~~~~~~~~~~~~~~~ .. autoclass:: ydb.QuerySession :members: :inherited-members: :undoc-members: -QueryTxContext -~~~~~~~~~~~~~~~~~~ .. autoclass:: ydb.QueryTxContext :members: :inherited-members: :undoc-members: + +AsyncIO +~~~~~~~ + +.. autoclass:: ydb.aio.QuerySessionPool + :members: + :inherited-members: + :undoc-members: + +.. autoclass:: ydb.aio.QuerySession + :members: + :inherited-members: + :undoc-members: + +.. autoclass:: ydb.aio.QueryTxContext + :members: + :inherited-members: + :undoc-members: + ------------------------ Table Service From c56fbce33ab762343cd54dd06a52827a8841d9ef Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:26:09 +0300 Subject: [PATCH 19/28] Update BaseRequestSettings type hints --- ydb/settings.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ydb/settings.py b/ydb/settings.py index 6739a46f..019b75a8 100644 --- a/ydb/settings.py +++ b/ydb/settings.py @@ -39,7 +39,7 @@ def make_copy(self): .with_need_rpc_auth(self.need_rpc_auth) ) - def with_compression(self, compression): + def with_compression(self, compression) -> "BaseRequestSettings": """ Enables compression for the specific RPC :param compression: An RPCCompression enum value. @@ -48,11 +48,11 @@ def with_compression(self, compression): self.compression = compression return self - def with_need_rpc_auth(self, need_rpc_auth): + def with_need_rpc_auth(self, need_rpc_auth) -> "BaseRequestSettings": self.need_rpc_auth = need_rpc_auth return self - def with_header(self, key, value): + def with_header(self, key, value) -> "BaseRequestSettings": """ Adds a key-value pair to the request headers. :param key: A string with a header key. @@ -62,7 +62,7 @@ def with_header(self, key, value): self.headers.append((key, value)) return self - def with_trace_id(self, trace_id): + def with_trace_id(self, trace_id) -> "BaseRequestSettings": """ Includes trace id for RPC headers :param trace_id: A trace id string @@ -71,7 +71,7 @@ def with_trace_id(self, trace_id): self.trace_id = trace_id return self - def with_request_type(self, request_type): + def with_request_type(self, request_type) -> "BaseRequestSettings": """ Includes request type for RPC headers :param request_type: A request type string @@ -80,7 +80,7 @@ def with_request_type(self, request_type): self.request_type = request_type return self - def with_operation_timeout(self, timeout): + def with_operation_timeout(self, timeout) -> "BaseRequestSettings": """ Indicates that client is no longer interested in the result of operation after the specified duration starting from the time operation arrives at the server. @@ -89,12 +89,12 @@ def with_operation_timeout(self, timeout): Timeout of operation does not tell anything about its result, it might be completed successfully or cancelled on server. :param timeout: - :return: + :return: The self instance """ self.operation_timeout = timeout return self - def with_cancel_after(self, timeout): + def with_cancel_after(self, timeout) -> "BaseRequestSettings": """ Server will try to cancel the operation after the specified duration starting from the time the operation arrives at server. @@ -102,12 +102,12 @@ def with_cancel_after(self, timeout): sent back to client if it was waiting for the operation result. In case when cancellation isn't possible, no action will be performed. :param timeout: - :return: + :return: The self instance """ self.cancel_after = timeout return self - def with_timeout(self, timeout): + def with_timeout(self, timeout) -> "BaseRequestSettings": """ Client-side timeout to complete request. Since YDB doesn't support request cancellation at this moment, this feature should be From 52b9587f395af3faa34fd864c9ee87cfdfd0d22f Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:30:43 +0300 Subject: [PATCH 20/28] restructure docs --- docs/apireference.rst | 69 ++++++++++++++++++++++++++++++++----------- ydb/aio/__init__.py | 2 +- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 82960198..39960526 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -10,48 +10,80 @@ YDB API Reference Driver ------ -.. autoclass:: ydb.Driver +DriverConfig +^^^^^^^^^^^^ + +.. autoclass:: ydb.DriverConfig :members: :inherited-members: :undoc-members: + :exclude-members: database, ca_cert, channel_options, secure_channel, endpoint, endpoints, credentials, use_all_nodes, root_certificates, certificate_chain, private_key, grpc_keep_alive_timeout, table_client_settings, primary_user_agent -.. autoclass:: ydb.DriverConfig +Driver +^^^^^^ + +.. autoclass:: ydb.Driver :members: :inherited-members: :undoc-members: - :exclude-members: database, ca_cert, channel_options, secure_channel, endpoint, endpoints, credentials, use_all_nodes, root_certificates, certificate_chain, private_key, grpc_keep_alive_timeout, table_client_settings, primary_user_agent -AsyncIO -~~~~~~~~~~~~~ +Driver (AsyncIO) +^^^^^^^^^^^^^^^ .. autoclass:: ydb.aio.Driver :members: :inherited-members: :undoc-members: - ------------------------ Common ------------- +BaseRequestSettings +^^^^^^^^^^^^^^^^^^^ + .. autoclass:: ydb.BaseRequestSettings :members: :inherited-members: :undoc-members: + +Result Sets +^^^^^^^^^^^ + +.. autoclass:: ydb.convert._ResultSet + :members: + :inherited-members: + :undoc-members: + + ------------------------ Query Service ------------- +QueryClientSettings +^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: ydb.QueryClientSettings + :members: + :inherited-members: + :undoc-members: + + +QuerySessionPool +^^^^^^^^^^^^^^^^ + .. autoclass:: ydb.QuerySessionPool :members: :inherited-members: :undoc-members: +QuerySession +^^^^^^^^^^^^ .. autoclass:: ydb.QuerySession :members: @@ -59,30 +91,42 @@ Query Service :undoc-members: +QueryTxContext +^^^^^^^^^^^^^^ + .. autoclass:: ydb.QueryTxContext :members: :inherited-members: :undoc-members: -AsyncIO -~~~~~~~ +QuerySessionPool (AsyncIO) +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: ydb.aio.QuerySessionPool :members: :inherited-members: :undoc-members: + +QuerySession (AsyncIO) +^^^^^^^^^^^^^^^^^^^^^^ + .. autoclass:: ydb.aio.QuerySession :members: :inherited-members: :undoc-members: + +QueryTxContext (AsyncIO) +^^^^^^^^^^^^^^^^^^^^^^^^ + .. autoclass:: ydb.aio.QueryTxContext :members: :inherited-members: :undoc-members: + ------------------------ Table Service @@ -150,12 +194,3 @@ SchemeClient ------------------ -Result Sets ------------ - -.. autoclass:: ydb.convert._ResultSet - :members: - :inherited-members: - :undoc-members: - ------------------------------ diff --git a/ydb/aio/__init__.py b/ydb/aio/__init__.py index a755713d..1c9c887c 100644 --- a/ydb/aio/__init__.py +++ b/ydb/aio/__init__.py @@ -1,3 +1,3 @@ from .driver import Driver # noqa from .table import SessionPool, retry_operation # noqa -from .query import QuerySessionPool, QuerySession # noqa +from .query import QuerySessionPool, QuerySession, QueryTxContext # noqa From e0507bfdb30dca4872810c4a0b8111f98fe85ba0 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:39:35 +0300 Subject: [PATCH 21/28] reformat headers in docs --- docs/apireference.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 39960526..4832e103 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -133,14 +133,14 @@ Table Service ------------- TableClient -~~~~~~~~~~~ +^^^^^^^^^^^ .. autoclass:: ydb.TableClient :members: :inherited-members: :undoc-members: TableClientSettings -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ .. autoclass:: ydb.TableClientSettings :members: @@ -148,7 +148,7 @@ TableClientSettings :undoc-members: Session Pool -~~~~~~~~~~~~ +^^^^^^^^^^^^ .. autoclass:: ydb.SessionPool :members: @@ -156,7 +156,7 @@ Session Pool :undoc-members: Session -~~~~~~~ +^^^^^^^ .. autoclass:: ydb.Session :members: @@ -164,7 +164,7 @@ Session :undoc-members: Transaction Context -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ .. autoclass:: ydb.TxContext :members: @@ -172,7 +172,7 @@ Transaction Context :undoc-members: DataQuery ---------- +^^^^^^^^^ .. autoclass:: ydb.DataQuery :members: @@ -185,7 +185,7 @@ Scheme ------ SchemeClient -~~~~~~~~~~~~ +^^^^^^^^^^^^ .. autoclass:: ydb.SchemeClient :members: From 3f817460e4990a89ff3244c35b163ab1b324bbbd Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:41:19 +0300 Subject: [PATCH 22/28] exclude slots from BaseRequestSettings docs --- docs/apireference.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/apireference.rst b/docs/apireference.rst index 4832e103..a20f7de3 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -49,6 +49,7 @@ BaseRequestSettings :members: :inherited-members: :undoc-members: + :exclude-members: trace_id, request_type, timeout, cancel_after, operation_timeout, compression, need_rpc_auth, headers, make_copy, tracer Result Sets From cdae01eac4c8da1fae3a45bf1095ba2488c752d2 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 17:49:37 +0300 Subject: [PATCH 23/28] Add query tx modes to docs --- docs/apireference.rst | 38 +++++++++++++++++++ .../grpcwrapper/ydb_query_public_types.py | 22 +++++++++++ ydb/query/__init__.py | 4 +- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index a20f7de3..f4f10cbd 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -128,6 +128,44 @@ QueryTxContext (AsyncIO) :undoc-members: +Query Tx Mode +^^^^^^^^^^^^^ + +.. autoclass:: ydb.BaseQueryTxMode + :members: + :inherited-members: + :undoc-members: + :exclude-members: name, to_proto + + +.. autoclass:: ydb.QueryOnlineReadOnly + :members: + :inherited-members: + :undoc-members: + :exclude-members: name, to_proto + + +.. autoclass:: ydb.QuerySerializableReadWrite + :members: + :inherited-members: + :undoc-members: + :exclude-members: name, to_proto + + +.. autoclass:: ydb.QuerySnapshotReadOnly + :members: + :inherited-members: + :undoc-members: + :exclude-members: name, to_proto + + +.. autoclass:: ydb.QueryStaleReadOnly + :members: + :inherited-members: + :undoc-members: + :exclude-members: name, to_proto + + ------------------------ Table Service diff --git a/ydb/_grpc/grpcwrapper/ydb_query_public_types.py b/ydb/_grpc/grpcwrapper/ydb_query_public_types.py index d79a2967..23c3c4f9 100644 --- a/ydb/_grpc/grpcwrapper/ydb_query_public_types.py +++ b/ydb/_grpc/grpcwrapper/ydb_query_public_types.py @@ -11,6 +11,7 @@ class BaseQueryTxMode(IToProto): + """Abstract class for Query Transaction Modes.""" @property @abc.abstractmethod def name(self) -> str: @@ -18,6 +19,10 @@ def name(self) -> str: class QuerySnapshotReadOnly(BaseQueryTxMode): + """All the read operations within a transaction access the database snapshot. + All the data reads are consistent. The snapshot is taken when the transaction begins, + meaning the transaction sees all changes committed before it began. + """ def __init__(self): self._name = "snapshot_read_only" @@ -30,6 +35,9 @@ def to_proto(self) -> ydb_query_pb2.SnapshotModeSettings: class QuerySerializableReadWrite(BaseQueryTxMode): + """This mode guarantees that the result of successful parallel transactions is equivalent + to their serial execution, and there are no read anomalies for successful transactions. + """ def __init__(self): self._name = "serializable_read_write" @@ -42,6 +50,15 @@ def to_proto(self) -> ydb_query_pb2.SerializableModeSettings: class QueryOnlineReadOnly(BaseQueryTxMode): + """Each read operation in the transaction is reading the data that is most recent at execution time. + The consistency of retrieved data depends on the allow_inconsistent_reads setting: + * false (consistent reads): Each individual read operation returns consistent data, + but no consistency is guaranteed between reads. + Reading the same table range twice may return different results. + * true (inconsistent reads): Even the data fetched by a particular + read operation may contain inconsistent results. + """ + def __init__(self, allow_inconsistent_reads: bool = False): self.allow_inconsistent_reads = allow_inconsistent_reads self._name = "online_read_only" @@ -55,6 +72,11 @@ def to_proto(self) -> ydb_query_pb2.OnlineModeSettings: class QueryStaleReadOnly(BaseQueryTxMode): + """Read operations within a transaction may return results that are slightly out-of-date + (lagging by fractions of a second). Each individual read returns consistent data, + but no consistency between different reads is guaranteed. + """ + def __init__(self): self._name = "stale_read_only" diff --git a/ydb/query/__init__.py b/ydb/query/__init__.py index 7431332e..0f818789 100644 --- a/ydb/query/__init__.py +++ b/ydb/query/__init__.py @@ -1,10 +1,11 @@ __all__ = [ + "BaseQueryTxMode", "QueryOnlineReadOnly", "QuerySerializableReadWrite", "QuerySnapshotReadOnly", "QueryStaleReadOnly", "QuerySessionPool", - "QueryClientSync", + "QueryClientSettings", "QuerySession", "QueryTxContext", ] @@ -20,6 +21,7 @@ from .._grpc.grpcwrapper import common_utils from .._grpc.grpcwrapper.ydb_query_public_types import ( + BaseQueryTxMode, QueryOnlineReadOnly, QuerySerializableReadWrite, QuerySnapshotReadOnly, From 1dbbd3fa7c6d0c1d0f7201678d854c5897a40e42 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 18:07:16 +0300 Subject: [PATCH 24/28] add retrysettings to docs --- docs/apireference.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/apireference.rst b/docs/apireference.rst index f4f10cbd..52cf78a4 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -52,6 +52,14 @@ BaseRequestSettings :exclude-members: trace_id, request_type, timeout, cancel_after, operation_timeout, compression, need_rpc_auth, headers, make_copy, tracer +RetrySettings +^^^^^^^^^^^^^ + +.. autoclass:: ydb.RetrySettings + :members: + :inherited-members: + :undoc-members: + Result Sets ^^^^^^^^^^^ From 0abe50bc5459697cd95f0ad1ee5f50f71fca8d15 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 18:09:37 +0300 Subject: [PATCH 25/28] small fixes --- docs/apireference.rst | 1 + docs/overview.rst | 6 +----- ydb/_grpc/grpcwrapper/ydb_query_public_types.py | 3 +++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 52cf78a4..1d6a2821 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -60,6 +60,7 @@ RetrySettings :inherited-members: :undoc-members: + Result Sets ^^^^^^^^^^^ diff --git a/docs/overview.rst b/docs/overview.rst index 4313b91c..81abf94d 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -14,11 +14,7 @@ The most recent published version of this documentation should be at https://ydb Community --------- -You can ask your questions in official Telegram chats: - -* **EN** `YDB chat `_. -* **RU** `YDB chat `_. - +You can ask your questions in official Telegram chats: `EN `_ | `RU `_. Bugs and feature enhancements to YDB Python SDK should be reported on the `GitHub issue tracker diff --git a/ydb/_grpc/grpcwrapper/ydb_query_public_types.py b/ydb/_grpc/grpcwrapper/ydb_query_public_types.py index 23c3c4f9..9888a677 100644 --- a/ydb/_grpc/grpcwrapper/ydb_query_public_types.py +++ b/ydb/_grpc/grpcwrapper/ydb_query_public_types.py @@ -12,6 +12,7 @@ class BaseQueryTxMode(IToProto): """Abstract class for Query Transaction Modes.""" + @property @abc.abstractmethod def name(self) -> str: @@ -23,6 +24,7 @@ class QuerySnapshotReadOnly(BaseQueryTxMode): All the data reads are consistent. The snapshot is taken when the transaction begins, meaning the transaction sees all changes committed before it began. """ + def __init__(self): self._name = "snapshot_read_only" @@ -38,6 +40,7 @@ class QuerySerializableReadWrite(BaseQueryTxMode): """This mode guarantees that the result of successful parallel transactions is equivalent to their serial execution, and there are no read anomalies for successful transactions. """ + def __init__(self): self._name = "serializable_read_write" From 52c6fece4a5db228d777269071963c12d4ceb320 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 10 Sep 2024 18:22:32 +0300 Subject: [PATCH 26/28] remove source button from docs --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 761a37ca..13d7e22d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -92,6 +92,8 @@ html_logo = '_static/logo.svg' html_favicon = '_static/logo.svg' +html_show_sourcelink = False + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. From 6a6b2234e3484d08dd7e6dd9f3cbbbdc4eafec8a Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 11 Sep 2024 10:51:23 +0300 Subject: [PATCH 27/28] refactor docstrings --- ydb/aio/query/pool.py | 17 ++++++++++++++++- ydb/aio/query/session.py | 1 + ydb/aio/query/transaction.py | 1 + ydb/query/pool.py | 24 ++++++++++++++++++++++-- ydb/query/session.py | 2 ++ ydb/query/transaction.py | 1 + 6 files changed, 43 insertions(+), 3 deletions(-) diff --git a/ydb/aio/query/pool.py b/ydb/aio/query/pool.py index f0b962c3..e8d53438 100644 --- a/ydb/aio/query/pool.py +++ b/ydb/aio/query/pool.py @@ -44,6 +44,13 @@ async def _create_new_session(self): return session async def acquire(self) -> QuerySession: + """WARNING: This API is experimental and could be changed. + + Acquire a session from Session Pool. + + :return A QuerySession object. + """ + if self._should_stop.is_set(): logger.error("An attempt to take session from closed session pool.") raise RuntimeError("An attempt to take session from closed session pool.") @@ -79,12 +86,18 @@ async def acquire(self) -> QuerySession: return session async def release(self, session: QuerySession) -> None: + """WARNING: This API is experimental and could be changed. + + Release a session back to Session Pool. + """ + self._queue.put_nowait(session) logger.debug("Session returned to queue: %s", session._state.session_id) def checkout(self) -> "SimpleQuerySessionCheckoutAsync": """WARNING: This API is experimental and could be changed. - Return a Session context manager, that opens session on enter and closes session on exit. + + Return a Session context manager, that acquires session on enter and releases session on exit. """ return SimpleQuerySessionCheckoutAsync(self) @@ -93,6 +106,7 @@ async def retry_operation_async( self, callee: Callable, retry_settings: Optional[RetrySettings] = None, *args, **kwargs ): """WARNING: This API is experimental and could be changed. + Special interface to execute a bunch of commands with session in a safe, retriable way. :param callee: A function, that works with session. @@ -118,6 +132,7 @@ async def execute_with_retries( **kwargs, ) -> List[convert.ResultSet]: """WARNING: This API is experimental and could be changed. + Special interface to execute a one-shot queries in a safe, retriable way. Note: this method loads all data from stream before return, do not use this method with huge read queries. diff --git a/ydb/aio/query/session.py b/ydb/aio/query/session.py index 5f51b671..4c1c1a10 100644 --- a/ydb/aio/query/session.py +++ b/ydb/aio/query/session.py @@ -116,6 +116,7 @@ async def execute( """WARNING: This API is experimental and could be changed. Sends a query to Query Service + :param query: (YQL or SQL text) to be executed. :param syntax: Syntax of the query, which is a one from the following choises: 1) QuerySyntax.YQL_V1, which is default; diff --git a/ydb/aio/query/transaction.py b/ydb/aio/query/transaction.py index 0e3ab602..b115a4b4 100644 --- a/ydb/aio/query/transaction.py +++ b/ydb/aio/query/transaction.py @@ -114,6 +114,7 @@ async def execute( """WARNING: This API is experimental and could be changed. Sends a query to Query Service + :param query: (YQL or SQL text) to be executed. :param parameters: dict with parameters and YDB types; :param commit_tx: A special flag that allows transaction commit. diff --git a/ydb/query/pool.py b/ydb/query/pool.py index 1ee9ea83..839d8688 100644 --- a/ydb/query/pool.py +++ b/ydb/query/pool.py @@ -29,7 +29,8 @@ class QuerySessionPool: def __init__(self, driver: common_utils.SupportedDriverType, size: int = 100): """ - :param driver: A driver instance + :param driver: A driver instance. + :param size: Max size of Session Pool. """ logger.warning("QuerySessionPool is an experimental API, which could be changed.") @@ -47,6 +48,15 @@ def _create_new_session(self, timeout: Optional[float]): return session def acquire(self, timeout: Optional[float] = None) -> QuerySession: + """WARNING: This API is experimental and could be changed. + + Acquire a session from Session Pool. + + :param timeout: A timeout to wait in seconds. + + :return A QuerySession object. + """ + start = time.monotonic() lock_acquire_timeout = timeout if timeout is not None else -1 @@ -92,18 +102,27 @@ def acquire(self, timeout: Optional[float] = None) -> QuerySession: self._lock.release() def release(self, session: QuerySession) -> None: + """WARNING: This API is experimental and could be changed. + + Release a session back to Session Pool. + """ + self._queue.put_nowait(session) logger.debug("Session returned to queue: %s", session._state.session_id) def checkout(self, timeout: Optional[float] = None) -> "SimpleQuerySessionCheckout": """WARNING: This API is experimental and could be changed. - Return a Session context manager, that opens session on enter and closes session on exit. + + Return a Session context manager, that acquires session on enter and releases session on exit. + + :param timeout: A timeout to wait in seconds. """ return SimpleQuerySessionCheckout(self, timeout) def retry_operation_sync(self, callee: Callable, retry_settings: Optional[RetrySettings] = None, *args, **kwargs): """WARNING: This API is experimental and could be changed. + Special interface to execute a bunch of commands with session in a safe, retriable way. :param callee: A function, that works with session. @@ -129,6 +148,7 @@ def execute_with_retries( **kwargs, ) -> List[convert.ResultSet]: """WARNING: This API is experimental and could be changed. + Special interface to execute a one-shot queries in a safe, retriable way. Note: this method loads all data from stream before return, do not use this method with huge read queries. diff --git a/ydb/query/session.py b/ydb/query/session.py index 66e86501..5b4db26c 100644 --- a/ydb/query/session.py +++ b/ydb/query/session.py @@ -269,6 +269,7 @@ def transaction(self, tx_mode: Optional[base.BaseQueryTxMode] = None) -> QueryTx """WARNING: This API is experimental and could be changed. Creates a transaction context manager with specified transaction mode. + :param tx_mode: Transaction mode, which is a one from the following choises: 1) QuerySerializableReadWrite() which is default mode; 2) QueryOnlineReadOnly(allow_inconsistent_reads=False); @@ -301,6 +302,7 @@ def execute( """WARNING: This API is experimental and could be changed. Sends a query to Query Service + :param query: (YQL or SQL text) to be executed. :param syntax: Syntax of the query, which is a one from the following choises: 1) QuerySyntax.YQL_V1, which is default; diff --git a/ydb/query/transaction.py b/ydb/query/transaction.py index 9ad3552f..21ba0279 100644 --- a/ydb/query/transaction.py +++ b/ydb/query/transaction.py @@ -394,6 +394,7 @@ def execute( """WARNING: This API is experimental and could be changed. Sends a query to Query Service + :param query: (YQL or SQL text) to be executed. :param parameters: dict with parameters and YDB types; :param commit_tx: A special flag that allows transaction commit. From 610697ff6d2b15425358df1a9809fed69c97106f Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 11 Sep 2024 15:42:28 +0300 Subject: [PATCH 28/28] change target branch to main for docs --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 97dc09d0..07b7645f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,7 +2,7 @@ name: Deploy Sphinx documentation to Github Pages on: push: - branches: [master, autogenetated_docs] # branch to trigger deployment + branches: [main] # branch to trigger deployment jobs: pages: