From 3627e384bcd3c8d43f6546814407df8aed263379 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 20:04:37 +0100 Subject: [PATCH 01/51] Chore: turbocharge gitignore --- .gitignore | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.gitignore b/.gitignore index 60e5afe..87e166d 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,59 @@ cover/ # tox .tox/ + +# misc +*.7z +*.apk +*.backup +*.bak +*.bk +*.bz2 +*.deb +*.doc +*.docx +*.gz +*.gzip +*.img +*.iso +*.jar +*.jpeg +*.jpg +*.log +*.ods +*.part +*.pdf +*.pkg +*.png +*.pps +*.ppsx +*.ppt +*.pptx +*.ps +*.pyc +*.rar +*.swp +*.sys +*.tar +*.tgz +*.tmp +*.xls +*.xlsx +*.xz +*.zip +**/*venv/ +.cache +.coverage* +.idea/ +.isort.cfg +**.directory +venv +.pytest_cache/ +.vscode/ +*.egg-info/ +.tox/ +.cargo/ +.expected +.hypothesis/ +.mypy_cache/ +**/__pycache__/ From 7e78582e747b612f0f3b2d0b0e6a94d02be4516e Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 22:28:50 +0100 Subject: [PATCH 02/51] Tooling: use poetry --- .gitignore | 3 +++ pyproject.toml | 42 ++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 18 ++++++++++++++++++ setup.in | 50 -------------------------------------------------- tox.ini | 12 ------------ 5 files changed, 63 insertions(+), 62 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg delete mode 100644 setup.in delete mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 87e166d..7856f1a 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,6 @@ venv .hypothesis/ .mypy_cache/ **/__pycache__/ + +# poetry +poetry.lock diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b90b156 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 + +[tool.poetry] +name = "clickhouse_orm" +version = "0.2.1" +description = "A simple ORM for working with the Clickhouse database" +authors = ["olliemath "] +license = "BSD" +homepage = "https://github.com/SuadeLabs/clickhouse_orm" +repository = "https://github.com/SuadeLabs/clickhouse_orm" +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Database" +] + +[tool.poetry.dependencies] +python = "^3.7" + +[tool.poetry.dev-dependencies] +flake8 = "^3.9.2" +flake8-bugbear = "^21.4.3" +pep8-naming = "^0.12.0" +pytest = "^6.2.4" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..0317882 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,18 @@ +[flake8] +max-line-length = 120 +select = + # pycodestyle + E, W + # pyflakes + F + # flake8-bugbear + B, B9 + # pydocstyle + D + # isort + I +ignore = + E203 # Whitespace after ':' + W503 # Operator after new line + B950 # We use E501 + B008 # Using callable in function defintion, required for FastAPI diff --git a/setup.in b/setup.in deleted file mode 100644 index e62522f..0000000 --- a/setup.in +++ /dev/null @@ -1,50 +0,0 @@ - -SETUP_INFO = dict( - name = '${project:name}', - version = '${infi.recipe.template.version:version}', - author = '${infi.recipe.template.version:author}', - author_email = '${infi.recipe.template.version:author_email}', - - url = ${infi.recipe.template.version:homepage}, - license = 'BSD', - description = """${project:description}""", - - # http://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers = [ - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.4", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Database" - ], - - install_requires = ${project:install_requires}, - namespace_packages = ${project:namespace_packages}, - - package_dir = {'': 'src'}, - package_data = {'': ${project:package_data}}, - include_package_data = True, - zip_safe = False, - - entry_points = dict( - console_scripts = ${project:console_scripts}, - gui_scripts = ${project:gui_scripts}, - ), -) - -if SETUP_INFO['url'] is None: - _ = SETUP_INFO.pop('url') - -def setup(): - from setuptools import setup as _setup - from setuptools import find_packages - SETUP_INFO['packages'] = find_packages('src') - _setup(**SETUP_INFO) - -if __name__ == '__main__': - setup() - diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 81299c8..0000000 --- a/tox.ini +++ /dev/null @@ -1,12 +0,0 @@ -[tox] -envlist = py27, py35, pypy - -[testenv] -deps = - nose - flake8 - -commands = - {envpython} -m compileall -q src/ tests/ - # {envbindir}/flake8 src/ tests/ --max-line-length=120 - nosetests -v {posargs} From e3953bded72807e2b8f6099081993b1082655871 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 22:33:11 +0100 Subject: [PATCH 03/51] Chore: remove intermediate dirs --- .noseids | 1073 ----------------- README.md | 2 +- buildout.cfg | 68 -- clickhouse_orm/__init__.py | 13 + .../database.py | 4 +- .../engines.py | 2 +- .../fields.py | 0 .../funcs.py | 0 .../migrations.py | 2 +- .../models.py | 2 +- .../query.py | 2 +- .../system_models.py | 0 .../utils.py | 2 +- docs/class_reference.md | 12 +- docs/contributing.md | 8 +- docs/expressions.md | 2 +- docs/field_options.md | 6 +- docs/field_types.md | 2 +- docs/importing_orm_classes.md | 32 +- docs/index.md | 6 +- docs/models_and_databases.md | 6 +- docs/ref.md | 10 +- docs/schema_migrations.md | 4 +- docs/system_models.md | 4 +- docs/toc.md | 12 +- docs/whats_new_in_version_2.md | 4 +- examples/cpu_usage/collect.py | 2 +- examples/cpu_usage/models.py | 2 +- examples/cpu_usage/requirements.txt | 2 +- examples/cpu_usage/results.py | 2 +- examples/db_explorer/requirements.txt | 2 +- examples/db_explorer/server.py | 2 +- examples/full_text_search/load.py | 2 +- examples/full_text_search/models.py | 2 +- examples/full_text_search/requirements.txt | 2 +- examples/full_text_search/search.py | 2 +- scripts/generate_ref.py | 14 +- scripts/test_python3.sh | 4 +- src/infi/__init__.py | 1 - src/infi/clickhouse_orm/__init__.py | 13 - tests/base_test_with_data.py | 8 +- tests/sample_migrations/0001_initial.py | 4 +- tests/sample_migrations/0002.py | 4 +- tests/sample_migrations/0003.py | 4 +- tests/sample_migrations/0004.py | 4 +- tests/sample_migrations/0005.py | 4 +- tests/sample_migrations/0006.py | 4 +- tests/sample_migrations/0007.py | 4 +- tests/sample_migrations/0008.py | 4 +- tests/sample_migrations/0009.py | 4 +- tests/sample_migrations/0010.py | 2 +- tests/sample_migrations/0011.py | 2 +- tests/sample_migrations/0012.py | 2 +- tests/sample_migrations/0013.py | 4 +- tests/sample_migrations/0014.py | 2 +- tests/sample_migrations/0015.py | 2 +- tests/sample_migrations/0016.py | 2 +- tests/sample_migrations/0017.py | 2 +- tests/sample_migrations/0018.py | 2 +- tests/sample_migrations/0019.py | 2 +- tests/test_alias_fields.py | 10 +- tests/test_array_fields.py | 10 +- tests/test_buffer.py | 4 +- tests/test_compressed_fields.py | 12 +- tests/test_constraints.py | 2 +- tests/test_custom_fields.py | 8 +- tests/test_database.py | 14 +- tests/test_datetime_fields.py | 8 +- tests/test_decimal_fields.py | 8 +- tests/test_dictionaries.py | 2 +- tests/test_engines.py | 2 +- tests/test_enum_fields.py | 8 +- tests/test_fixed_string_fields.py | 8 +- tests/test_funcs.py | 6 +- tests/test_indexes.py | 2 +- tests/test_inheritance.py | 8 +- tests/test_ip_fields.py | 8 +- tests/test_join.py | 2 +- tests/test_materialized_fields.py | 10 +- tests/test_migrations.py | 10 +- tests/test_models.py | 8 +- tests/test_mutations.py | 2 +- tests/test_nullable_fields.py | 10 +- tests/test_querysets.py | 6 +- tests/test_readonly.py | 2 +- tests/test_server_errors.py | 2 +- tests/test_simple_fields.py | 2 +- tests/test_system_models.py | 10 +- tests/test_uuid_fields.py | 8 +- 89 files changed, 223 insertions(+), 1365 deletions(-) delete mode 100644 .noseids delete mode 100644 buildout.cfg create mode 100644 clickhouse_orm/__init__.py rename {src/infi/clickhouse_orm => clickhouse_orm}/database.py (99%) rename {src/infi/clickhouse_orm => clickhouse_orm}/engines.py (99%) rename {src/infi/clickhouse_orm => clickhouse_orm}/fields.py (100%) rename {src/infi/clickhouse_orm => clickhouse_orm}/funcs.py (100%) rename {src/infi/clickhouse_orm => clickhouse_orm}/migrations.py (99%) rename {src/infi/clickhouse_orm => clickhouse_orm}/models.py (99%) rename {src/infi/clickhouse_orm => clickhouse_orm}/query.py (99%) rename {src/infi/clickhouse_orm => clickhouse_orm}/system_models.py (100%) rename {src/infi/clickhouse_orm => clickhouse_orm}/utils.py (98%) delete mode 100644 src/infi/__init__.py delete mode 100644 src/infi/clickhouse_orm/__init__.py diff --git a/.noseids b/.noseids deleted file mode 100644 index 5023878..0000000 --- a/.noseids +++ /dev/null @@ -1,1073 +0,0 @@ -(dp1 -S'failed' -p2 -(lp3 -S'84' -p4 -aS'87' -p5 -aS'88' -p6 -aS'89' -p7 -aS'122' -p8 -aS'132' -p9 -aS'136' -p10 -aS'137' -p11 -aS'138' -p12 -aS'139' -p13 -aS'143' -p14 -aS'144' -p15 -aS'145' -p16 -aS'146' -p17 -aS'147' -p18 -aS'148' -p19 -aS'149' -p20 -aS'150' -p21 -aS'152' -p22 -aS'153' -p23 -aS'156' -p24 -aS'157' -p25 -aS'158' -p26 -aS'159' -p27 -aS'162' -p28 -aS'163' -p29 -aS'164' -p30 -aS'165' -p31 -aS'166' -p32 -aS'168' -p33 -aS'191' -p34 -asS'source_names' -p35 -(lp36 -S'.' -asS'ids' -p37 -(dp38 -I1 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_alias_fields.py' -S'tests.test_alias_fields' -p39 -S'MaterializedFieldsTest.test_assignment_error' -tp40 -sI2 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_alias_fields.py' -g39 -S'MaterializedFieldsTest.test_duplicate_default' -tp41 -sI3 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_alias_fields.py' -g39 -S'MaterializedFieldsTest.test_insert_and_select' -tp42 -sI4 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_alias_fields.py' -g39 -S'MaterializedFieldsTest.test_wrong_field' -tp43 -sI5 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_array_fields.py' -S'tests.test_array_fields' -p44 -S'ArrayFieldsTest.test_assignment_error' -tp45 -sI6 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_array_fields.py' -g44 -S'ArrayFieldsTest.test_conversion' -tp46 -sI7 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_array_fields.py' -g44 -S'ArrayFieldsTest.test_insert_and_select' -tp47 -sI8 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_array_fields.py' -g44 -S'ArrayFieldsTest.test_invalid_inner_field' -tp48 -sI9 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_array_fields.py' -g44 -S'ArrayFieldsTest.test_parse_array' -tp49 -sI10 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_buffer.py' -S'tests.test_buffer' -p50 -S'BufferTestCase.test_insert_buffer' -tp51 -sI11 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_compressed_fields.py' -S'tests.test_compressed_fields' -p52 -S'CompressedFieldsTestCase.test_assignment' -tp53 -sI12 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_compressed_fields.py' -g52 -S'CompressedFieldsTestCase.test_confirm_compression_codec' -tp54 -sI13 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_compressed_fields.py' -g52 -S'CompressedFieldsTestCase.test_defaults' -tp55 -sI14 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_compressed_fields.py' -g52 -S'CompressedFieldsTestCase.test_string_conversion' -tp56 -sI15 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_compressed_fields.py' -g52 -S'CompressedFieldsTestCase.test_to_dict' -tp57 -sI16 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_custom_fields.py' -S'tests.test_custom_fields' -p58 -S'CustomFieldsTest.test_boolean_field' -tp59 -sI17 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -S'tests.test_database' -p60 -S'DatabaseTestCase.test_add_setting' -tp61 -sI18 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_count' -tp62 -sI19 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_does_table_exist' -tp63 -sI20 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_dollar_in_select' -tp64 -sI21 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__empty' -tp65 -sI22 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__generator' -tp66 -sI23 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__iterator' -tp67 -sI24 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__list' -tp68 -sI25 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__medium_batches' -tp69 -sI26 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_insert__small_batches' -tp70 -sI27 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_invalid_user' -tp71 -sI28 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_missing_engine' -tp72 -sI29 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_nonexisting_db' -tp73 -sI30 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_pagination' -tp74 -sI31 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_pagination_empty_page' -tp75 -sI32 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_pagination_invalid_page' -tp76 -sI33 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_pagination_last_page' -tp77 -sI34 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_pagination_with_conditions' -tp78 -sI35 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_potentially_problematic_field_names' -tp79 -sI36 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_preexisting_db' -tp80 -sI37 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_raw' -tp81 -sI38 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_select' -tp82 -sI39 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_select_ad_hoc_model' -tp83 -sI40 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_select_partial_fields' -tp84 -sI41 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_select_with_totals' -tp85 -sI42 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_database.py' -g60 -S'DatabaseTestCase.test_special_chars' -tp86 -sI43 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_datetime_fields.py' -S'tests.test_datetime_fields' -p87 -S'DateFieldsTest.test_ad_hoc_model' -tp88 -sI44 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -S'tests.test_decimal_fields' -p89 -S'DecimalFieldsTest.test_ad_hoc_model' -tp90 -sI45 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_aggregation' -tp91 -sI46 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_assignment_error' -tp92 -sI47 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_assignment_ok' -tp93 -sI48 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_insert_and_select' -tp94 -sI49 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_min_max' -tp95 -sI50 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_precision_and_scale' -tp96 -sI51 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_decimal_fields.py' -g89 -S'DecimalFieldsTest.test_rounding' -tp97 -sI52 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -S'tests.test_engines' -p98 -S'DistributedTestCase.test_bad_cluster_name' -tp99 -sI53 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_insert_distributed_select_local' -tp100 -sI54 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_insert_local_select_distributed' -tp101 -sI55 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_minimal_engine' -tp102 -sI56 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_minimal_engine_no_superclasses' -tp103 -sI57 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_minimal_engine_two_superclasses' -tp104 -sI58 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_verbose_engine_two_superclasses' -tp105 -sI59 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_with_table_name' -tp106 -sI60 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'DistributedTestCase.test_without_table_name' -tp107 -sI61 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_collapsing_merge_tree' -tp108 -sI62 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_custom_partitioning' -tp109 -sI63 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_log' -tp110 -sI64 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_memory' -tp111 -sI65 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_merge' -tp112 -sI66 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_merge_tree' -tp113 -sI67 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_merge_tree_with_granularity' -tp114 -sI68 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_merge_tree_with_sampling' -tp115 -sI69 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_replacing_merge_tree' -tp116 -sI70 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_replicated_merge_tree' -tp117 -sI71 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_replicated_merge_tree_incomplete' -tp118 -sI72 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_summing_merge_tree' -tp119 -sI73 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_engines.py' -g98 -S'EnginesTestCase.test_tiny_log' -tp120 -sI74 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -S'tests.test_enum_fields' -p121 -S'EnumFieldsTest.test_ad_hoc_model' -tp122 -sI75 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -g121 -S'EnumFieldsTest.test_assignment_error' -tp123 -sI76 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -g121 -S'EnumFieldsTest.test_conversion' -tp124 -sI77 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -g121 -S'EnumFieldsTest.test_default_value' -tp125 -sI78 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -g121 -S'EnumFieldsTest.test_enum_array' -tp126 -sI79 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_enum_fields.py' -g121 -S'EnumFieldsTest.test_insert_and_select' -tp127 -sI80 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_fixed_string_fields.py' -S'tests.test_fixed_string_fields' -p128 -S'FixedStringFieldsTest.test_ad_hoc_model' -tp129 -sI81 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_fixed_string_fields.py' -g128 -S'FixedStringFieldsTest.test_assignment_error' -tp130 -sI82 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_fixed_string_fields.py' -g128 -S'FixedStringFieldsTest.test_insert_and_select' -tp131 -sI83 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -S'tests.test_funcs' -p132 -S'FuncsTestCase.test_arithmetic_operators' -tp133 -sI84 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_base64_functions' -tp134 -sI85 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_comparison_operators' -tp135 -sI86 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_date_functions' -tp136 -sI87 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_filter_date_field' -tp137 -sI88 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_filter_float_field' -tp138 -sI89 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_func_as_field_value' -tp139 -sI90 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_func_to_sql' -tp140 -sI91 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_logical_operators' -tp141 -sI92 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_string_functions' -tp142 -sI93 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_funcs.py' -g132 -S'FuncsTestCase.test_type_conversion_functions' -tp143 -sI94 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_inheritance.py' -S'tests.test_inheritance' -p144 -S'InheritanceTestCase.test_create_table_sql' -tp145 -sI95 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_inheritance.py' -g144 -S'InheritanceTestCase.test_field_inheritance' -tp146 -sI96 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_inheritance.py' -g144 -S'InheritanceTestCase.test_get_field' -tp147 -sI97 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_join.py' -S'tests.test_join' -p148 -S'JoinTest.test_with_db_name' -tp149 -sI98 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_join.py' -g148 -S'JoinTest.test_with_subquery' -tp150 -sI99 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_join.py' -g148 -S'JoinTest.test_without_db_name' -tp151 -sI100 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_materialized_fields.py' -S'tests.test_materialized_fields' -p152 -S'MaterializedFieldsTest.test_assignment_error' -tp153 -sI101 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_materialized_fields.py' -g152 -S'MaterializedFieldsTest.test_duplicate_default' -tp154 -sI102 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_materialized_fields.py' -g152 -S'MaterializedFieldsTest.test_insert_and_select' -tp155 -sI103 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_materialized_fields.py' -g152 -S'MaterializedFieldsTest.test_wrong_field' -tp156 -sI104 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_migrations.py' -S'tests.test_migrations' -p157 -S'MigrationsTestCase.test_migrations' -tp158 -sI105 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -S'tests.test_models' -p159 -S'ModelTestCase.test_assignment' -tp160 -sI106 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_assignment_error' -tp161 -sI107 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_defaults' -tp162 -sI108 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_field_name_in_error_message_for_invalid_value_in_assignment' -tp163 -sI109 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_field_name_in_error_message_for_invalid_value_in_constructor' -tp164 -sI110 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_string_conversion' -tp165 -sI111 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_models.py' -g159 -S'ModelTestCase.test_to_dict' -tp166 -sI112 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -S'tests.test_nullable_fields' -p167 -S'NullableFieldsTest.test_ad_hoc_model' -tp168 -sI113 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -g167 -S'NullableFieldsTest.test_insert_and_select' -tp169 -sI114 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -g167 -S'NullableFieldsTest.test_isinstance' -tp170 -sI115 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -g167 -S'NullableFieldsTest.test_nullable_datetime_field' -tp171 -sI116 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -g167 -S'NullableFieldsTest.test_nullable_string_field' -tp172 -sI117 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_nullable_fields.py' -g167 -S'NullableFieldsTest.test_nullable_uint8_field' -tp173 -sI118 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -S'tests.test_querysets' -p174 -S'AggregateTestCase.test_aggregate_no_grouping' -tp175 -sI119 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_on_aggregate' -tp176 -sI120 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_distinct' -tp177 -sI121 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_explicit_grouping' -tp178 -sI122 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_filter' -tp179 -sI123 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_implicit_grouping' -tp180 -sI124 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_indexing' -tp181 -sI125 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_no_calculated_fields' -tp182 -sI126 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_only' -tp183 -sI127 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_order_by' -tp184 -sI128 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_pagination' -tp185 -sI129 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_slicing' -tp186 -sI130 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_totals' -tp187 -sI131 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_aggregate_with_wrong_grouping' -tp188 -sI132 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_double_underscore_field' -tp189 -sI133 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'AggregateTestCase.test_filter_on_calculated_field' -tp190 -sI134 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_arithmetic_operators' -tp191 -sI135 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_comparison_operators' -tp192 -sI136 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_filter_date_field' -tp193 -sI137 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_filter_float_field' -tp194 -sI138 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_func_as_field_value' -tp195 -sI139 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'FuncsTestCase.test_func_to_sql' -tp196 -sI140 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_alias_field' -tp197 -sI141 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_count_of_slice' -tp198 -sI142 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_distinct' -tp199 -sI143 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_date_field' -tp200 -sI144 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_enum_field' -tp201 -sI145 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_float_field' -tp202 -sI146 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_int_field' -tp203 -sI147 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_null_value' -tp204 -sI148 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_string_field' -tp205 -sI149 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_unicode_string' -tp206 -sI150 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_filter_with_q_objects' -tp207 -sI151 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_final' -tp208 -sI152 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_in_subquery' -tp209 -sI153 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_invalid_filter' -tp210 -sI154 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_invalid_slicing' -tp211 -sI155 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_materialized_field' -tp212 -sI156 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_mixed_filter' -tp213 -sI157 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_multiple_exclude' -tp214 -sI158 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_mutiple_filter' -tp215 -sI159 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_no_filtering' -tp216 -sI160 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_only' -tp217 -sI161 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_order_by' -tp218 -sI162 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_pagination' -tp219 -sI163 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_pagination_invalid_page' -tp220 -sI164 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_pagination_last_page' -tp221 -sI165 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_pagination_with_conditions' -tp222 -sI166 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_prewhere' -tp223 -sI167 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_slicing' -tp224 -sI168 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_querysets.py' -g174 -S'QuerySetTestCase.test_truthiness' -tp225 -sI169 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -S'tests.test_readonly' -p226 -S'ReadonlyTestCase.test_create_readonly_table' -tp227 -sI170 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -g226 -S'ReadonlyTestCase.test_drop_readonly_table' -tp228 -sI171 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -g226 -S'ReadonlyTestCase.test_insert_readonly' -tp229 -sI172 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -g226 -S'ReadonlyTestCase.test_nonexisting_readonly_database' -tp230 -sI173 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -g226 -S'ReadonlyTestCase.test_readonly_db_with_default_user' -tp231 -sI174 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_readonly.py' -g226 -S'ReadonlyTestCase.test_readonly_db_with_readonly_user' -tp232 -sI175 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_server_errors.py' -S'tests.test_server_errors' -p233 -S'ServerErrorTest.test_new_format' -tp234 -sI176 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_server_errors.py' -g233 -S'ServerErrorTest.test_old_format' -tp235 -sI177 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_simple_fields.py' -S'tests.test_simple_fields' -p236 -S'SimpleFieldsTest.test_date_field' -tp237 -sI178 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_simple_fields.py' -g236 -S'SimpleFieldsTest.test_date_field_timezone' -tp238 -sI179 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_simple_fields.py' -g236 -S'SimpleFieldsTest.test_datetime_field' -tp239 -sI180 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_simple_fields.py' -g236 -S'SimpleFieldsTest.test_datetime_field_timezone' -tp240 -sI181 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_simple_fields.py' -g236 -S'SimpleFieldsTest.test_uint8_field' -tp241 -sI182 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -S'tests.test_system_models' -p242 -S'SystemPartTest.test_attach_detach' -tp243 -sI183 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_drop' -tp244 -sI184 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_fetch' -tp245 -sI185 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_freeze' -tp246 -sI186 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_get_active' -tp247 -sI187 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_get_all' -tp248 -sI188 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_get_conditions' -tp249 -sI189 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_is_read_only' -tp250 -sI190 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_is_system_model' -tp251 -sI191 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemPartTest.test_query' -tp252 -sI192 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemTest.test_create_readonly_table' -tp253 -sI193 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemTest.test_drop_readonly_table' -tp254 -sI194 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_system_models.py' -g242 -S'SystemTest.test_insert_system' -tp255 -sI195 -(S'/home/itais/workspace/infinidat/infi.clickhouse_orm/tests/test_uuid_fields.py' -S'tests.test_uuid_fields' -p256 -S'UUIDFieldsTest.test_uuid_field' -tp257 -ss. \ No newline at end of file diff --git a/README.md b/README.md index 462c8f4..247d928 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Let's jump right in with a simple example of monitoring CPU usage. First we need connect to the database and create a table for the model: ```python -from infi.clickhouse_orm import Database, Model, DateTimeField, UInt16Field, Float32Field, Memory, F +from clickhouse_orm import Database, Model, DateTimeField, UInt16Field, Float32Field, Memory, F class CPUStats(Model): diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index 872c54b..0000000 --- a/buildout.cfg +++ /dev/null @@ -1,68 +0,0 @@ -[buildout] -prefer-final = false -newest = false -download-cache = .cache -develop = . -parts = -relative-paths = true - -[project] -name = infi.clickhouse_orm -company = Infinidat -namespace_packages = ['infi'] -install_requires = [ - 'iso8601 >= 0.1.12', - 'pytz', - 'requests', - 'setuptools' - ] -version_file = src/infi/clickhouse_orm/__version__.py -description = A Python library for working with the ClickHouse database -long_description = A Python library for working with the ClickHouse database -console_scripts = [] -gui_scripts = [] -package_data = [] -upgrade_code = {58530fba-3932-11e6-a20e-7071bc32067f} -product_name = infi.clickhouse_orm -post_install_script_name = None -pre_uninstall_script_name = None -homepage = https://github.com/Infinidat/infi.clickhouse_orm - -[isolated-python] -recipe = infi.recipe.python -version = v3.8.0.2 - -[setup.py] -recipe = infi.recipe.template.version -input = setup.in -output = setup.py - -[__version__.py] -recipe = infi.recipe.template.version -output = ${project:version_file} - -[development-scripts] -dependent-scripts = true -recipe = infi.recipe.console_scripts -eggs = ${project:name} - ipython<6 - nose - coverage - enum-compat - infi.unittest - infi.traceback - memory_profiler - profilehooks - psutil - zc.buildout -scripts = ipython - nosetests -interpreter = python - -[pack] -recipe = infi.recipe.application_packager - -[sublime] -recipe = corneti.recipes.codeintel -eggs = ${development-scripts:eggs} - diff --git a/clickhouse_orm/__init__.py b/clickhouse_orm/__init__.py new file mode 100644 index 0000000..292e25c --- /dev/null +++ b/clickhouse_orm/__init__.py @@ -0,0 +1,13 @@ +__import__("pkg_resources").declare_namespace(__name__) + +from clickhouse_orm.database import * +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import * +from clickhouse_orm.migrations import * +from clickhouse_orm.models import * +from clickhouse_orm.query import * +from clickhouse_orm.system_models import * + +from inspect import isclass +__all__ = [c.__name__ for c in locals().values() if isclass(c)] diff --git a/src/infi/clickhouse_orm/database.py b/clickhouse_orm/database.py similarity index 99% rename from src/infi/clickhouse_orm/database.py rename to clickhouse_orm/database.py index d42e224..9419d7c 100644 --- a/src/infi/clickhouse_orm/database.py +++ b/clickhouse_orm/database.py @@ -249,7 +249,7 @@ def count(self, model_class, conditions=None): - `model_class`: the model to count. - `conditions`: optional SQL conditions (contents of the WHERE clause). ''' - from infi.clickhouse_orm.query import Q + from clickhouse_orm.query import Q query = 'SELECT count() FROM $table' if conditions: if isinstance(conditions, Q): @@ -306,7 +306,7 @@ def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions= The result is a namedtuple containing `objects` (list), `number_of_objects`, `pages_total`, `number` (of the current page), and `page_size`. ''' - from infi.clickhouse_orm.query import Q + from clickhouse_orm.query import Q count = self.count(model_class, conditions) pages_total = int(ceil(count / float(page_size))) if page_num == -1: diff --git a/src/infi/clickhouse_orm/engines.py b/clickhouse_orm/engines.py similarity index 99% rename from src/infi/clickhouse_orm/engines.py rename to clickhouse_orm/engines.py index 7fb83be..285a848 100644 --- a/src/infi/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -91,7 +91,7 @@ def create_table_sql(self, db): elif not self.date_col: # Can't import it globally due to circular import - from infi.clickhouse_orm.database import DatabaseException + from clickhouse_orm.database import DatabaseException raise DatabaseException("Custom partitioning is not supported before ClickHouse 1.1.54310. " "Please update your server or use date_col syntax." "https://clickhouse.tech/docs/en/table_engines/custom_partitioning_key/") diff --git a/src/infi/clickhouse_orm/fields.py b/clickhouse_orm/fields.py similarity index 100% rename from src/infi/clickhouse_orm/fields.py rename to clickhouse_orm/fields.py diff --git a/src/infi/clickhouse_orm/funcs.py b/clickhouse_orm/funcs.py similarity index 100% rename from src/infi/clickhouse_orm/funcs.py rename to clickhouse_orm/funcs.py diff --git a/src/infi/clickhouse_orm/migrations.py b/clickhouse_orm/migrations.py similarity index 99% rename from src/infi/clickhouse_orm/migrations.py rename to clickhouse_orm/migrations.py index c8c656a..dc37f20 100644 --- a/src/infi/clickhouse_orm/migrations.py +++ b/clickhouse_orm/migrations.py @@ -97,7 +97,7 @@ def apply(self, database): # Identify fields whose type was changed # The order of class attributes can be changed any time, so we can't count on it # Secondly, MATERIALIZED and ALIAS fields are always at the end of the DESC, so we can't expect them to save - # attribute position. Watch https://github.com/Infinidat/infi.clickhouse_orm/issues/47 + # attribute position. Watch https://github.com/Infinidat/clickhouse_orm/issues/47 model_fields = {name: field.get_sql(with_default_expression=False, db=database) for name, field in self.model_class.fields().items()} for field_name, field_sql in self._get_table_fields(database): diff --git a/src/infi/clickhouse_orm/models.py b/clickhouse_orm/models.py similarity index 99% rename from src/infi/clickhouse_orm/models.py rename to clickhouse_orm/models.py index e3f95e3..0fc5e7f 100644 --- a/src/infi/clickhouse_orm/models.py +++ b/clickhouse_orm/models.py @@ -200,7 +200,7 @@ def create_ad_hoc_model(cls, fields, model_name='AdHocModel'): @classmethod def create_ad_hoc_field(cls, db_type): - import infi.clickhouse_orm.fields as orm_fields + import clickhouse_orm.fields as orm_fields # Enums if db_type.startswith('Enum'): return orm_fields.BaseEnumField.create_ad_hoc_field(db_type) diff --git a/src/infi/clickhouse_orm/query.py b/clickhouse_orm/query.py similarity index 99% rename from src/infi/clickhouse_orm/query.py rename to clickhouse_orm/query.py index 92efec4..675c98a 100644 --- a/src/infi/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -23,7 +23,7 @@ def to_sql(self, model_cls, field_name, value): raise NotImplementedError # pragma: no cover def _value_to_sql(self, field, value, quote=True): - from infi.clickhouse_orm.funcs import F + from clickhouse_orm.funcs import F if isinstance(value, F): return value.to_sql() return field.to_db_string(field.to_python(value, pytz.utc), quote) diff --git a/src/infi/clickhouse_orm/system_models.py b/clickhouse_orm/system_models.py similarity index 100% rename from src/infi/clickhouse_orm/system_models.py rename to clickhouse_orm/system_models.py diff --git a/src/infi/clickhouse_orm/utils.py b/clickhouse_orm/utils.py similarity index 98% rename from src/infi/clickhouse_orm/utils.py rename to clickhouse_orm/utils.py index c0d0325..78701ff 100644 --- a/src/infi/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -48,7 +48,7 @@ def arg_to_sql(arg): Supports functions, model fields, strings, dates, datetimes, timedeltas, booleans, None, numbers, timezones, arrays/iterables. """ - from infi.clickhouse_orm import Field, StringField, DateTimeField, DateField, F, QuerySet + from clickhouse_orm import Field, StringField, DateTimeField, DateField, F, QuerySet if isinstance(arg, F): return arg.to_sql() if isinstance(arg, Field): diff --git a/docs/class_reference.md b/docs/class_reference.md index 08716a5..3d91cdc 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -1,7 +1,7 @@ Class Reference =============== -infi.clickhouse_orm.database +clickhouse_orm.database ---------------------------- ### Database @@ -152,7 +152,7 @@ Extends Exception Raised when a database operation fails. -infi.clickhouse_orm.models +clickhouse_orm.models -------------------------- ### Model @@ -811,7 +811,7 @@ separated by non-alphanumeric characters. - `random_seed` — The seed for Bloom filter hash functions. -infi.clickhouse_orm.fields +clickhouse_orm.fields -------------------------- ### ArrayField @@ -1046,7 +1046,7 @@ Extends Field #### UUIDField(default=None, alias=None, materialized=None, readonly=None, codec=None) -infi.clickhouse_orm.engines +clickhouse_orm.engines --------------------------- ### Engine @@ -1140,7 +1140,7 @@ Extends MergeTree #### ReplacingMergeTree(date_col=None, order_by=(), ver_col=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, primary_key=None) -infi.clickhouse_orm.query +clickhouse_orm.query ------------------------- ### QuerySet @@ -1443,7 +1443,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier #### to_sql(model_cls) -infi.clickhouse_orm.funcs +clickhouse_orm.funcs ------------------------- ### F diff --git a/docs/contributing.md b/docs/contributing.md index c173cb9..c46fa0b 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,7 +1,7 @@ Contributing ============ -This project is hosted on GitHub - [https://github.com/Infinidat/infi.clickhouse_orm/](https://github.com/Infinidat/infi.clickhouse_orm/). +This project is hosted on GitHub - [https://github.com/Infinidat/clickhouse_orm/](https://github.com/Infinidat/clickhouse_orm/). Please open an issue there if you encounter a bug or want to request a feature. Pull requests are also welcome. @@ -12,7 +12,7 @@ Building After cloning the project, run the following commands: easy_install -U infi.projector - cd infi.clickhouse_orm + cd clickhouse_orm projector devenv build A `setup.py` file will be generated, which you can use to install the development version of the package: @@ -28,7 +28,7 @@ To run the tests, ensure that the ClickHouse server is running on >](class_reference.md) \ No newline at end of file +[<< System Models](system_models.md) | [Table of Contents](toc.md) | [Class Reference >>](class_reference.md) diff --git a/docs/expressions.md b/docs/expressions.md index d9237bd..c339af8 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -13,7 +13,7 @@ Using Expressions Expressions usually include ClickHouse database functions, which are made available by the `F` class. Here's a simple function: ```python -from infi.clickhouse_orm import F +from clickhouse_orm import F expr = F.today() ``` diff --git a/docs/field_options.md b/docs/field_options.md index 3019c98..77d1beb 100644 --- a/docs/field_options.md +++ b/docs/field_options.md @@ -25,7 +25,7 @@ class Event(Model): engine = Memory() ... ``` -When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `infi.clickhouse_orm.utils.NO_VALUE` instead. For example: +When creating a model instance, any fields you do not specify get their default value. Fields that use a default expression are assigned a sentinel value of `clickhouse_orm.utils.NO_VALUE` instead. For example: ```python >>> event = Event() >>> print(event.to_dict()) @@ -63,7 +63,7 @@ db.select('SELECT created, created_date, username, name FROM $db.event', model_c # created_date and username will contain a default value db.select('SELECT * FROM $db.event', model_class=Event) ``` -When creating a model instance, any alias or materialized fields are assigned a sentinel value of `infi.clickhouse_orm.utils.NO_VALUE` since their real values can only be known after insertion to the database. +When creating a model instance, any alias or materialized fields are assigned a sentinel value of `clickhouse_orm.utils.NO_VALUE` since their real values can only be known after insertion to the database. ## codec @@ -109,4 +109,4 @@ This attribute is set automatically for fields with `alias` or `materialized` at --- -[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md) \ No newline at end of file +[<< Querysets](querysets.md) | [Table of Contents](toc.md) | [Field Types >>](field_types.md) diff --git a/docs/field_types.md b/docs/field_types.md index 7613276..b8c16b3 100644 --- a/docs/field_types.md +++ b/docs/field_types.md @@ -166,7 +166,7 @@ For example, we can create a BooleanField which will hold `True` and `False` val Here's the full implementation: ```python -from infi.clickhouse_orm import Field +from clickhouse_orm import Field class BooleanField(Field): diff --git a/docs/importing_orm_classes.md b/docs/importing_orm_classes.md index 77d04e4..13bda0d 100644 --- a/docs/importing_orm_classes.md +++ b/docs/importing_orm_classes.md @@ -7,24 +7,24 @@ The ORM supports different styles of importing and referring to its classes, so Importing Everything -------------------- -It is safe to use `import *` from `infi.clickhouse_orm` or its submodules. Only classes that are needed by users of the ORM will get imported, and nothing else: +It is safe to use `import *` from `clickhouse_orm` or its submodules. Only classes that are needed by users of the ORM will get imported, and nothing else: ```python -from infi.clickhouse_orm import * +from clickhouse_orm import * ``` This is exactly equivalent to the following import statements: ```python -from infi.clickhouse_orm.database import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.funcs import * -from infi.clickhouse_orm.migrations import * -from infi.clickhouse_orm.models import * -from infi.clickhouse_orm.query import * -from infi.clickhouse_orm.system_models import * +from clickhouse_orm.database import * +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import * +from clickhouse_orm.migrations import * +from clickhouse_orm.models import * +from clickhouse_orm.query import * +from clickhouse_orm.system_models import * ``` By importing everything, all of the ORM's public classes can be used directly. For example: ```python -from infi.clickhouse_orm import * +from clickhouse_orm import * class Event(Model): @@ -40,8 +40,8 @@ Importing Everything into a Namespace To prevent potential name clashes and to make the code more readable, you can import the ORM's classes into a namespace of your choosing, e.g. `orm`. For brevity, it is recommended to import the `F` class explicitly: ```python -import infi.clickhouse_orm as orm -from infi.clickhouse_orm import F +import clickhouse_orm as orm +from clickhouse_orm import F class Event(orm.Model): @@ -57,7 +57,7 @@ Importing Specific Submodules It is possible to import only the submodules you need, and use their names to qualify the ORM's class names. This option is more verbose, but makes it clear where each class comes from. For example: ```python -from infi.clickhouse_orm import models, fields, engines, F +from clickhouse_orm import models, fields, engines, F class Event(models.Model): @@ -71,9 +71,9 @@ class Event(models.Model): Importing Specific Classes -------------------------- -If you prefer, you can import only the specific ORM classes that you need directly from `infi.clickhouse_orm`: +If you prefer, you can import only the specific ORM classes that you need directly from `clickhouse_orm`: ```python -from infi.clickhouse_orm import Model, StringField, UInt32Field, DateTimeField, F, Memory +from clickhouse_orm import Model, StringField, UInt32Field, DateTimeField, F, Memory class Event(Model): diff --git a/docs/index.md b/docs/index.md index db75910..a19d21a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,10 +8,10 @@ Version 1.x supports Python 2.7 and 3.5+. Version 2.x dropped support for Python Installation ------------ -To install infi.clickhouse_orm: +To install clickhouse_orm: - pip install infi.clickhouse_orm + pip install clickhouse_orm --- -[Table of Contents](toc.md) | [Models and Databases >>](models_and_databases.md) \ No newline at end of file +[Table of Contents](toc.md) | [Models and Databases >>](models_and_databases.md) diff --git a/docs/models_and_databases.md b/docs/models_and_databases.md index 28ab6ad..06790ef 100644 --- a/docs/models_and_databases.md +++ b/docs/models_and_databases.md @@ -10,7 +10,7 @@ Defining Models Models are defined in a way reminiscent of Django's ORM, by subclassing `Model`: ```python -from infi.clickhouse_orm import Model, StringField, DateField, Float32Field, MergeTree +from clickhouse_orm import Model, StringField, DateField, Float32Field, MergeTree class Person(Model): @@ -133,7 +133,7 @@ Inserting to the Database To write your instances to ClickHouse, you need a `Database` instance: - from infi.clickhouse_orm import Database + from clickhouse_orm import Database db = Database('my_test_db') @@ -247,4 +247,4 @@ Note that `order_by` must be chosen so that the ordering is unique, otherwise th --- -[<< Overview](index.md) | [Table of Contents](toc.md) | [Expressions >>](expressions.md) \ No newline at end of file +[<< Overview](index.md) | [Table of Contents](toc.md) | [Expressions >>](expressions.md) diff --git a/docs/ref.md b/docs/ref.md index 4679b2b..6b513ad 100644 --- a/docs/ref.md +++ b/docs/ref.md @@ -1,7 +1,7 @@ Class Reference =============== -infi.clickhouse_orm.database +clickhouse_orm.database ---------------------------- ### Database @@ -104,7 +104,7 @@ Extends Exception Raised when a database operation fails. -infi.clickhouse_orm.models +clickhouse_orm.models -------------------------- ### Model @@ -263,7 +263,7 @@ Returns the instance's column values as a tab-separated line. A newline is not i - `include_readonly`: if false, returns only fields that can be inserted into database. -infi.clickhouse_orm.fields +clickhouse_orm.fields -------------------------- ### Field @@ -419,7 +419,7 @@ Extends BaseEnumField #### Enum16Field(enum_cls, default=None, alias=None, materialized=None) -infi.clickhouse_orm.engines +clickhouse_orm.engines --------------------------- ### Engine @@ -474,7 +474,7 @@ Extends MergeTree #### ReplacingMergeTree(date_col, key_cols, ver_col=None, sampling_expr=None, index_granularity=8192, replica_table_path=None, replica_name=None) -infi.clickhouse_orm.query +clickhouse_orm.query ------------------------- ### QuerySet diff --git a/docs/schema_migrations.md b/docs/schema_migrations.md index 9e3fa01..cb8d4f1 100644 --- a/docs/schema_migrations.md +++ b/docs/schema_migrations.md @@ -22,7 +22,7 @@ To write migrations, create a Python package. Then create a python file for the Each migration file is expected to contain a list of `operations`, for example: - from infi.clickhouse_orm import migrations + from clickhouse_orm import migrations from analytics import models operations = [ @@ -109,4 +109,4 @@ Note that you may have more than one migrations package. --- -[<< Table Engines](table_engines.md) | [Table of Contents](toc.md) | [System Models >>](system_models.md) \ No newline at end of file +[<< Table Engines](table_engines.md) | [Table of Contents](toc.md) | [System Models >>](system_models.md) diff --git a/docs/system_models.md b/docs/system_models.md index 01979b3..214dba1 100644 --- a/docs/system_models.md +++ b/docs/system_models.md @@ -30,7 +30,7 @@ A partition in a table is data for a single calendar month. Table "system.parts" Usage example: - from infi.clickhouse_orm import Database, SystemPart + from clickhouse_orm import Database, SystemPart db = Database('my_test_db', db_url='http://192.168.1.1:8050', username='scott', password='tiger') partitions = SystemPart.get_active(db, conditions='') # Getting all active partitions of the database if len(partitions) > 0: @@ -43,4 +43,4 @@ Usage example: --- -[<< Schema Migrations](schema_migrations.md) | [Table of Contents](toc.md) | [Contributing >>](contributing.md) \ No newline at end of file +[<< Schema Migrations](schema_migrations.md) | [Table of Contents](toc.md) | [Contributing >>](contributing.md) diff --git a/docs/toc.md b/docs/toc.md index 2fd878f..e973772 100644 --- a/docs/toc.md +++ b/docs/toc.md @@ -78,17 +78,17 @@ * [Tests](contributing.md#tests) * [Class Reference](class_reference.md#class-reference) - * [infi.clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase) + * [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase) * [Database](class_reference.md#database) * [DatabaseException](class_reference.md#databaseexception) - * [infi.clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels) + * [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels) * [Model](class_reference.md#model) * [BufferModel](class_reference.md#buffermodel) * [MergeModel](class_reference.md#mergemodel) * [DistributedModel](class_reference.md#distributedmodel) * [Constraint](class_reference.md#constraint) * [Index](class_reference.md#index) - * [infi.clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields) + * [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields) * [ArrayField](class_reference.md#arrayfield) * [BaseEnumField](class_reference.md#baseenumfield) * [BaseFloatField](class_reference.md#basefloatfield) @@ -120,7 +120,7 @@ * [UInt64Field](class_reference.md#uint64field) * [UInt8Field](class_reference.md#uint8field) * [UUIDField](class_reference.md#uuidfield) - * [infi.clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines) + * [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines) * [Engine](class_reference.md#engine) * [TinyLog](class_reference.md#tinylog) * [Log](class_reference.md#log) @@ -132,10 +132,10 @@ * [CollapsingMergeTree](class_reference.md#collapsingmergetree) * [SummingMergeTree](class_reference.md#summingmergetree) * [ReplacingMergeTree](class_reference.md#replacingmergetree) - * [infi.clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery) + * [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery) * [QuerySet](class_reference.md#queryset) * [AggregateQuerySet](class_reference.md#aggregatequeryset) * [Q](class_reference.md#q) - * [infi.clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs) + * [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs) * [F](class_reference.md#f) diff --git a/docs/whats_new_in_version_2.md b/docs/whats_new_in_version_2.md index 378adca..df8a77c 100644 --- a/docs/whats_new_in_version_2.md +++ b/docs/whats_new_in_version_2.md @@ -50,9 +50,9 @@ for row in QueryLog.objects_in(db).filter(QueryLog.query_duration_ms > 10000): ## Convenient ways to import ORM classes -You can now import all ORM classes directly from `infi.clickhouse_orm`, without worrying about sub-modules. For example: +You can now import all ORM classes directly from `clickhouse_orm`, without worrying about sub-modules. For example: ```python -from infi.clickhouse_orm import Database, Model, StringField, DateTimeField, MergeTree +from clickhouse_orm import Database, Model, StringField, DateTimeField, MergeTree ``` See [Importing ORM Classes](importing_orm_classes.md). diff --git a/examples/cpu_usage/collect.py b/examples/cpu_usage/collect.py index 34ee5b4..62102ac 100644 --- a/examples/cpu_usage/collect.py +++ b/examples/cpu_usage/collect.py @@ -1,5 +1,5 @@ import psutil, time, datetime -from infi.clickhouse_orm import Database +from clickhouse_orm import Database from models import CPUStats diff --git a/examples/cpu_usage/models.py b/examples/cpu_usage/models.py index c19007a..fe9afc6 100644 --- a/examples/cpu_usage/models.py +++ b/examples/cpu_usage/models.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import Model, DateTimeField, UInt16Field, Float32Field, Memory +from clickhouse_orm import Model, DateTimeField, UInt16Field, Float32Field, Memory class CPUStats(Model): diff --git a/examples/cpu_usage/requirements.txt b/examples/cpu_usage/requirements.txt index 5e08b8f..4890efc 100644 --- a/examples/cpu_usage/requirements.txt +++ b/examples/cpu_usage/requirements.txt @@ -1,2 +1,2 @@ -infi.clickhouse_orm +clickhouse_orm psutil diff --git a/examples/cpu_usage/results.py b/examples/cpu_usage/results.py index 80b892f..06ee1f0 100644 --- a/examples/cpu_usage/results.py +++ b/examples/cpu_usage/results.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import Database, F +from clickhouse_orm import Database, F from models import CPUStats diff --git a/examples/db_explorer/requirements.txt b/examples/db_explorer/requirements.txt index 8dee9f8..80447d7 100644 --- a/examples/db_explorer/requirements.txt +++ b/examples/db_explorer/requirements.txt @@ -3,7 +3,7 @@ chardet==3.0.4 click==7.1.2 Flask==1.1.2 idna==2.9 -infi.clickhouse-orm==2.0.1 +clickhouse-orm==2.0.1 iso8601==0.1.12 itsdangerous==1.1.0 Jinja2==2.11.2 diff --git a/examples/db_explorer/server.py b/examples/db_explorer/server.py index 6241ed9..9fbcc03 100644 --- a/examples/db_explorer/server.py +++ b/examples/db_explorer/server.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import Database, F +from clickhouse_orm import Database, F from charts import tables_piechart, columns_piechart, number_formatter, bytes_formatter from flask import Flask from flask import render_template diff --git a/examples/full_text_search/load.py b/examples/full_text_search/load.py index 7cf43b0..51564f4 100644 --- a/examples/full_text_search/load.py +++ b/examples/full_text_search/load.py @@ -2,7 +2,7 @@ import nltk from nltk.stem.porter import PorterStemmer from glob import glob -from infi.clickhouse_orm import Database +from clickhouse_orm import Database from models import Fragment diff --git a/examples/full_text_search/models.py b/examples/full_text_search/models.py index 130fe83..80de2b7 100644 --- a/examples/full_text_search/models.py +++ b/examples/full_text_search/models.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import * +from clickhouse_orm import * class Fragment(Model): diff --git a/examples/full_text_search/requirements.txt b/examples/full_text_search/requirements.txt index 6d2f877..d157381 100644 --- a/examples/full_text_search/requirements.txt +++ b/examples/full_text_search/requirements.txt @@ -1,4 +1,4 @@ -infi.clickhouse_orm +clickhouse_orm nltk requests colorama diff --git a/examples/full_text_search/search.py b/examples/full_text_search/search.py index ff5fcea..c4d0918 100644 --- a/examples/full_text_search/search.py +++ b/examples/full_text_search/search.py @@ -1,7 +1,7 @@ import sys from colorama import init, Fore, Back, Style from nltk.stem.porter import PorterStemmer -from infi.clickhouse_orm import Database, F +from clickhouse_orm import Database, F from models import Fragment from load import trim_punctuation diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py index 22850ab..16473b4 100644 --- a/scripts/generate_ref.py +++ b/scripts/generate_ref.py @@ -120,13 +120,13 @@ def all_subclasses(cls): if __name__ == '__main__': - from infi.clickhouse_orm import database - from infi.clickhouse_orm import fields - from infi.clickhouse_orm import engines - from infi.clickhouse_orm import models - from infi.clickhouse_orm import query - from infi.clickhouse_orm import funcs - from infi.clickhouse_orm import system_models + from clickhouse_orm import database + from clickhouse_orm import fields + from clickhouse_orm import engines + from clickhouse_orm import models + from clickhouse_orm import query + from clickhouse_orm import funcs + from clickhouse_orm import system_models print('Class Reference') print('===============') diff --git a/scripts/test_python3.sh b/scripts/test_python3.sh index 86016cd..455d5b7 100755 --- a/scripts/test_python3.sh +++ b/scripts/test_python3.sh @@ -5,7 +5,7 @@ virtualenv -p python3 /tmp/orm_env cd /tmp/orm_env source bin/activate pip install infi.projector -git clone https://github.com/Infinidat/infi.clickhouse_orm.git -cd infi.clickhouse_orm +git clone https://github.com/Infinidat/clickhouse_orm.git +cd clickhouse_orm projector devenv build bin/nosetests diff --git a/src/infi/__init__.py b/src/infi/__init__.py deleted file mode 100644 index 5284146..0000000 --- a/src/infi/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__import__("pkg_resources").declare_namespace(__name__) diff --git a/src/infi/clickhouse_orm/__init__.py b/src/infi/clickhouse_orm/__init__.py deleted file mode 100644 index c7982cc..0000000 --- a/src/infi/clickhouse_orm/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -__import__("pkg_resources").declare_namespace(__name__) - -from infi.clickhouse_orm.database import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.funcs import * -from infi.clickhouse_orm.migrations import * -from infi.clickhouse_orm.models import * -from infi.clickhouse_orm.query import * -from infi.clickhouse_orm.system_models import * - -from inspect import isclass -__all__ = [c.__name__ for c in locals().values() if isclass(c)] diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index 53d2343..f48d11b 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import unittest -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * import logging logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/sample_migrations/0001_initial.py b/tests/sample_migrations/0001_initial.py index a289d86..c06bac9 100644 --- a/tests/sample_migrations/0001_initial.py +++ b/tests/sample_migrations/0001_initial.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.CreateTable(Model1) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0002.py b/tests/sample_migrations/0002.py index 6e4e0d9..2904c12 100644 --- a/tests/sample_migrations/0002.py +++ b/tests/sample_migrations/0002.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.DropTable(Model1) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0003.py b/tests/sample_migrations/0003.py index a289d86..c06bac9 100644 --- a/tests/sample_migrations/0003.py +++ b/tests/sample_migrations/0003.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.CreateTable(Model1) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0004.py b/tests/sample_migrations/0004.py index 6d10205..7a3322d 100644 --- a/tests/sample_migrations/0004.py +++ b/tests/sample_migrations/0004.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.AlterTable(Model2) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0005.py b/tests/sample_migrations/0005.py index f2633ef..95e1950 100644 --- a/tests/sample_migrations/0005.py +++ b/tests/sample_migrations/0005.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.AlterTable(Model3) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0006.py b/tests/sample_migrations/0006.py index fefb325..cb8299b 100644 --- a/tests/sample_migrations/0006.py +++ b/tests/sample_migrations/0006.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.CreateTable(EnumModel1) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0007.py b/tests/sample_migrations/0007.py index da23040..1db30f7 100644 --- a/tests/sample_migrations/0007.py +++ b/tests/sample_migrations/0007.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.AlterTable(EnumModel2) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0008.py b/tests/sample_migrations/0008.py index 691a762..5631567 100644 --- a/tests/sample_migrations/0008.py +++ b/tests/sample_migrations/0008.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.CreateTable(MaterializedModel) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0009.py b/tests/sample_migrations/0009.py index 7841f17..22b53a0 100644 --- a/tests/sample_migrations/0009.py +++ b/tests/sample_migrations/0009.py @@ -1,6 +1,6 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ migrations.CreateTable(AliasModel) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0010.py b/tests/sample_migrations/0010.py index 3892583..c01c00a 100644 --- a/tests/sample_migrations/0010.py +++ b/tests/sample_migrations/0010.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0011.py b/tests/sample_migrations/0011.py index dd9d09e..0d88556 100644 --- a/tests/sample_migrations/0011.py +++ b/tests/sample_migrations/0011.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0012.py b/tests/sample_migrations/0012.py index 1dfe3bf..0120a5c 100644 --- a/tests/sample_migrations/0012.py +++ b/tests/sample_migrations/0012.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations operations = [ migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "), diff --git a/tests/sample_migrations/0013.py b/tests/sample_migrations/0013.py index 5ee6498..8bb39b9 100644 --- a/tests/sample_migrations/0013.py +++ b/tests/sample_migrations/0013.py @@ -1,6 +1,6 @@ import datetime -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from test_migrations import Model3 @@ -12,4 +12,4 @@ def forward(database): operations = [ migrations.RunPython(forward) -] \ No newline at end of file +] diff --git a/tests/sample_migrations/0014.py b/tests/sample_migrations/0014.py index 14553f3..7cbcc32 100644 --- a/tests/sample_migrations/0014.py +++ b/tests/sample_migrations/0014.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0015.py b/tests/sample_migrations/0015.py index be1d378..de75d76 100644 --- a/tests/sample_migrations/0015.py +++ b/tests/sample_migrations/0015.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0016.py b/tests/sample_migrations/0016.py index 6f0f814..21387d1 100644 --- a/tests/sample_migrations/0016.py +++ b/tests/sample_migrations/0016.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0017.py b/tests/sample_migrations/0017.py index 4151189..1a1089c 100644 --- a/tests/sample_migrations/0017.py +++ b/tests/sample_migrations/0017.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0018.py b/tests/sample_migrations/0018.py index c34c137..97f52a5 100644 --- a/tests/sample_migrations/0018.py +++ b/tests/sample_migrations/0018.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/sample_migrations/0019.py b/tests/sample_migrations/0019.py index 67ba244..ede957f 100644 --- a/tests/sample_migrations/0019.py +++ b/tests/sample_migrations/0019.py @@ -1,4 +1,4 @@ -from infi.clickhouse_orm import migrations +from clickhouse_orm import migrations from ..test_migrations import * operations = [ diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index 0039fa6..8ddcc3c 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -1,11 +1,11 @@ import unittest from datetime import date -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model, NO_VALUE -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.funcs import F +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model, NO_VALUE +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.funcs import F class AliasFieldsTest(unittest.TestCase): diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index b0b18a2..20972be 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -1,10 +1,10 @@ import unittest from datetime import date -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * class ArrayFieldsTest(unittest.TestCase): @@ -47,7 +47,7 @@ def test_assignment_error(self): instance.arr_int = value def test_parse_array(self): - from infi.clickhouse_orm.utils import parse_array, unescape + from clickhouse_orm.utils import parse_array, unescape self.assertEqual(parse_array("[]"), []) self.assertEqual(parse_array("[1, 2, 395, -44]"), ["1", "2", "395", "-44"]) self.assertEqual(parse_array("['big','mouse','','!']"), ["big", "mouse", "", "!"]) diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 14cc59a..71dafea 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import unittest -from infi.clickhouse_orm.models import BufferModel -from infi.clickhouse_orm.engines import * +from clickhouse_orm.models import BufferModel +from clickhouse_orm.engines import * from .base_test_with_data import * diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index 8d17571..828a57c 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -2,11 +2,11 @@ import datetime import pytz -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model, NO_VALUE -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.utils import parse_tsv +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model, NO_VALUE +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.utils import parse_tsv class CompressedFieldsTestCase(unittest.TestCase): @@ -120,4 +120,4 @@ class CompressedModel(Model): float_field = Float32Field(codec='NONE') alias_field = Float32Field(alias='float_field', codec='ZSTD(4)') - engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) \ No newline at end of file + engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) diff --git a/tests/test_constraints.py b/tests/test_constraints.py index 1b5892d..b66b54a 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -1,6 +1,6 @@ import unittest -from infi.clickhouse_orm import * +from clickhouse_orm import * from .base_test_with_data import Person diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py index 641da27..fee6730 100644 --- a/tests/test_custom_fields.py +++ b/tests/test_custom_fields.py @@ -1,8 +1,8 @@ import unittest -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.fields import Field, Int16Field -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.engines import Memory +from clickhouse_orm.database import Database +from clickhouse_orm.fields import Field, Int16Field +from clickhouse_orm.models import Model +from clickhouse_orm.engines import Memory class CustomFieldsTest(unittest.TestCase): diff --git a/tests/test_database.py b/tests/test_database.py index 38681d4..d66e23c 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -2,12 +2,12 @@ import unittest import datetime -from infi.clickhouse_orm.database import ServerError, DatabaseException -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.engines import Memory -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.funcs import F -from infi.clickhouse_orm.query import Q +from clickhouse_orm.database import ServerError, DatabaseException +from clickhouse_orm.models import Model +from clickhouse_orm.engines import Memory +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import F +from clickhouse_orm.query import Q from .base_test_with_data import * @@ -248,7 +248,7 @@ def test_add_setting(self): def test_create_ad_hoc_field(self): # Tests that create_ad_hoc_field works for all column types in the database - from infi.clickhouse_orm.models import ModelBase + from clickhouse_orm.models import ModelBase query = "SELECT DISTINCT type FROM system.columns" for row in self.database.select(query): ModelBase.create_ad_hoc_field(row.type) diff --git a/tests/test_datetime_fields.py b/tests/test_datetime_fields.py index 6c30ffb..a8783bf 100644 --- a/tests/test_datetime_fields.py +++ b/tests/test_datetime_fields.py @@ -2,10 +2,10 @@ import datetime import pytz -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * class DateFieldsTest(unittest.TestCase): diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index 622c6d6..f26fe26 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -2,10 +2,10 @@ import unittest from decimal import Decimal -from infi.clickhouse_orm.database import Database, ServerError -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database, ServerError +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * class DecimalFieldsTest(unittest.TestCase): diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 7da4160..40bead6 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -1,7 +1,7 @@ import unittest import logging -from infi.clickhouse_orm import * +from clickhouse_orm import * class DictionaryTestMixin: diff --git a/tests/test_engines.py b/tests/test_engines.py index 2fcc8c2..77781df 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -1,7 +1,7 @@ import unittest import datetime -from infi.clickhouse_orm import * +from clickhouse_orm import * import logging logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py index 9ad4cdb..f53ce69 100644 --- a/tests/test_enum_fields.py +++ b/tests/test_enum_fields.py @@ -1,9 +1,9 @@ import unittest -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * from enum import Enum diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py index a29f3ca..6b548a1 100644 --- a/tests/test_fixed_string_fields.py +++ b/tests/test_fixed_string_fields.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import unittest -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * class FixedStringFieldsTest(unittest.TestCase): diff --git a/tests/test_funcs.py b/tests/test_funcs.py index a262e83..fb250e6 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -7,9 +7,9 @@ import logging from decimal import Decimal -from infi.clickhouse_orm.database import ServerError -from infi.clickhouse_orm.utils import NO_VALUE -from infi.clickhouse_orm.funcs import F +from clickhouse_orm.database import ServerError +from clickhouse_orm.utils import NO_VALUE +from clickhouse_orm.funcs import F class FuncsTestCase(TestCaseWithData): diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 0dccaea..63ce7c3 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -1,6 +1,6 @@ import unittest -from infi.clickhouse_orm import * +from clickhouse_orm import * class IndexesTest(unittest.TestCase): diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index 705a9b7..e15899f 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -2,10 +2,10 @@ import datetime import pytz -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * class InheritanceTestCase(unittest.TestCase): diff --git a/tests/test_ip_fields.py b/tests/test_ip_fields.py index 220aa1b..34301df 100644 --- a/tests/test_ip_fields.py +++ b/tests/test_ip_fields.py @@ -1,9 +1,9 @@ import unittest from ipaddress import IPv4Address, IPv6Address -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.fields import Int16Field, IPv4Field, IPv6Field -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.engines import Memory +from clickhouse_orm.database import Database +from clickhouse_orm.fields import Int16Field, IPv4Field, IPv6Field +from clickhouse_orm.models import Model +from clickhouse_orm.engines import Memory class IPFieldsTest(unittest.TestCase): diff --git a/tests/test_join.py b/tests/test_join.py index 48da1b3..b00c523 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -2,7 +2,7 @@ import unittest import json -from infi.clickhouse_orm import database, engines, fields, models +from clickhouse_orm import database, engines, fields, models class JoinTest(unittest.TestCase): diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index 8893229..93a9ee0 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -1,11 +1,11 @@ import unittest from datetime import date -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model, NO_VALUE -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.funcs import F +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model, NO_VALUE +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.funcs import F class MaterializedFieldsTest(unittest.TestCase): diff --git a/tests/test_migrations.py b/tests/test_migrations.py index d00357a..faaf4e5 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,10 +1,10 @@ import unittest -from infi.clickhouse_orm.database import Database, ServerError -from infi.clickhouse_orm.models import Model, BufferModel, Constraint, Index -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.migrations import MigrationHistory +from clickhouse_orm.database import Database, ServerError +from clickhouse_orm.models import Model, BufferModel, Constraint, Index +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.migrations import MigrationHistory from enum import Enum # Add tests to path so that migrations will be importable diff --git a/tests/test_models.py b/tests/test_models.py index 579c2dd..10d8b77 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,10 +2,10 @@ import datetime import pytz -from infi.clickhouse_orm.models import Model, NO_VALUE -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.funcs import F +from clickhouse_orm.models import Model, NO_VALUE +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.funcs import F class ModelTestCase(unittest.TestCase): diff --git a/tests/test_mutations.py b/tests/test_mutations.py index 874c7bd..ec291b9 100644 --- a/tests/test_mutations.py +++ b/tests/test_mutations.py @@ -1,5 +1,5 @@ import unittest -from infi.clickhouse_orm import F +from clickhouse_orm import F from .base_test_with_data import * from time import sleep diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py index ab7a777..a0357ac 100644 --- a/tests/test_nullable_fields.py +++ b/tests/test_nullable_fields.py @@ -1,11 +1,11 @@ import unittest import pytz -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.utils import comma_join +from clickhouse_orm.database import Database +from clickhouse_orm.models import Model +from clickhouse_orm.fields import * +from clickhouse_orm.engines import * +from clickhouse_orm.utils import comma_join from datetime import date, datetime diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 7f161e0..6f7116c 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import unittest -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.query import Q -from infi.clickhouse_orm.funcs import F +from clickhouse_orm.database import Database +from clickhouse_orm.query import Q +from clickhouse_orm.funcs import F from .base_test_with_data import * from datetime import date, datetime from enum import Enum diff --git a/tests/test_readonly.py b/tests/test_readonly.py index bc9b252..e136b9a 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from infi.clickhouse_orm.database import DatabaseException, ServerError +from clickhouse_orm.database import DatabaseException, ServerError from .base_test_with_data import * diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py index 60fbd2b..578ae9d 100644 --- a/tests/test_server_errors.py +++ b/tests/test_server_errors.py @@ -1,6 +1,6 @@ import unittest -from infi.clickhouse_orm.database import ServerError +from clickhouse_orm.database import ServerError class ServerErrorTest(unittest.TestCase): diff --git a/tests/test_simple_fields.py b/tests/test_simple_fields.py index 247e096..d87dcbc 100644 --- a/tests/test_simple_fields.py +++ b/tests/test_simple_fields.py @@ -1,5 +1,5 @@ import unittest -from infi.clickhouse_orm.fields import * +from clickhouse_orm.fields import * from datetime import date, datetime import pytz diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 5d3a862..77a2b78 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -3,11 +3,11 @@ import os -from infi.clickhouse_orm.database import Database, DatabaseException -from infi.clickhouse_orm.engines import * -from infi.clickhouse_orm.fields import * -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.system_models import SystemPart +from clickhouse_orm.database import Database, DatabaseException +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model +from clickhouse_orm.system_models import SystemPart class SystemTest(unittest.TestCase): diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py index 284d8f5..90e5acd 100644 --- a/tests/test_uuid_fields.py +++ b/tests/test_uuid_fields.py @@ -1,9 +1,9 @@ import unittest from uuid import UUID -from infi.clickhouse_orm.database import Database -from infi.clickhouse_orm.fields import Int16Field, UUIDField -from infi.clickhouse_orm.models import Model -from infi.clickhouse_orm.engines import Memory +from clickhouse_orm.database import Database +from clickhouse_orm.fields import Int16Field, UUIDField +from clickhouse_orm.models import Model +from clickhouse_orm.engines import Memory class UUIDFieldsTest(unittest.TestCase): From 0213aed397f34e41b9425868d6c4991ecb76d921 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 22:47:01 +0100 Subject: [PATCH 04/51] Chore: update dependencies --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b90b156..e5ea5a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.7" +requests = "^2.26.0" +pytz = "^2021.1" +iso8601 = "^0.1.16" [tool.poetry.dev-dependencies] flake8 = "^3.9.2" From 13655da35d87ef10be65e5c2341f9bee4d5baa3d Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 22:56:45 +0100 Subject: [PATCH 05/51] Chore: fix linting on query.py --- clickhouse_orm/query.py | 198 +++++++++++++++++++++------------------- 1 file changed, 106 insertions(+), 92 deletions(-) diff --git a/clickhouse_orm/query.py b/clickhouse_orm/query.py index 675c98a..d9be1be 100644 --- a/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -3,13 +3,13 @@ import pytz from copy import copy, deepcopy from math import ceil -from datetime import date, datetime from .utils import comma_join, string_or_func, arg_to_sql # TODO # - check that field names are valid + class Operator(object): """ Base class for filtering operators. @@ -20,10 +20,11 @@ def to_sql(self, model_cls, field_name, value): Subclasses should implement this method. It returns an SQL string that applies this operator on the given field and value. """ - raise NotImplementedError # pragma: no cover + raise NotImplementedError # pragma: no cover def _value_to_sql(self, field, value, quote=True): from clickhouse_orm.funcs import F + if isinstance(value, F): return value.to_sql() return field.to_db_string(field.to_python(value, pytz.utc), quote) @@ -41,9 +42,9 @@ def __init__(self, sql_operator, sql_for_null=None): def to_sql(self, model_cls, field_name, value): field = getattr(model_cls, field_name) value = self._value_to_sql(field, value) - if value == '\\N' and self._sql_for_null is not None: - return ' '.join([field_name, self._sql_for_null]) - return ' '.join([field_name, self._sql_operator, value]) + if value == "\\N" and self._sql_for_null is not None: + return " ".join([field_name, self._sql_for_null]) + return " ".join([field_name, self._sql_operator, value]) class InOperator(Operator): @@ -63,7 +64,7 @@ def to_sql(self, model_cls, field_name, value): pass else: value = comma_join([self._value_to_sql(field, v) for v in value]) - return '%s IN (%s)' % (field_name, value) + return "%s IN (%s)" % (field_name, value) class LikeOperator(Operator): @@ -79,12 +80,12 @@ def __init__(self, pattern, case_sensitive=True): def to_sql(self, model_cls, field_name, value): field = getattr(model_cls, field_name) value = self._value_to_sql(field, value, quote=False) - value = value.replace('\\', '\\\\').replace('%', '\\\\%').replace('_', '\\\\_') + value = value.replace("\\", "\\\\").replace("%", "\\\\%").replace("_", "\\\\_") pattern = self._pattern.format(value) if self._case_sensitive: - return '%s LIKE \'%s\'' % (field_name, pattern) + return "%s LIKE '%s'" % (field_name, pattern) else: - return 'lowerUTF8(%s) LIKE lowerUTF8(\'%s\')' % (field_name, pattern) + return "lowerUTF8(%s) LIKE lowerUTF8('%s')" % (field_name, pattern) class IExactOperator(Operator): @@ -95,7 +96,7 @@ class IExactOperator(Operator): def to_sql(self, model_cls, field_name, value): field = getattr(model_cls, field_name) value = self._value_to_sql(field, value) - return 'lowerUTF8(%s) = lowerUTF8(%s)' % (field_name, value) + return "lowerUTF8(%s) = lowerUTF8(%s)" % (field_name, value) class NotOperator(Operator): @@ -108,7 +109,7 @@ def __init__(self, base_operator): def to_sql(self, model_cls, field_name, value): # Negate the base operator - return 'NOT (%s)' % self._base_operator.to_sql(model_cls, field_name, value) + return "NOT (%s)" % self._base_operator.to_sql(model_cls, field_name, value) class BetweenOperator(Operator): @@ -126,35 +127,38 @@ def to_sql(self, model_cls, field_name, value): value0 = self._value_to_sql(field, value[0]) if value[0] is not None or len(str(value[0])) > 0 else None value1 = self._value_to_sql(field, value[1]) if value[1] is not None or len(str(value[1])) > 0 else None if value0 and value1: - return '%s BETWEEN %s AND %s' % (field_name, value0, value1) + return "%s BETWEEN %s AND %s" % (field_name, value0, value1) if value0 and not value1: - return ' '.join([field_name, '>=', value0]) + return " ".join([field_name, ">=", value0]) if value1 and not value0: - return ' '.join([field_name, '<=', value1]) + return " ".join([field_name, "<=", value1]) + # Define the set of builtin operators _operators = {} + def register_operator(name, sql): _operators[name] = sql -register_operator('eq', SimpleOperator('=', 'IS NULL')) -register_operator('ne', SimpleOperator('!=', 'IS NOT NULL')) -register_operator('gt', SimpleOperator('>')) -register_operator('gte', SimpleOperator('>=')) -register_operator('lt', SimpleOperator('<')) -register_operator('lte', SimpleOperator('<=')) -register_operator('between', BetweenOperator()) -register_operator('in', InOperator()) -register_operator('not_in', NotOperator(InOperator())) -register_operator('contains', LikeOperator('%{}%')) -register_operator('startswith', LikeOperator('{}%')) -register_operator('endswith', LikeOperator('%{}')) -register_operator('icontains', LikeOperator('%{}%', False)) -register_operator('istartswith', LikeOperator('{}%', False)) -register_operator('iendswith', LikeOperator('%{}', False)) -register_operator('iexact', IExactOperator()) + +register_operator("eq", SimpleOperator("=", "IS NULL")) +register_operator("ne", SimpleOperator("!=", "IS NOT NULL")) +register_operator("gt", SimpleOperator(">")) +register_operator("gte", SimpleOperator(">=")) +register_operator("lt", SimpleOperator("<")) +register_operator("lte", SimpleOperator("<=")) +register_operator("between", BetweenOperator()) +register_operator("in", InOperator()) +register_operator("not_in", NotOperator(InOperator())) +register_operator("contains", LikeOperator("%{}%")) +register_operator("startswith", LikeOperator("{}%")) +register_operator("endswith", LikeOperator("%{}")) +register_operator("icontains", LikeOperator("%{}%", False)) +register_operator("istartswith", LikeOperator("{}%", False)) +register_operator("iendswith", LikeOperator("%{}", False)) +register_operator("iexact", IExactOperator()) class Cond(object): @@ -170,19 +174,20 @@ class FieldCond(Cond): """ A single query condition made up of Field + Operator + Value. """ + def __init__(self, field_name, operator, value): self._field_name = field_name self._operator = _operators.get(operator) if self._operator is None: # The field name contains __ like my__field - self._field_name = field_name + '__' + operator - self._operator = _operators['eq'] + self._field_name = field_name + "__" + operator + self._operator = _operators["eq"] self._value = value def to_sql(self, model_cls): return self._operator.to_sql(model_cls, self._field_name, self._value) - def __deepcopy__(self, memodict={}): + def __deepcopy__(self, memo): res = copy(self) res._value = deepcopy(self._value) return res @@ -190,8 +195,8 @@ def __deepcopy__(self, memodict={}): class Q(object): - AND_MODE = 'AND' - OR_MODE = 'OR' + AND_MODE = "AND" + OR_MODE = "OR" def __init__(self, *filter_funcs, **filter_fields): self._conds = list(filter_funcs) + [self._build_cond(k, v) for k, v in filter_fields.items()] @@ -224,10 +229,10 @@ def _construct_from(cls, l_child, r_child, mode): return q def _build_cond(self, key, value): - if '__' in key: - field_name, operator = key.rsplit('__', 1) + if "__" in key: + field_name, operator = key.rsplit("__", 1) else: - field_name, operator = key, 'eq' + field_name, operator = key, "eq" return FieldCond(field_name, operator, value) def to_sql(self, model_cls): @@ -241,16 +246,16 @@ def to_sql(self, model_cls): if not condition_sql: # Empty Q() object returns everything - sql = '1' + sql = "1" elif len(condition_sql) == 1: # Skip not needed brackets over single condition sql = condition_sql[0] else: # Each condition must be enclosed in brackets, or order of operations may be wrong - sql = '(%s)' % ') {} ('.format(self._mode).join(condition_sql) + sql = "(%s)" % ") {} (".format(self._mode).join(condition_sql) if self._negate: - sql = 'NOT (%s)' % sql + sql = "NOT (%s)" % sql return sql @@ -268,7 +273,7 @@ def __invert__(self): def __bool__(self): return not self.is_empty - def __deepcopy__(self, memodict={}): + def __deepcopy__(self, memo): q = Q() q._conds = [deepcopy(cond) for cond in self._conds] q._negate = self._negate @@ -318,7 +323,7 @@ def __bool__(self): """ return bool(self.count()) - def __nonzero__(self): # Python 2 compatibility + def __nonzero__(self): # Python 2 compatibility return type(self).__bool__(self) def __str__(self): @@ -327,17 +332,17 @@ def __str__(self): def __getitem__(self, s): if isinstance(s, int): # Single index - assert s >= 0, 'negative indexes are not supported' + assert s >= 0, "negative indexes are not supported" qs = copy(self) qs._limits = (s, 1) return next(iter(qs)) else: # Slice - assert s.step in (None, 1), 'step is not supported in slices' + assert s.step in (None, 1), "step is not supported in slices" start = s.start or 0 - stop = s.stop or 2**63 - 1 - assert start >= 0 and stop >= 0, 'negative indexes are not supported' - assert start <= stop, 'start of slice cannot be smaller than its end' + stop = s.stop or 2 ** 63 - 1 + assert start >= 0 and stop >= 0, "negative indexes are not supported" + assert start <= stop, "start of slice cannot be smaller than its end" qs = copy(self) qs._limits = (start, stop - start) return qs @@ -353,7 +358,7 @@ def limit_by(self, offset_limit, *fields_or_expr): offset_limit = (0, offset_limit) offset = offset_limit[0] limit = offset_limit[1] - assert offset >= 0 and limit >= 0, 'negative limits are not supported' + assert offset >= 0 and limit >= 0, "negative limits are not supported" qs = copy(self) qs._limit_by = (offset, limit) qs._limit_by_fields = fields_or_expr @@ -363,44 +368,44 @@ def select_fields_as_sql(self): """ Returns the selected fields or expressions as a SQL string. """ - fields = '*' + fields = "*" if self._fields: - fields = comma_join('`%s`' % field for field in self._fields) + fields = comma_join("`%s`" % field for field in self._fields) return fields def as_sql(self): """ Returns the whole query as a SQL string. """ - distinct = 'DISTINCT ' if self._distinct else '' - final = ' FINAL' if self._final else '' - table_name = '`%s`' % self._model_cls.table_name() + distinct = "DISTINCT " if self._distinct else "" + final = " FINAL" if self._final else "" + table_name = "`%s`" % self._model_cls.table_name() if self._model_cls.is_system_model(): - table_name = '`system`.' + table_name + table_name = "`system`." + table_name params = (distinct, self.select_fields_as_sql(), table_name, final) - sql = u'SELECT %s%s\nFROM %s%s' % params + sql = "SELECT %s%s\nFROM %s%s" % params if self._prewhere_q and not self._prewhere_q.is_empty: - sql += '\nPREWHERE ' + self.conditions_as_sql(prewhere=True) + sql += "\nPREWHERE " + self.conditions_as_sql(prewhere=True) if self._where_q and not self._where_q.is_empty: - sql += '\nWHERE ' + self.conditions_as_sql(prewhere=False) + sql += "\nWHERE " + self.conditions_as_sql(prewhere=False) if self._grouping_fields: - sql += '\nGROUP BY %s' % comma_join('`%s`' % field for field in self._grouping_fields) + sql += "\nGROUP BY %s" % comma_join("`%s`" % field for field in self._grouping_fields) if self._grouping_with_totals: - sql += ' WITH TOTALS' + sql += " WITH TOTALS" if self._order_by: - sql += '\nORDER BY ' + self.order_by_as_sql() + sql += "\nORDER BY " + self.order_by_as_sql() if self._limit_by: - sql += '\nLIMIT %d, %d' % self._limit_by - sql += ' BY %s' % comma_join(string_or_func(field) for field in self._limit_by_fields) + sql += "\nLIMIT %d, %d" % self._limit_by + sql += " BY %s" % comma_join(string_or_func(field) for field in self._limit_by_fields) if self._limits: - sql += '\nLIMIT %d, %d' % self._limits + sql += "\nLIMIT %d, %d" % self._limits return sql @@ -408,10 +413,12 @@ def order_by_as_sql(self): """ Returns the contents of the query's `ORDER BY` clause as a string. """ - return comma_join([ - '%s DESC' % field[1:] if isinstance(field, str) and field[0] == '-' else str(field) - for field in self._order_by - ]) + return comma_join( + [ + "%s DESC" % field[1:] if isinstance(field, str) and field[0] == "-" else str(field) + for field in self._order_by + ] + ) def conditions_as_sql(self, prewhere=False): """ @@ -426,7 +433,7 @@ def count(self): """ if self._distinct or self._limits: # Use a subquery, since a simple count won't be accurate - sql = u'SELECT count() FROM (%s)' % self.as_sql() + sql = "SELECT count() FROM (%s)" % self.as_sql() raw = self._database.raw(sql) return int(raw) if raw else 0 @@ -455,8 +462,8 @@ def only(self, *field_names): def _filter_or_exclude(self, *q, **kwargs): from .funcs import F - inverse = kwargs.pop('_inverse', False) - prewhere = kwargs.pop('prewhere', False) + inverse = kwargs.pop("_inverse", False) + prewhere = kwargs.pop("prewhere", False) qs = copy(self) @@ -510,19 +517,20 @@ def paginate(self, page_num=1, page_size=100): `pages_total`, `number` (of the current page), and `page_size`. """ from .database import Page + count = self.count() pages_total = int(ceil(count / float(page_size))) if page_num == -1: page_num = pages_total elif page_num < 1: - raise ValueError('Invalid page number: %d' % page_num) + raise ValueError("Invalid page number: %d" % page_num) offset = (page_num - 1) * page_size return Page( objects=list(self[offset : offset + page_size]), number_of_objects=count, pages_total=pages_total, number=page_num, - page_size=page_size + page_size=page_size, ) def distinct(self): @@ -540,8 +548,11 @@ def final(self): Can be used with the `CollapsingMergeTree` and `ReplacingMergeTree` engines only. """ from .engines import CollapsingMergeTree, ReplacingMergeTree + if not isinstance(self._model_cls.engine, (CollapsingMergeTree, ReplacingMergeTree)): - raise TypeError('final() method can be used only with the CollapsingMergeTree and ReplacingMergeTree engines') + raise TypeError( + "final() method can be used only with the CollapsingMergeTree and ReplacingMergeTree engines" + ) qs = copy(self) qs._final = True @@ -554,7 +565,7 @@ def delete(self): """ self._verify_mutation_allowed() conditions = (self._where_q & self._prewhere_q).to_sql(self._model_cls) - sql = 'ALTER TABLE $db.`%s` DELETE WHERE %s' % (self._model_cls.table_name(), conditions) + sql = "ALTER TABLE $db.`%s` DELETE WHERE %s" % (self._model_cls.table_name(), conditions) self._database.raw(sql) return self @@ -564,22 +575,22 @@ def update(self, **kwargs): Keyword arguments specify the field names and expressions to use for the update. Note that ClickHouse performs updates in the background, so they are not immediate. """ - assert kwargs, 'No fields specified for update' + assert kwargs, "No fields specified for update" self._verify_mutation_allowed() - fields = comma_join('`%s` = %s' % (name, arg_to_sql(expr)) for name, expr in kwargs.items()) + fields = comma_join("`%s` = %s" % (name, arg_to_sql(expr)) for name, expr in kwargs.items()) conditions = (self._where_q & self._prewhere_q).to_sql(self._model_cls) - sql = 'ALTER TABLE $db.`%s` UPDATE %s WHERE %s' % (self._model_cls.table_name(), fields, conditions) + sql = "ALTER TABLE $db.`%s` UPDATE %s WHERE %s" % (self._model_cls.table_name(), fields, conditions) self._database.raw(sql) return self def _verify_mutation_allowed(self): - ''' + """ Checks that the queryset's state allows mutations. Raises an AssertionError if not. - ''' - assert not self._limits, 'Mutations are not allowed after slicing the queryset' - assert not self._limit_by, 'Mutations are not allowed after calling limit_by(...)' - assert not self._distinct, 'Mutations are not allowed after calling distinct()' - assert not self._final, 'Mutations are not allowed after calling final()' + """ + assert not self._limits, "Mutations are not allowed after slicing the queryset" + assert not self._limit_by, "Mutations are not allowed after calling limit_by(...)" + assert not self._distinct, "Mutations are not allowed after calling distinct()" + assert not self._final, "Mutations are not allowed after calling final()" def aggregate(self, *args, **kwargs): """ @@ -619,7 +630,7 @@ def __init__(self, base_qs, grouping_fields, calculated_fields): At least one calculated field is required. """ super(AggregateQuerySet, self).__init__(base_qs._model_cls, base_qs._database) - assert calculated_fields, 'No calculated fields specified for aggregation' + assert calculated_fields, "No calculated fields specified for aggregation" self._fields = grouping_fields self._grouping_fields = grouping_fields self._calculated_fields = calculated_fields @@ -636,8 +647,9 @@ def group_by(self, *args): created with. """ for name in args: - assert name in self._fields or name in self._calculated_fields, \ - 'Cannot group by `%s` since it is not included in the query' % name + assert name in self._fields or name in self._calculated_fields, ( + "Cannot group by `%s` since it is not included in the query" % name + ) qs = copy(self) qs._grouping_fields = args return qs @@ -652,22 +664,24 @@ def aggregate(self, *args, **kwargs): """ This method is not supported on `AggregateQuerySet`. """ - raise NotImplementedError('Cannot re-aggregate an AggregateQuerySet') + raise NotImplementedError("Cannot re-aggregate an AggregateQuerySet") def select_fields_as_sql(self): """ Returns the selected fields or expressions as a SQL string. """ - return comma_join([str(f) for f in self._fields] + ['%s AS %s' % (v, k) for k, v in self._calculated_fields.items()]) + return comma_join( + [str(f) for f in self._fields] + ["%s AS %s" % (v, k) for k, v in self._calculated_fields.items()] + ) def __iter__(self): - return self._database.select(self.as_sql()) # using an ad-hoc model + return self._database.select(self.as_sql()) # using an ad-hoc model def count(self): """ Returns the number of rows after aggregation. """ - sql = u'SELECT count() FROM (%s)' % self.as_sql() + sql = "SELECT count() FROM (%s)" % self.as_sql() raw = self._database.raw(sql) return int(raw) if raw else 0 @@ -682,7 +696,7 @@ def with_totals(self): return qs def _verify_mutation_allowed(self): - raise AssertionError('Cannot mutate an AggregateQuerySet') + raise AssertionError("Cannot mutate an AggregateQuerySet") # Expose only relevant classes in import * From cfdf2fd389c2a4383bebee96fc974d8d01b33225 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 22:58:22 +0100 Subject: [PATCH 06/51] Chore: fix linting on database.py --- clickhouse_orm/database.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse_orm/database.py b/clickhouse_orm/database.py index 9419d7c..7d70c6a 100644 --- a/clickhouse_orm/database.py +++ b/clickhouse_orm/database.py @@ -4,7 +4,7 @@ import requests from collections import namedtuple from .models import ModelBase -from .utils import escape, parse_tsv, import_submodules +from .utils import parse_tsv, import_submodules from math import ceil import datetime from string import Template @@ -107,7 +107,7 @@ def __init__(self, db_name, db_url='http://localhost:8123/', self.request_session.auth = (username, password or '') self.log_statements = log_statements self.settings = {} - self.db_exists = False # this is required before running _is_existing_database + self.db_exists = False # this is required before running _is_existing_database self.db_exists = self._is_existing_database() if readonly: if not self.db_exists: @@ -144,7 +144,7 @@ def create_table(self, model_class): ''' if model_class.is_system_model(): raise DatabaseException("You can't create system table") - if getattr(model_class, 'engine') is None: + if model_class.engine is None: raise DatabaseException("%s class must define an engine" % model_class.__name__) self._send(model_class.create_table_sql(self)) From ce68a8f55b014f9a49f952c269e3ee738f6f80ef Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:00:06 +0100 Subject: [PATCH 07/51] Chore: fix linting on engines.py --- clickhouse_orm/engines.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clickhouse_orm/engines.py b/clickhouse_orm/engines.py index 285a848..9b13f82 100644 --- a/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -175,7 +175,7 @@ class Buffer(Engine): Read more [here](https://clickhouse.tech/docs/en/engines/table-engines/special/buffer/). """ - #Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) + # Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) def __init__(self, main_model, num_layers=16, min_time=10, max_time=100, min_rows=10000, max_rows=1000000, min_bytes=10000000, max_bytes=100000000): self.main_model = main_model @@ -191,10 +191,10 @@ def create_table_sql(self, db): # Overriden create_table_sql example: # sql = 'ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000)' sql = 'ENGINE = Buffer(`%s`, `%s`, %d, %d, %d, %d, %d, %d, %d)' % ( - db.db_name, self.main_model.table_name(), self.num_layers, - self.min_time, self.max_time, self.min_rows, - self.max_rows, self.min_bytes, self.max_bytes - ) + db.db_name, self.main_model.table_name(), self.num_layers, + self.min_time, self.max_time, self.min_rows, + self.max_rows, self.min_bytes, self.max_bytes + ) return sql From aab92d88aa1fdad5959edd77e7fc2754c583249d Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:02:01 +0100 Subject: [PATCH 08/51] Chore: fix linting on fields.py --- clickhouse_orm/fields.py | 319 ++++++++++++++++++++------------------- 1 file changed, 167 insertions(+), 152 deletions(-) diff --git a/clickhouse_orm/fields.py b/clickhouse_orm/fields.py index 4f631cc..a890385 100644 --- a/clickhouse_orm/fields.py +++ b/clickhouse_orm/fields.py @@ -1,39 +1,45 @@ from __future__ import unicode_literals + import datetime -import iso8601 -import pytz from calendar import timegm from decimal import Decimal, localcontext -from uuid import UUID +from ipaddress import IPv4Address, IPv6Address from logging import getLogger +from uuid import UUID + +import iso8601 +import pytz from pytz import BaseTzInfo -from .utils import escape, parse_array, comma_join, string_or_func, get_subclass_names + from .funcs import F, FunctionOperatorsMixin -from ipaddress import IPv4Address, IPv6Address +from .utils import comma_join, escape, get_subclass_names, parse_array, string_or_func -logger = getLogger('clickhouse_orm') +logger = getLogger("clickhouse_orm") class Field(FunctionOperatorsMixin): - ''' + """ Abstract base class for all field types. - ''' - name = None # this is set by the parent model - parent = None # this is set by the parent model - creation_counter = 0 # used for keeping the model fields ordered - class_default = 0 # should be overridden by concrete subclasses - db_type = None # should be overridden by concrete subclasses + """ + + name = None # this is set by the parent model + parent = None # this is set by the parent model + creation_counter = 0 # used for keeping the model fields ordered + class_default = 0 # should be overridden by concrete subclasses + db_type = None # should be overridden by concrete subclasses def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None): - assert [default, alias, materialized].count(None) >= 2, \ - "Only one of default, alias and materialized parameters can be given" - assert alias is None or isinstance(alias, F) or isinstance(alias, str) and alias != "",\ - "Alias parameter must be a string or function object, if given" - assert materialized is None or isinstance(materialized, F) or isinstance(materialized, str) and materialized != "",\ - "Materialized parameter must be a string or function object, if given" + assert [default, alias, materialized].count( + None + ) >= 2, "Only one of default, alias and materialized parameters can be given" + assert ( + alias is None or isinstance(alias, F) or isinstance(alias, str) and alias != "" + ), "Alias parameter must be a string or function object, if given" + assert ( + materialized is None or isinstance(materialized, F) or isinstance(materialized, str) and materialized != "" + ), "Materialized parameter must be a string or function object, if given" assert readonly is None or type(readonly) is bool, "readonly parameter must be bool if given" - assert codec is None or isinstance(codec, str) and codec != "", \ - "Codec field must be string, if given" + assert codec is None or isinstance(codec, str) and codec != "", "Codec field must be string, if given" self.creation_counter = Field.creation_counter Field.creation_counter += 1 @@ -47,49 +53,51 @@ def __str__(self): return self.name def __repr__(self): - return '<%s>' % self.__class__.__name__ + return "<%s>" % self.__class__.__name__ def to_python(self, value, timezone_in_use): - ''' + """ Converts the input value into the expected Python data type, raising ValueError if the data can't be converted. Returns the converted value. Subclasses should override this. The timezone_in_use parameter should be consulted when parsing datetime fields. - ''' - return value # pragma: no cover + """ + return value # pragma: no cover def validate(self, value): - ''' + """ Called after to_python to validate that the value is suitable for the field's database type. Subclasses should override this. - ''' + """ pass def _range_check(self, value, min_value, max_value): - ''' + """ Utility method to check that the given value is between min_value and max_value. - ''' + """ if value < min_value or value > max_value: - raise ValueError('%s out of range - %s is not between %s and %s' % (self.__class__.__name__, value, min_value, max_value)) + raise ValueError( + "%s out of range - %s is not between %s and %s" % (self.__class__.__name__, value, min_value, max_value) + ) def to_db_string(self, value, quote=True): - ''' + """ Returns the field's value prepared for writing to the database. When quote is true, strings are surrounded by single quotes. - ''' + """ return escape(value, quote) def get_sql(self, with_default_expression=True, db=None): - ''' + """ Returns an SQL expression describing the field (e.g. for CREATE TABLE). - `with_default_expression`: If True, adds default value to sql. It doesn't affect fields with alias and materialized values. - `db`: Database, used for checking supported features. - ''' + """ sql = self.db_type args = self.get_db_type_args() if args: - sql += '(%s)' % comma_join(args) + sql += "(%s)" % comma_join(args) if with_default_expression: sql += self._extra_params(db) return sql @@ -99,18 +107,18 @@ def get_db_type_args(self): return [] def _extra_params(self, db): - sql = '' + sql = "" if self.alias: - sql += ' ALIAS %s' % string_or_func(self.alias) + sql += " ALIAS %s" % string_or_func(self.alias) elif self.materialized: - sql += ' MATERIALIZED %s' % string_or_func(self.materialized) + sql += " MATERIALIZED %s" % string_or_func(self.materialized) elif isinstance(self.default, F): - sql += ' DEFAULT %s' % self.default.to_sql() + sql += " DEFAULT %s" % self.default.to_sql() elif self.default: default = self.to_db_string(self.default) - sql += ' DEFAULT %s' % default + sql += " DEFAULT %s" % default if self.codec and db and db.has_codec_support: - sql += ' CODEC(%s)' % self.codec + sql += " CODEC(%s)" % self.codec return sql def isinstance(self, types): @@ -124,43 +132,42 @@ def isinstance(self, types): """ if isinstance(self, types): return True - inner_field = getattr(self, 'inner_field', None) + inner_field = getattr(self, "inner_field", None) while inner_field: if isinstance(inner_field, types): return True - inner_field = getattr(inner_field, 'inner_field', None) + inner_field = getattr(inner_field, "inner_field", None) return False class StringField(Field): - class_default = '' - db_type = 'String' + class_default = "" + db_type = "String" def to_python(self, value, timezone_in_use): if isinstance(value, str): return value if isinstance(value, bytes): - return value.decode('UTF-8') - raise ValueError('Invalid value for %s: %r' % (self.__class__.__name__, value)) + return value.decode("UTF-8") + raise ValueError("Invalid value for %s: %r" % (self.__class__.__name__, value)) class FixedStringField(StringField): - def __init__(self, length, default=None, alias=None, materialized=None, readonly=None): self._length = length - self.db_type = 'FixedString(%d)' % length + self.db_type = "FixedString(%d)" % length super(FixedStringField, self).__init__(default, alias, materialized, readonly) def to_python(self, value, timezone_in_use): value = super(FixedStringField, self).to_python(value, timezone_in_use) - return value.rstrip('\0') + return value.rstrip("\0") def validate(self, value): if isinstance(value, str): - value = value.encode('UTF-8') + value = value.encode("UTF-8") if len(value) > self._length: - raise ValueError('Value of %d bytes is too long for FixedStringField(%d)' % (len(value), self._length)) + raise ValueError("Value of %d bytes is too long for FixedStringField(%d)" % (len(value), self._length)) class DateField(Field): @@ -168,7 +175,7 @@ class DateField(Field): min_value = datetime.date(1970, 1, 1) max_value = datetime.date(2105, 12, 31) class_default = min_value - db_type = 'Date' + db_type = "Date" def to_python(self, value, timezone_in_use): if isinstance(value, datetime.datetime): @@ -178,10 +185,10 @@ def to_python(self, value, timezone_in_use): if isinstance(value, int): return DateField.class_default + datetime.timedelta(days=value) if isinstance(value, str): - if value == '0000-00-00': + if value == "0000-00-00": return DateField.min_value - return datetime.datetime.strptime(value, '%Y-%m-%d').date() - raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + return datetime.datetime.strptime(value, "%Y-%m-%d").date() + raise ValueError("Invalid value for %s - %r" % (self.__class__.__name__, value)) def validate(self, value): self._range_check(value, DateField.min_value, DateField.max_value) @@ -193,10 +200,9 @@ def to_db_string(self, value, quote=True): class DateTimeField(Field): class_default = datetime.datetime.fromtimestamp(0, pytz.utc) - db_type = 'DateTime' + db_type = "DateTime" - def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None, - timezone=None): + def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None, timezone=None): super().__init__(default, alias, materialized, readonly, codec) # assert not timezone, 'Temporarily field timezone is not supported' if timezone: @@ -217,7 +223,7 @@ def to_python(self, value, timezone_in_use): if isinstance(value, int): return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc) if isinstance(value, str): - if value == '0000-00-00 00:00:00': + if value == "0000-00-00 00:00:00": return self.class_default if len(value) == 10: try: @@ -235,19 +241,20 @@ def to_python(self, value, timezone_in_use): if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None: dt = timezone_in_use.localize(dt) return dt - raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + raise ValueError("Invalid value for %s - %r" % (self.__class__.__name__, value)) def to_db_string(self, value, quote=True): - return escape('%010d' % timegm(value.utctimetuple()), quote) + return escape("%010d" % timegm(value.utctimetuple()), quote) class DateTime64Field(DateTimeField): - db_type = 'DateTime64' + db_type = "DateTime64" - def __init__(self, default=None, alias=None, materialized=None, readonly=None, codec=None, - timezone=None, precision=6): + def __init__( + self, default=None, alias=None, materialized=None, readonly=None, codec=None, timezone=None, precision=6 + ): super().__init__(default, alias, materialized, readonly, codec, timezone) - assert precision is None or isinstance(precision, int), 'Precision must be int type' + assert precision is None or isinstance(precision, int), "Precision must be int type" self.precision = precision def get_db_type_args(self): @@ -263,11 +270,10 @@ def to_db_string(self, value, quote=True): Returns string in 0000000000.000000 format, where remainder digits count is equal to precision """ return escape( - '{timestamp:0{width}.{precision}f}'.format( - timestamp=value.timestamp(), - width=11 + self.precision, - precision=self.precision), - quote + "{timestamp:0{width}.{precision}f}".format( + timestamp=value.timestamp(), width=11 + self.precision, precision=self.precision + ), + quote, ) def to_python(self, value, timezone_in_use): @@ -277,8 +283,8 @@ def to_python(self, value, timezone_in_use): if isinstance(value, (int, float)): return datetime.datetime.utcfromtimestamp(value).replace(tzinfo=pytz.utc) if isinstance(value, str): - left_part = value.split('.')[0] - if left_part == '0000-00-00 00:00:00': + left_part = value.split(".")[0] + if left_part == "0000-00-00 00:00:00": return self.class_default if len(left_part) == 10: try: @@ -290,14 +296,15 @@ def to_python(self, value, timezone_in_use): class BaseIntField(Field): - ''' + """ Abstract base class for all integer-type fields. - ''' + """ + def to_python(self, value, timezone_in_use): try: return int(value) - except: - raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + except Exception: + raise ValueError("Invalid value for %s - %r" % (self.__class__.__name__, value)) def to_db_string(self, value, quote=True): # There's no need to call escape since numbers do not contain @@ -311,69 +318,69 @@ def validate(self, value): class UInt8Field(BaseIntField): min_value = 0 - max_value = 2**8 - 1 - db_type = 'UInt8' + max_value = 2 ** 8 - 1 + db_type = "UInt8" class UInt16Field(BaseIntField): min_value = 0 - max_value = 2**16 - 1 - db_type = 'UInt16' + max_value = 2 ** 16 - 1 + db_type = "UInt16" class UInt32Field(BaseIntField): min_value = 0 - max_value = 2**32 - 1 - db_type = 'UInt32' + max_value = 2 ** 32 - 1 + db_type = "UInt32" class UInt64Field(BaseIntField): min_value = 0 - max_value = 2**64 - 1 - db_type = 'UInt64' + max_value = 2 ** 64 - 1 + db_type = "UInt64" class Int8Field(BaseIntField): - min_value = -2**7 - max_value = 2**7 - 1 - db_type = 'Int8' + min_value = -(2 ** 7) + max_value = 2 ** 7 - 1 + db_type = "Int8" class Int16Field(BaseIntField): - min_value = -2**15 - max_value = 2**15 - 1 - db_type = 'Int16' + min_value = -(2 ** 15) + max_value = 2 ** 15 - 1 + db_type = "Int16" class Int32Field(BaseIntField): - min_value = -2**31 - max_value = 2**31 - 1 - db_type = 'Int32' + min_value = -(2 ** 31) + max_value = 2 ** 31 - 1 + db_type = "Int32" class Int64Field(BaseIntField): - min_value = -2**63 - max_value = 2**63 - 1 - db_type = 'Int64' + min_value = -(2 ** 63) + max_value = 2 ** 63 - 1 + db_type = "Int64" class BaseFloatField(Field): - ''' + """ Abstract base class for all float-type fields. - ''' + """ def to_python(self, value, timezone_in_use): try: return float(value) - except: - raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + except Exception: + raise ValueError("Invalid value for %s - %r" % (self.__class__.__name__, value)) def to_db_string(self, value, quote=True): # There's no need to call escape since numbers do not contain @@ -383,28 +390,28 @@ def to_db_string(self, value, quote=True): class Float32Field(BaseFloatField): - db_type = 'Float32' + db_type = "Float32" class Float64Field(BaseFloatField): - db_type = 'Float64' + db_type = "Float64" class DecimalField(Field): - ''' + """ Base class for all decimal fields. Can also be used directly. - ''' + """ def __init__(self, precision, scale, default=None, alias=None, materialized=None, readonly=None): - assert 1 <= precision <= 38, 'Precision must be between 1 and 38' - assert 0 <= scale <= precision, 'Scale must be between 0 and the given precision' + assert 1 <= precision <= 38, "Precision must be between 1 and 38" + assert 0 <= scale <= precision, "Scale must be between 0 and the given precision" self.precision = precision self.scale = scale - self.db_type = 'Decimal(%d,%d)' % (self.precision, self.scale) + self.db_type = "Decimal(%d,%d)" % (self.precision, self.scale) with localcontext() as ctx: ctx.prec = 38 - self.exp = Decimal(10) ** -self.scale # for rounding to the required scale + self.exp = Decimal(10) ** -self.scale # for rounding to the required scale self.max_value = Decimal(10 ** (self.precision - self.scale)) - self.exp self.min_value = -self.max_value super(DecimalField, self).__init__(default, alias, materialized, readonly) @@ -413,10 +420,10 @@ def to_python(self, value, timezone_in_use): if not isinstance(value, Decimal): try: value = Decimal(value) - except: - raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value)) + except Exception: + raise ValueError("Invalid value for %s - %r" % (self.__class__.__name__, value)) if not value.is_finite(): - raise ValueError('Non-finite value for %s - %r' % (self.__class__.__name__, value)) + raise ValueError("Non-finite value for %s - %r" % (self.__class__.__name__, value)) return self._round(value) def to_db_string(self, value, quote=True): @@ -432,30 +439,27 @@ def validate(self, value): class Decimal32Field(DecimalField): - def __init__(self, scale, default=None, alias=None, materialized=None, readonly=None): super(Decimal32Field, self).__init__(9, scale, default, alias, materialized, readonly) - self.db_type = 'Decimal32(%d)' % scale + self.db_type = "Decimal32(%d)" % scale class Decimal64Field(DecimalField): - def __init__(self, scale, default=None, alias=None, materialized=None, readonly=None): super(Decimal64Field, self).__init__(18, scale, default, alias, materialized, readonly) - self.db_type = 'Decimal64(%d)' % scale + self.db_type = "Decimal64(%d)" % scale class Decimal128Field(DecimalField): - def __init__(self, scale, default=None, alias=None, materialized=None, readonly=None): super(Decimal128Field, self).__init__(38, scale, default, alias, materialized, readonly) - self.db_type = 'Decimal128(%d)' % scale + self.db_type = "Decimal128(%d)" % scale class BaseEnumField(Field): - ''' + """ Abstract base class for all enum-type fields. - ''' + """ def __init__(self, enum_cls, default=None, alias=None, materialized=None, readonly=None, codec=None): self.enum_cls = enum_cls @@ -473,7 +477,7 @@ def to_python(self, value, timezone_in_use): except Exception: return self.enum_cls(value) if isinstance(value, bytes): - decoded = value.decode('UTF-8') + decoded = value.decode("UTF-8") try: return self.enum_cls[decoded] except Exception: @@ -482,38 +486,39 @@ def to_python(self, value, timezone_in_use): return self.enum_cls(value) except (KeyError, ValueError): pass - raise ValueError('Invalid value for %s: %r' % (self.enum_cls.__name__, value)) + raise ValueError("Invalid value for %s: %r" % (self.enum_cls.__name__, value)) def to_db_string(self, value, quote=True): return escape(value.name, quote) def get_db_type_args(self): - return ['%s = %d' % (escape(item.name), item.value) for item in self.enum_cls] + return ["%s = %d" % (escape(item.name), item.value) for item in self.enum_cls] @classmethod def create_ad_hoc_field(cls, db_type): - ''' + """ Give an SQL column description such as "Enum8('apple' = 1, 'banana' = 2, 'orange' = 3)" this method returns a matching enum field. - ''' + """ import re from enum import Enum + members = {} for match in re.finditer(r"'([\w ]+)' = (-?\d+)", db_type): members[match.group(1)] = int(match.group(2)) - enum_cls = Enum('AdHocEnum', members) - field_class = Enum8Field if db_type.startswith('Enum8') else Enum16Field + enum_cls = Enum("AdHocEnum", members) + field_class = Enum8Field if db_type.startswith("Enum8") else Enum16Field return field_class(enum_cls) class Enum8Field(BaseEnumField): - db_type = 'Enum8' + db_type = "Enum8" class Enum16Field(BaseEnumField): - db_type = 'Enum16' + db_type = "Enum16" class ArrayField(Field): @@ -530,9 +535,9 @@ def to_python(self, value, timezone_in_use): if isinstance(value, str): value = parse_array(value) elif isinstance(value, bytes): - value = parse_array(value.decode('UTF-8')) + value = parse_array(value.decode("UTF-8")) elif not isinstance(value, (list, tuple)): - raise ValueError('ArrayField expects list or tuple, not %s' % type(value)) + raise ValueError("ArrayField expects list or tuple, not %s" % type(value)) return [self.inner_field.to_python(v, timezone_in_use) for v in value] def validate(self, value): @@ -541,19 +546,19 @@ def validate(self, value): def to_db_string(self, value, quote=True): array = [self.inner_field.to_db_string(v, quote=True) for v in value] - return '[' + comma_join(array) + ']' + return "[" + comma_join(array) + "]" def get_sql(self, with_default_expression=True, db=None): - sql = 'Array(%s)' % self.inner_field.get_sql(with_default_expression=False, db=db) + sql = "Array(%s)" % self.inner_field.get_sql(with_default_expression=False, db=db) if with_default_expression and self.codec and db and db.has_codec_support: - sql+= ' CODEC(%s)' % self.codec + sql += " CODEC(%s)" % self.codec return sql class UUIDField(Field): class_default = UUID(int=0) - db_type = 'UUID' + db_type = "UUID" def to_python(self, value, timezone_in_use): if isinstance(value, UUID): @@ -567,7 +572,7 @@ def to_python(self, value, timezone_in_use): elif isinstance(value, tuple): return UUID(fields=value) else: - raise ValueError('Invalid value for UUIDField: %r' % value) + raise ValueError("Invalid value for UUIDField: %r" % value) def to_db_string(self, value, quote=True): return escape(str(value), quote) @@ -576,7 +581,7 @@ def to_db_string(self, value, quote=True): class IPv4Field(Field): class_default = 0 - db_type = 'IPv4' + db_type = "IPv4" def to_python(self, value, timezone_in_use): if isinstance(value, IPv4Address): @@ -584,7 +589,7 @@ def to_python(self, value, timezone_in_use): elif isinstance(value, (bytes, str, int)): return IPv4Address(value) else: - raise ValueError('Invalid value for IPv4Address: %r' % value) + raise ValueError("Invalid value for IPv4Address: %r" % value) def to_db_string(self, value, quote=True): return escape(str(value), quote) @@ -593,7 +598,7 @@ def to_db_string(self, value, quote=True): class IPv6Field(Field): class_default = 0 - db_type = 'IPv6' + db_type = "IPv6" def to_python(self, value, timezone_in_use): if isinstance(value, IPv6Address): @@ -601,7 +606,7 @@ def to_python(self, value, timezone_in_use): elif isinstance(value, (bytes, str, int)): return IPv6Address(value) else: - raise ValueError('Invalid value for IPv6Address: %r' % value) + raise ValueError("Invalid value for IPv6Address: %r" % value) def to_db_string(self, value, quote=True): return escape(str(value), quote) @@ -611,9 +616,10 @@ class NullableField(Field): class_default = None - def __init__(self, inner_field, default=None, alias=None, materialized=None, - extra_null_values=None, codec=None): - assert isinstance(inner_field, Field), "The first argument of NullableField must be a Field instance. Not: {}".format(inner_field) + def __init__(self, inner_field, default=None, alias=None, materialized=None, extra_null_values=None, codec=None): + assert isinstance( + inner_field, Field + ), "The first argument of NullableField must be a Field instance. Not: {}".format(inner_field) self.inner_field = inner_field self._null_values = [None] if extra_null_values: @@ -621,7 +627,7 @@ def __init__(self, inner_field, default=None, alias=None, materialized=None, super(NullableField, self).__init__(default, alias, materialized, readonly=None, codec=codec) def to_python(self, value, timezone_in_use): - if value == '\\N' or value in self._null_values: + if value == "\\N" or value in self._null_values: return None return self.inner_field.to_python(value, timezone_in_use) @@ -630,22 +636,27 @@ def validate(self, value): def to_db_string(self, value, quote=True): if value in self._null_values: - return '\\N' + return "\\N" return self.inner_field.to_db_string(value, quote=quote) def get_sql(self, with_default_expression=True, db=None): - sql = 'Nullable(%s)' % self.inner_field.get_sql(with_default_expression=False, db=db) + sql = "Nullable(%s)" % self.inner_field.get_sql(with_default_expression=False, db=db) if with_default_expression: sql += self._extra_params(db) return sql class LowCardinalityField(Field): - def __init__(self, inner_field, default=None, alias=None, materialized=None, readonly=None, codec=None): - assert isinstance(inner_field, Field), "The first argument of LowCardinalityField must be a Field instance. Not: {}".format(inner_field) - assert not isinstance(inner_field, LowCardinalityField), "LowCardinality inner fields are not supported by the ORM" - assert not isinstance(inner_field, ArrayField), "Array field inside LowCardinality are not supported by the ORM. Use Array(LowCardinality) instead" + assert isinstance( + inner_field, Field + ), "The first argument of LowCardinalityField must be a Field instance. Not: {}".format(inner_field) + assert not isinstance( + inner_field, LowCardinalityField + ), "LowCardinality inner fields are not supported by the ORM" + assert not isinstance( + inner_field, ArrayField + ), "Array field inside LowCardinality are not supported by the ORM. Use Array(LowCardinality) instead" self.inner_field = inner_field self.class_default = self.inner_field.class_default super(LowCardinalityField, self).__init__(default, alias, materialized, readonly, codec) @@ -661,10 +672,14 @@ def to_db_string(self, value, quote=True): def get_sql(self, with_default_expression=True, db=None): if db and db.has_low_cardinality_support: - sql = 'LowCardinality(%s)' % self.inner_field.get_sql(with_default_expression=False) + sql = "LowCardinality(%s)" % self.inner_field.get_sql(with_default_expression=False) else: sql = self.inner_field.get_sql(with_default_expression=False) - logger.warning('LowCardinalityField not supported on clickhouse-server version < 19.0 using {} as fallback'.format(self.inner_field.__class__.__name__)) + logger.warning( + "LowCardinalityField not supported on clickhouse-server version < 19.0 using {} as fallback".format( + self.inner_field.__class__.__name__ + ) + ) if with_default_expression: sql += self._extra_params(db) return sql From 8fd0f2a42246fb6be9772400b74303b1d6e29065 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:04:03 +0100 Subject: [PATCH 09/51] Chore: fix linting on funcs.py --- clickhouse_orm/funcs.py | 805 ++++++++++++++++++++-------------------- 1 file changed, 410 insertions(+), 395 deletions(-) diff --git a/clickhouse_orm/funcs.py b/clickhouse_orm/funcs.py index d84c761..192b0b3 100644 --- a/clickhouse_orm/funcs.py +++ b/clickhouse_orm/funcs.py @@ -10,11 +10,13 @@ def binary_operator(func): """ Decorates a function to mark it as a binary operator. """ + @wraps(func) def wrapper(*args, **kwargs): ret = func(*args, **kwargs) ret.is_binary_operator = True return ret + return wrapper @@ -24,10 +26,12 @@ def type_conversion(func): The metaclass automatically generates "OrZero" and "OrNull" combinators for the decorated function. """ + @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) - wrapper.f_type = 'type_conversion' + + wrapper.f_type = "type_conversion" return wrapper @@ -37,10 +41,12 @@ def aggregate(func): The metaclass automatically generates combinators such as "OrDefault", "OrNull", "If" etc. for the decorated function. """ + @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) - wrapper.f_type = 'aggregate' + + wrapper.f_type = "aggregate" return wrapper @@ -49,10 +55,12 @@ def with_utf8_support(func): Decorates a function to mark it as a string function that has a UTF8 variant. The metaclass automatically generates a "UTF8" combinator for the decorated function. """ + @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) - wrapper.f_type = 'with_utf8_support' + + wrapper.f_type = "with_utf8_support" return wrapper @@ -61,6 +69,7 @@ def parametric(func): Decorates a function to convert it to a parametric function, such as `quantile(level)(expr)`. """ + @wraps(func) def wrapper(*parameters): @wraps(func) @@ -68,9 +77,11 @@ def inner(*args, **kwargs): f = func(*args, **kwargs) # Append the parameter to the function name parameters_str = comma_join(parameters, stringify=True) - f.name = '%s(%s)' % (f.name, parameters_str) + f.name = "%s(%s)" % (f.name, parameters_str) return f + return inner + wrapper.f_parametric = True return wrapper @@ -177,29 +188,29 @@ def isNotIn(self, others): class FMeta(type): FUNCTION_COMBINATORS = { - 'type_conversion': [ - {'suffix': 'OrZero'}, - {'suffix': 'OrNull'}, + "type_conversion": [ + {"suffix": "OrZero"}, + {"suffix": "OrNull"}, + ], + "aggregate": [ + {"suffix": "OrDefault"}, + {"suffix": "OrNull"}, + {"suffix": "If", "args": ["cond"]}, + {"suffix": "OrDefaultIf", "args": ["cond"]}, + {"suffix": "OrNullIf", "args": ["cond"]}, ], - 'aggregate': [ - {'suffix': 'OrDefault'}, - {'suffix': 'OrNull'}, - {'suffix': 'If', 'args': ['cond']}, - {'suffix': 'OrDefaultIf', 'args': ['cond']}, - {'suffix': 'OrNullIf', 'args': ['cond']}, + "with_utf8_support": [ + {"suffix": "UTF8"}, ], - 'with_utf8_support': [ - {'suffix': 'UTF8'}, - ] } def __init__(cls, name, bases, dct): for name, obj in dct.items(): - if hasattr(obj, '__func__'): - f_type = getattr(obj.__func__, 'f_type', '') + if hasattr(obj, "__func__"): + f_type = getattr(obj.__func__, "f_type", "") for combinator in FMeta.FUNCTION_COMBINATORS.get(f_type, []): - new_name = name + combinator['suffix'] - FMeta._add_func(cls, obj.__func__, new_name, combinator.get('args')) + new_name = name + combinator["suffix"] + FMeta._add_func(cls, obj.__func__, new_name, combinator.get("args")) @staticmethod def _add_func(cls, base_func, new_name, extra_args): @@ -208,7 +219,7 @@ def _add_func(cls, base_func, new_name, extra_args): """ # Get the function's signature sig = signature(base_func) - new_sig = str(sig)[1 : -1] # omit the parentheses + new_sig = str(sig)[1:-1] # omit the parentheses args = comma_join(sig.parameters) # Add extra args if extra_args: @@ -221,11 +232,12 @@ def _add_func(cls, base_func, new_name, extra_args): # Get default values for args argdefs = tuple(p.default for p in sig.parameters.values() if p.default != Parameter.empty) # Build the new function - new_code = compile('def {new_name}({new_sig}): return F("{new_name}", {args})'.format(**locals()), - __file__, 'exec') + new_code = compile( + 'def {new_name}({new_sig}): return F("{new_name}", {args})'.format(**locals()), __file__, "exec" + ) new_func = FunctionType(code=new_code.co_consts[0], globals=globals(), name=new_name, argdefs=argdefs) # If base_func was parametric, new_func should be too - if getattr(base_func, 'f_parametric', False): + if getattr(base_func, "f_parametric", False): new_func = parametric(new_func) # Attach to class setattr(cls, new_name, new_func) @@ -236,6 +248,7 @@ class F(Cond, FunctionOperatorsMixin, metaclass=FMeta): Represents a database function call and its arguments. It doubles as a query condition when the function returns a boolean result. """ + def __init__(self, name, *args): """ Initializer. @@ -257,116 +270,116 @@ def to_sql(self, *args): gcd(12, 300) """ if self.is_binary_operator: - prefix = '' - sep = ' ' + self.name + ' ' + prefix = "" + sep = " " + self.name + " " else: prefix = self.name - sep = ', ' + sep = ", " arg_strs = (arg_to_sql(arg) for arg in self.args if arg != NO_VALUE) - return prefix + '(' + sep.join(arg_strs) + ')' + return prefix + "(" + sep.join(arg_strs) + ")" # Arithmetic functions @staticmethod @binary_operator def plus(a, b): - return F('+', a, b) + return F("+", a, b) @staticmethod @binary_operator def minus(a, b): - return F('-', a, b) + return F("-", a, b) @staticmethod @binary_operator def multiply(a, b): - return F('*', a, b) + return F("*", a, b) @staticmethod @binary_operator def divide(a, b): - return F('/', a, b) + return F("/", a, b) @staticmethod def intDiv(a, b): - return F('intDiv', a, b) + return F("intDiv", a, b) @staticmethod def intDivOrZero(a, b): - return F('intDivOrZero', a, b) + return F("intDivOrZero", a, b) @staticmethod @binary_operator def modulo(a, b): - return F('%', a, b) + return F("%", a, b) @staticmethod def negate(a): - return F('negate', a) + return F("negate", a) @staticmethod def abs(a): - return F('abs', a) + return F("abs", a) @staticmethod def gcd(a, b): - return F('gcd', a, b) + return F("gcd", a, b) @staticmethod def lcm(a, b): - return F('lcm', a, b) + return F("lcm", a, b) # Comparison functions @staticmethod @binary_operator def equals(a, b): - return F('=', a, b) + return F("=", a, b) @staticmethod @binary_operator def notEquals(a, b): - return F('!=', a, b) + return F("!=", a, b) @staticmethod @binary_operator def less(a, b): - return F('<', a, b) + return F("<", a, b) @staticmethod @binary_operator def greater(a, b): - return F('>', a, b) + return F(">", a, b) @staticmethod @binary_operator def lessOrEquals(a, b): - return F('<=', a, b) + return F("<=", a, b) @staticmethod @binary_operator def greaterOrEquals(a, b): - return F('>=', a, b) + return F(">=", a, b) # Logical functions (should be used as python operators: & | ^ ~) @staticmethod @binary_operator def _and(a, b): - return F('AND', a, b) + return F("AND", a, b) @staticmethod @binary_operator def _or(a, b): - return F('OR', a, b) + return F("OR", a, b) @staticmethod def _xor(a, b): - return F('xor', a, b) + return F("xor", a, b) @staticmethod def _not(a): - return F('not', a) + return F("not", a) # in / not in @@ -375,1443 +388,1445 @@ def _not(a): def _in(a, b): if is_iterable(b) and not isinstance(b, (tuple, QuerySet)): b = tuple(b) - return F('IN', a, b) + return F("IN", a, b) @staticmethod @binary_operator def _notIn(a, b): if is_iterable(b) and not isinstance(b, (tuple, QuerySet)): b = tuple(b) - return F('NOT IN', a, b) + return F("NOT IN", a, b) # Functions for working with dates and times @staticmethod def toYear(d): - return F('toYear', d) + return F("toYear", d) @staticmethod - def toISOYear(d, timezone=''): - return F('toISOYear', d, timezone) + def toISOYear(d, timezone=""): + return F("toISOYear", d, timezone) @staticmethod - def toQuarter(d, timezone=''): - return F('toQuarter', d, timezone) if timezone else F('toQuarter', d) + def toQuarter(d, timezone=""): + return F("toQuarter", d, timezone) if timezone else F("toQuarter", d) @staticmethod def toMonth(d): - return F('toMonth', d) + return F("toMonth", d) @staticmethod - def toWeek(d, mode=0, timezone=''): - return F('toWeek', d, mode, timezone) + def toWeek(d, mode=0, timezone=""): + return F("toWeek", d, mode, timezone) @staticmethod - def toISOWeek(d, timezone=''): - return F('toISOWeek', d, timezone) if timezone else F('toISOWeek', d) + def toISOWeek(d, timezone=""): + return F("toISOWeek", d, timezone) if timezone else F("toISOWeek", d) @staticmethod def toDayOfYear(d): - return F('toDayOfYear', d) + return F("toDayOfYear", d) @staticmethod def toDayOfMonth(d): - return F('toDayOfMonth', d) + return F("toDayOfMonth", d) @staticmethod def toDayOfWeek(d): - return F('toDayOfWeek', d) + return F("toDayOfWeek", d) @staticmethod def toHour(d): - return F('toHour', d) + return F("toHour", d) @staticmethod def toMinute(d): - return F('toMinute', d) + return F("toMinute", d) @staticmethod def toSecond(d): - return F('toSecond', d) + return F("toSecond", d) @staticmethod def toMonday(d): - return F('toMonday', d) + return F("toMonday", d) @staticmethod def toStartOfMonth(d): - return F('toStartOfMonth', d) + return F("toStartOfMonth", d) @staticmethod def toStartOfQuarter(d): - return F('toStartOfQuarter', d) + return F("toStartOfQuarter", d) @staticmethod def toStartOfYear(d): - return F('toStartOfYear', d) + return F("toStartOfYear", d) @staticmethod def toStartOfISOYear(d): - return F('toStartOfISOYear', d) + return F("toStartOfISOYear", d) @staticmethod def toStartOfTenMinutes(d): - return F('toStartOfTenMinutes', d) + return F("toStartOfTenMinutes", d) @staticmethod def toStartOfWeek(d, mode=0): - return F('toStartOfWeek', d) + return F("toStartOfWeek", d) @staticmethod def toStartOfMinute(d): - return F('toStartOfMinute', d) + return F("toStartOfMinute", d) @staticmethod def toStartOfFiveMinute(d): - return F('toStartOfFiveMinute', d) + return F("toStartOfFiveMinute", d) @staticmethod def toStartOfFifteenMinutes(d): - return F('toStartOfFifteenMinutes', d) + return F("toStartOfFifteenMinutes", d) @staticmethod def toStartOfHour(d): - return F('toStartOfHour', d) + return F("toStartOfHour", d) @staticmethod def toStartOfDay(d): - return F('toStartOfDay', d) + return F("toStartOfDay", d) @staticmethod - def toTime(d, timezone=''): - return F('toTime', d, timezone) + def toTime(d, timezone=""): + return F("toTime", d, timezone) @staticmethod def toTimeZone(dt, timezone): - return F('toTimeZone', dt, timezone) + return F("toTimeZone", dt, timezone) @staticmethod - def toUnixTimestamp(dt, timezone=''): - return F('toUnixTimestamp', dt, timezone) + def toUnixTimestamp(dt, timezone=""): + return F("toUnixTimestamp", dt, timezone) @staticmethod - def toYYYYMM(dt, timezone=''): - return F('toYYYYMM', dt, timezone) if timezone else F('toYYYYMM', dt) + def toYYYYMM(dt, timezone=""): + return F("toYYYYMM", dt, timezone) if timezone else F("toYYYYMM", dt) @staticmethod - def toYYYYMMDD(dt, timezone=''): - return F('toYYYYMMDD', dt, timezone) if timezone else F('toYYYYMMDD', dt) + def toYYYYMMDD(dt, timezone=""): + return F("toYYYYMMDD", dt, timezone) if timezone else F("toYYYYMMDD", dt) @staticmethod - def toYYYYMMDDhhmmss(dt, timezone=''): - return F('toYYYYMMDDhhmmss', dt, timezone) if timezone else F('toYYYYMMDDhhmmss', dt) + def toYYYYMMDDhhmmss(dt, timezone=""): + return F("toYYYYMMDDhhmmss", dt, timezone) if timezone else F("toYYYYMMDDhhmmss", dt) @staticmethod - def toRelativeYearNum(d, timezone=''): - return F('toRelativeYearNum', d, timezone) + def toRelativeYearNum(d, timezone=""): + return F("toRelativeYearNum", d, timezone) @staticmethod - def toRelativeMonthNum(d, timezone=''): - return F('toRelativeMonthNum', d, timezone) + def toRelativeMonthNum(d, timezone=""): + return F("toRelativeMonthNum", d, timezone) @staticmethod - def toRelativeWeekNum(d, timezone=''): - return F('toRelativeWeekNum', d, timezone) + def toRelativeWeekNum(d, timezone=""): + return F("toRelativeWeekNum", d, timezone) @staticmethod - def toRelativeDayNum(d, timezone=''): - return F('toRelativeDayNum', d, timezone) + def toRelativeDayNum(d, timezone=""): + return F("toRelativeDayNum", d, timezone) @staticmethod - def toRelativeHourNum(d, timezone=''): - return F('toRelativeHourNum', d, timezone) + def toRelativeHourNum(d, timezone=""): + return F("toRelativeHourNum", d, timezone) @staticmethod - def toRelativeMinuteNum(d, timezone=''): - return F('toRelativeMinuteNum', d, timezone) + def toRelativeMinuteNum(d, timezone=""): + return F("toRelativeMinuteNum", d, timezone) @staticmethod - def toRelativeSecondNum(d, timezone=''): - return F('toRelativeSecondNum', d, timezone) + def toRelativeSecondNum(d, timezone=""): + return F("toRelativeSecondNum", d, timezone) @staticmethod def now(): - return F('now') + return F("now") @staticmethod def today(): - return F('today') + return F("today") @staticmethod def yesterday(): - return F('yesterday') + return F("yesterday") @staticmethod def timeSlot(d): - return F('timeSlot', d) + return F("timeSlot", d) @staticmethod def timeSlots(start_time, duration): - return F('timeSlots', start_time, F.toUInt32(duration)) + return F("timeSlots", start_time, F.toUInt32(duration)) @staticmethod - def formatDateTime(d, format, timezone=''): - return F('formatDateTime', d, format, timezone) + def formatDateTime(d, format, timezone=""): + return F("formatDateTime", d, format, timezone) @staticmethod def addDays(d, n, timezone=NO_VALUE): - return F('addDays', d, n, timezone) + return F("addDays", d, n, timezone) @staticmethod def addHours(d, n, timezone=NO_VALUE): - return F('addHours', d, n, timezone) + return F("addHours", d, n, timezone) @staticmethod def addMinutes(d, n, timezone=NO_VALUE): - return F('addMinutes', d, n, timezone) + return F("addMinutes", d, n, timezone) @staticmethod def addMonths(d, n, timezone=NO_VALUE): - return F('addMonths', d, n, timezone) + return F("addMonths", d, n, timezone) @staticmethod def addQuarters(d, n, timezone=NO_VALUE): - return F('addQuarters', d, n, timezone) + return F("addQuarters", d, n, timezone) @staticmethod def addSeconds(d, n, timezone=NO_VALUE): - return F('addSeconds', d, n, timezone) + return F("addSeconds", d, n, timezone) @staticmethod def addWeeks(d, n, timezone=NO_VALUE): - return F('addWeeks', d, n, timezone) + return F("addWeeks", d, n, timezone) @staticmethod def addYears(d, n, timezone=NO_VALUE): - return F('addYears', d, n, timezone) + return F("addYears", d, n, timezone) @staticmethod def subtractDays(d, n, timezone=NO_VALUE): - return F('subtractDays', d, n, timezone) + return F("subtractDays", d, n, timezone) @staticmethod def subtractHours(d, n, timezone=NO_VALUE): - return F('subtractHours', d, n, timezone) + return F("subtractHours", d, n, timezone) @staticmethod def subtractMinutes(d, n, timezone=NO_VALUE): - return F('subtractMinutes', d, n, timezone) + return F("subtractMinutes", d, n, timezone) @staticmethod def subtractMonths(d, n, timezone=NO_VALUE): - return F('subtractMonths', d, n, timezone) + return F("subtractMonths", d, n, timezone) @staticmethod def subtractQuarters(d, n, timezone=NO_VALUE): - return F('subtractQuarters', d, n, timezone) + return F("subtractQuarters", d, n, timezone) @staticmethod def subtractSeconds(d, n, timezone=NO_VALUE): - return F('subtractSeconds', d, n, timezone) + return F("subtractSeconds", d, n, timezone) @staticmethod def subtractWeeks(d, n, timezone=NO_VALUE): - return F('subtractWeeks', d, n, timezone) + return F("subtractWeeks", d, n, timezone) @staticmethod def subtractYears(d, n, timezone=NO_VALUE): - return F('subtractYears', d, n, timezone) + return F("subtractYears", d, n, timezone) @staticmethod def toIntervalSecond(number): - return F('toIntervalSecond', number) + return F("toIntervalSecond", number) @staticmethod def toIntervalMinute(number): - return F('toIntervalMinute', number) + return F("toIntervalMinute", number) @staticmethod def toIntervalHour(number): - return F('toIntervalHour', number) + return F("toIntervalHour", number) @staticmethod def toIntervalDay(number): - return F('toIntervalDay', number) + return F("toIntervalDay", number) @staticmethod def toIntervalWeek(number): - return F('toIntervalWeek', number) + return F("toIntervalWeek", number) @staticmethod def toIntervalMonth(number): - return F('toIntervalMonth', number) + return F("toIntervalMonth", number) @staticmethod def toIntervalQuarter(number): - return F('toIntervalQuarter', number) + return F("toIntervalQuarter", number) @staticmethod def toIntervalYear(number): - return F('toIntervalYear', number) - + return F("toIntervalYear", number) # Type conversion functions @staticmethod @type_conversion def toUInt8(x): - return F('toUInt8', x) + return F("toUInt8", x) @staticmethod @type_conversion def toUInt16(x): - return F('toUInt16', x) + return F("toUInt16", x) @staticmethod @type_conversion def toUInt32(x): - return F('toUInt32', x) + return F("toUInt32", x) @staticmethod @type_conversion def toUInt64(x): - return F('toUInt64', x) + return F("toUInt64", x) @staticmethod @type_conversion def toInt8(x): - return F('toInt8', x) + return F("toInt8", x) @staticmethod @type_conversion def toInt16(x): - return F('toInt16', x) + return F("toInt16", x) @staticmethod @type_conversion def toInt32(x): - return F('toInt32', x) + return F("toInt32", x) @staticmethod @type_conversion def toInt64(x): - return F('toInt64', x) + return F("toInt64", x) @staticmethod @type_conversion def toFloat32(x): - return F('toFloat32', x) + return F("toFloat32", x) @staticmethod @type_conversion def toFloat64(x): - return F('toFloat64', x) + return F("toFloat64", x) @staticmethod @type_conversion def toDecimal32(x, scale): - return F('toDecimal32', x, scale) + return F("toDecimal32", x, scale) @staticmethod @type_conversion def toDecimal64(x, scale): - return F('toDecimal64', x, scale) + return F("toDecimal64", x, scale) @staticmethod @type_conversion def toDecimal128(x, scale): - return F('toDecimal128', x, scale) + return F("toDecimal128", x, scale) @staticmethod @type_conversion def toDate(x): - return F('toDate', x) + return F("toDate", x) @staticmethod @type_conversion def toDateTime(x): - return F('toDateTime', x) + return F("toDateTime", x) @staticmethod @type_conversion def toDateTime64(x, precision, timezone=NO_VALUE): - return F('toDateTime64', x, precision, timezone) + return F("toDateTime64", x, precision, timezone) @staticmethod def toString(x): - return F('toString', x) + return F("toString", x) @staticmethod def toFixedString(s, length): - return F('toFixedString', s, length) + return F("toFixedString", s, length) @staticmethod def toStringCutToZero(s): - return F('toStringCutToZero', s) + return F("toStringCutToZero", s) @staticmethod def CAST(x, type): - return F('CAST', x, type) + return F("CAST", x, type) @staticmethod @type_conversion def parseDateTimeBestEffort(d, timezone=NO_VALUE): - return F('parseDateTimeBestEffort', d, timezone) + return F("parseDateTimeBestEffort", d, timezone) # Functions for working with strings @staticmethod def empty(s): - return F('empty', s) + return F("empty", s) @staticmethod def notEmpty(s): - return F('notEmpty', s) + return F("notEmpty", s) @staticmethod @with_utf8_support def length(s): - return F('length', s) + return F("length", s) @staticmethod @with_utf8_support def lower(s): - return F('lower', s) + return F("lower", s) @staticmethod @with_utf8_support def upper(s): - return F('upper', s) + return F("upper", s) @staticmethod @with_utf8_support def reverse(s): - return F('reverse', s) + return F("reverse", s) @staticmethod def concat(*args): - return F('concat', *args) + return F("concat", *args) @staticmethod @with_utf8_support def substring(s, offset, length): - return F('substring', s, offset, length) + return F("substring", s, offset, length) @staticmethod def appendTrailingCharIfAbsent(s, c): - return F('appendTrailingCharIfAbsent', s, c) + return F("appendTrailingCharIfAbsent", s, c) @staticmethod def convertCharset(s, from_charset, to_charset): - return F('convertCharset', s, from_charset, to_charset) + return F("convertCharset", s, from_charset, to_charset) @staticmethod def base64Encode(s): - return F('base64Encode', s) + return F("base64Encode", s) @staticmethod def base64Decode(s): - return F('base64Decode', s) + return F("base64Decode", s) @staticmethod def tryBase64Decode(s): - return F('tryBase64Decode', s) + return F("tryBase64Decode", s) @staticmethod def endsWith(s, suffix): - return F('endsWith', s, suffix) + return F("endsWith", s, suffix) @staticmethod def startsWith(s, prefix): - return F('startsWith', s, prefix) + return F("startsWith", s, prefix) @staticmethod def trimLeft(s): - return F('trimLeft', s) + return F("trimLeft", s) @staticmethod def trimRight(s): - return F('trimRight', s) + return F("trimRight", s) @staticmethod def trimBoth(s): - return F('trimBoth', s) + return F("trimBoth", s) @staticmethod def CRC32(s): - return F('CRC32', s) + return F("CRC32", s) # Functions for searching in strings @staticmethod @with_utf8_support def position(haystack, needle): - return F('position', haystack, needle) + return F("position", haystack, needle) @staticmethod @with_utf8_support def positionCaseInsensitive(haystack, needle): - return F('positionCaseInsensitive', haystack, needle) + return F("positionCaseInsensitive", haystack, needle) @staticmethod def like(haystack, pattern): - return F('like', haystack, pattern) + return F("like", haystack, pattern) @staticmethod def notLike(haystack, pattern): - return F('notLike', haystack, pattern) + return F("notLike", haystack, pattern) @staticmethod def match(haystack, pattern): - return F('match', haystack, pattern) + return F("match", haystack, pattern) @staticmethod def extract(haystack, pattern): - return F('extract', haystack, pattern) + return F("extract", haystack, pattern) @staticmethod def extractAll(haystack, pattern): - return F('extractAll', haystack, pattern) + return F("extractAll", haystack, pattern) @staticmethod @with_utf8_support def ngramDistance(haystack, needle): - return F('ngramDistance', haystack, needle) + return F("ngramDistance", haystack, needle) @staticmethod @with_utf8_support def ngramDistanceCaseInsensitive(haystack, needle): - return F('ngramDistanceCaseInsensitive', haystack, needle) + return F("ngramDistanceCaseInsensitive", haystack, needle) @staticmethod @with_utf8_support def ngramSearch(haystack, needle): - return F('ngramSearch', haystack, needle) + return F("ngramSearch", haystack, needle) @staticmethod @with_utf8_support def ngramSearchCaseInsensitive(haystack, needle): - return F('ngramSearchCaseInsensitive', haystack, needle) + return F("ngramSearchCaseInsensitive", haystack, needle) # Functions for replacing in strings @staticmethod def replace(haystack, pattern, replacement): - return F('replace', haystack, pattern, replacement) + return F("replace", haystack, pattern, replacement) + replaceAll = replace @staticmethod def replaceAll(haystack, pattern, replacement): - return F('replaceAll', haystack, pattern, replacement) + return F("replaceAll", haystack, pattern, replacement) @staticmethod def replaceOne(haystack, pattern, replacement): - return F('replaceOne', haystack, pattern, replacement) + return F("replaceOne", haystack, pattern, replacement) @staticmethod def replaceRegexpAll(haystack, pattern, replacement): - return F('replaceRegexpAll', haystack, pattern, replacement) + return F("replaceRegexpAll", haystack, pattern, replacement) @staticmethod def replaceRegexpOne(haystack, pattern, replacement): - return F('replaceRegexpOne', haystack, pattern, replacement) + return F("replaceRegexpOne", haystack, pattern, replacement) @staticmethod def regexpQuoteMeta(x): - return F('regexpQuoteMeta', x) + return F("regexpQuoteMeta", x) # Mathematical functions @staticmethod def e(): - return F('e') + return F("e") @staticmethod def pi(): - return F('pi') + return F("pi") @staticmethod def exp(x): - return F('exp', x) + return F("exp", x) @staticmethod def log(x): - return F('log', x) + return F("log", x) + ln = log @staticmethod def exp2(x): - return F('exp2', x) + return F("exp2", x) @staticmethod def log2(x): - return F('log2', x) + return F("log2", x) @staticmethod def exp10(x): - return F('exp10', x) + return F("exp10", x) @staticmethod def log10(x): - return F('log10', x) + return F("log10", x) @staticmethod def sqrt(x): - return F('sqrt', x) + return F("sqrt", x) @staticmethod def cbrt(x): - return F('cbrt', x) + return F("cbrt", x) @staticmethod def erf(x): - return F('erf', x) + return F("erf", x) @staticmethod def erfc(x): - return F('erfc', x) + return F("erfc", x) @staticmethod def lgamma(x): - return F('lgamma', x) + return F("lgamma", x) @staticmethod def tgamma(x): - return F('tgamma', x) + return F("tgamma", x) @staticmethod def sin(x): - return F('sin', x) + return F("sin", x) @staticmethod def cos(x): - return F('cos', x) + return F("cos", x) @staticmethod def tan(x): - return F('tan', x) + return F("tan", x) @staticmethod def asin(x): - return F('asin', x) + return F("asin", x) @staticmethod def acos(x): - return F('acos', x) + return F("acos", x) @staticmethod def atan(x): - return F('atan', x) + return F("atan", x) @staticmethod def power(x, y): - return F('power', x, y) + return F("power", x, y) + pow = power @staticmethod def intExp10(x): - return F('intExp10', x) + return F("intExp10", x) @staticmethod def intExp2(x): - return F('intExp2', x) + return F("intExp2", x) # Rounding functions @staticmethod def floor(x, n=None): - return F('floor', x, n) if n else F('floor', x) + return F("floor", x, n) if n else F("floor", x) @staticmethod def ceiling(x, n=None): - return F('ceiling', x, n) if n else F('ceiling', x) + return F("ceiling", x, n) if n else F("ceiling", x) + ceil = ceiling @staticmethod def round(x, n=None): - return F('round', x, n) if n else F('round', x) + return F("round", x, n) if n else F("round", x) @staticmethod def roundAge(x): - return F('roundAge', x) + return F("roundAge", x) @staticmethod def roundDown(x, y): - return F('roundDown', x, y) + return F("roundDown", x, y) @staticmethod def roundDuration(x): - return F('roundDuration', x) + return F("roundDuration", x) @staticmethod def roundToExp2(x): - return F('roundToExp2', x) + return F("roundToExp2", x) # Functions for working with arrays @staticmethod def emptyArrayDate(): - return F('emptyArrayDate') + return F("emptyArrayDate") @staticmethod def emptyArrayDateTime(): - return F('emptyArrayDateTime') + return F("emptyArrayDateTime") @staticmethod def emptyArrayFloat32(): - return F('emptyArrayFloat32') + return F("emptyArrayFloat32") @staticmethod def emptyArrayFloat64(): - return F('emptyArrayFloat64') + return F("emptyArrayFloat64") @staticmethod def emptyArrayInt16(): - return F('emptyArrayInt16') + return F("emptyArrayInt16") @staticmethod def emptyArrayInt32(): - return F('emptyArrayInt32') + return F("emptyArrayInt32") @staticmethod def emptyArrayInt64(): - return F('emptyArrayInt64') + return F("emptyArrayInt64") @staticmethod def emptyArrayInt8(): - return F('emptyArrayInt8') + return F("emptyArrayInt8") @staticmethod def emptyArrayString(): - return F('emptyArrayString') + return F("emptyArrayString") @staticmethod def emptyArrayUInt16(): - return F('emptyArrayUInt16') + return F("emptyArrayUInt16") @staticmethod def emptyArrayUInt32(): - return F('emptyArrayUInt32') + return F("emptyArrayUInt32") @staticmethod def emptyArrayUInt64(): - return F('emptyArrayUInt64') + return F("emptyArrayUInt64") @staticmethod def emptyArrayUInt8(): - return F('emptyArrayUInt8') + return F("emptyArrayUInt8") @staticmethod def emptyArrayToSingle(x): - return F('emptyArrayToSingle', x) + return F("emptyArrayToSingle", x) @staticmethod def range(n): - return F('range', n) + return F("range", n) @staticmethod def array(*args): - return F('array', *args) + return F("array", *args) @staticmethod def arrayConcat(*args): - return F('arrayConcat', *args) + return F("arrayConcat", *args) @staticmethod def arrayElement(arr, n): - return F('arrayElement', arr, n) + return F("arrayElement", arr, n) @staticmethod def has(arr, x): - return F('has', arr, x) + return F("has", arr, x) @staticmethod def hasAll(arr, x): - return F('hasAll', arr, x) + return F("hasAll", arr, x) @staticmethod def hasAny(arr, x): - return F('hasAny', arr, x) + return F("hasAny", arr, x) @staticmethod def indexOf(arr, x): - return F('indexOf', arr, x) + return F("indexOf", arr, x) @staticmethod def countEqual(arr, x): - return F('countEqual', arr, x) + return F("countEqual", arr, x) @staticmethod def arrayEnumerate(arr): - return F('arrayEnumerate', arr) + return F("arrayEnumerate", arr) @staticmethod def arrayEnumerateDense(*args): - return F('arrayEnumerateDense', *args) + return F("arrayEnumerateDense", *args) @staticmethod def arrayEnumerateDenseRanked(*args): - return F('arrayEnumerateDenseRanked', *args) + return F("arrayEnumerateDenseRanked", *args) @staticmethod def arrayEnumerateUniq(*args): - return F('arrayEnumerateUniq', *args) + return F("arrayEnumerateUniq", *args) @staticmethod def arrayEnumerateUniqRanked(*args): - return F('arrayEnumerateUniqRanked', *args) + return F("arrayEnumerateUniqRanked", *args) @staticmethod def arrayPopBack(arr): - return F('arrayPopBack', arr) + return F("arrayPopBack", arr) @staticmethod def arrayPopFront(arr): - return F('arrayPopFront', arr) + return F("arrayPopFront", arr) @staticmethod def arrayPushBack(arr, x): - return F('arrayPushBack', arr, x) + return F("arrayPushBack", arr, x) @staticmethod def arrayPushFront(arr, x): - return F('arrayPushFront', arr, x) + return F("arrayPushFront", arr, x) @staticmethod def arrayResize(array, size, extender=None): - return F('arrayResize', array, size, extender) if extender is not None else F('arrayResize', array, size) + return F("arrayResize", array, size, extender) if extender is not None else F("arrayResize", array, size) @staticmethod def arraySlice(array, offset, length=None): - return F('arraySlice', array, offset, length) if length is not None else F('arraySlice', array, offset) + return F("arraySlice", array, offset, length) if length is not None else F("arraySlice", array, offset) @staticmethod def arrayUniq(*args): - return F('arrayUniq', *args) + return F("arrayUniq", *args) @staticmethod def arrayJoin(arr): - return F('arrayJoin', arr) + return F("arrayJoin", arr) @staticmethod def arrayDifference(arr): - return F('arrayDifference', arr) + return F("arrayDifference", arr) @staticmethod def arrayDistinct(x): - return F('arrayDistinct', x) + return F("arrayDistinct", x) @staticmethod def arrayIntersect(*args): - return F('arrayIntersect', *args) + return F("arrayIntersect", *args) @staticmethod def arrayReduce(agg_func_name, *args): - return F('arrayReduce', agg_func_name, *args) + return F("arrayReduce", agg_func_name, *args) @staticmethod def arrayReverse(arr): - return F('arrayReverse', arr) + return F("arrayReverse", arr) # Functions for splitting and merging strings and arrays @staticmethod def splitByChar(sep, s): - return F('splitByChar', sep, s) + return F("splitByChar", sep, s) @staticmethod def splitByString(sep, s): - return F('splitByString', sep, s) + return F("splitByString", sep, s) @staticmethod def arrayStringConcat(arr, sep=None): - return F('arrayStringConcat', arr, sep) if sep else F('arrayStringConcat', arr) + return F("arrayStringConcat", arr, sep) if sep else F("arrayStringConcat", arr) @staticmethod def alphaTokens(s): - return F('alphaTokens', s) + return F("alphaTokens", s) # Bit functions @staticmethod def bitAnd(x, y): - return F('bitAnd', x, y) + return F("bitAnd", x, y) @staticmethod def bitNot(x): - return F('bitNot', x) + return F("bitNot", x) @staticmethod def bitOr(x, y): - return F('bitOr', x, y) + return F("bitOr", x, y) @staticmethod def bitRotateLeft(x, y): - return F('bitRotateLeft', x, y) + return F("bitRotateLeft", x, y) @staticmethod def bitRotateRight(x, y): - return F('bitRotateRight', x, y) + return F("bitRotateRight", x, y) @staticmethod def bitShiftLeft(x, y): - return F('bitShiftLeft', x, y) + return F("bitShiftLeft", x, y) @staticmethod def bitShiftRight(x, y): - return F('bitShiftRight', x, y) + return F("bitShiftRight", x, y) @staticmethod def bitTest(x, y): - return F('bitTest', x, y) + return F("bitTest", x, y) @staticmethod def bitTestAll(x, *args): - return F('bitTestAll', x, *args) + return F("bitTestAll", x, *args) @staticmethod def bitTestAny(x, *args): - return F('bitTestAny', x, *args) + return F("bitTestAny", x, *args) @staticmethod def bitXor(x, y): - return F('bitXor', x, y) + return F("bitXor", x, y) # Bitmap functions @staticmethod def bitmapAnd(x, y): - return F('bitmapAnd', x, y) + return F("bitmapAnd", x, y) @staticmethod def bitmapAndCardinality(x, y): - return F('bitmapAndCardinality', x, y) + return F("bitmapAndCardinality", x, y) @staticmethod def bitmapAndnot(x, y): - return F('bitmapAndnot', x, y) + return F("bitmapAndnot", x, y) @staticmethod def bitmapAndnotCardinality(x, y): - return F('bitmapAndnotCardinality', x, y) + return F("bitmapAndnotCardinality", x, y) @staticmethod def bitmapBuild(x): - return F('bitmapBuild', x) + return F("bitmapBuild", x) @staticmethod def bitmapCardinality(x): - return F('bitmapCardinality', x) + return F("bitmapCardinality", x) @staticmethod def bitmapContains(haystack, needle): - return F('bitmapContains', haystack, needle) + return F("bitmapContains", haystack, needle) @staticmethod def bitmapHasAll(x, y): - return F('bitmapHasAll', x, y) + return F("bitmapHasAll", x, y) @staticmethod def bitmapHasAny(x, y): - return F('bitmapHasAny', x, y) + return F("bitmapHasAny", x, y) @staticmethod def bitmapOr(x, y): - return F('bitmapOr', x, y) + return F("bitmapOr", x, y) @staticmethod def bitmapOrCardinality(x, y): - return F('bitmapOrCardinality', x, y) + return F("bitmapOrCardinality", x, y) @staticmethod def bitmapToArray(x): - return F('bitmapToArray', x) + return F("bitmapToArray", x) @staticmethod def bitmapXor(x, y): - return F('bitmapXor', x, y) + return F("bitmapXor", x, y) @staticmethod def bitmapXorCardinality(x, y): - return F('bitmapXorCardinality', x, y) + return F("bitmapXorCardinality", x, y) # Hash functions @staticmethod def halfMD5(*args): - return F('halfMD5', *args) + return F("halfMD5", *args) @staticmethod def MD5(s): - return F('MD5', s) + return F("MD5", s) @staticmethod def sipHash128(*args): - return F('sipHash128', *args) + return F("sipHash128", *args) @staticmethod def sipHash64(*args): - return F('sipHash64', *args) + return F("sipHash64", *args) @staticmethod def cityHash64(*args): - return F('cityHash64', *args) + return F("cityHash64", *args) @staticmethod def intHash32(x): - return F('intHash32', x) + return F("intHash32", x) @staticmethod def intHash64(x): - return F('intHash64', x) + return F("intHash64", x) @staticmethod def SHA1(s): - return F('SHA1', s) + return F("SHA1", s) @staticmethod def SHA224(s): - return F('SHA224', s) + return F("SHA224", s) @staticmethod def SHA256(s): - return F('SHA256', s) + return F("SHA256", s) @staticmethod def URLHash(url, n=None): - return F('URLHash', url, n) if n is not None else F('URLHash', url) + return F("URLHash", url, n) if n is not None else F("URLHash", url) @staticmethod def farmHash64(*args): - return F('farmHash64',*args) + return F("farmHash64", *args) @staticmethod def javaHash(s): - return F('javaHash', s) + return F("javaHash", s) @staticmethod def hiveHash(s): - return F('hiveHash', s) + return F("hiveHash", s) @staticmethod def metroHash64(*args): - return F('metroHash64', *args) + return F("metroHash64", *args) @staticmethod def jumpConsistentHash(x, buckets): - return F('jumpConsistentHash', x, buckets) + return F("jumpConsistentHash", x, buckets) @staticmethod def murmurHash2_32(*args): - return F('murmurHash2_32', *args) + return F("murmurHash2_32", *args) @staticmethod def murmurHash2_64(*args): - return F('murmurHash2_64', *args) + return F("murmurHash2_64", *args) @staticmethod def murmurHash3_32(*args): - return F('murmurHash3_32', *args) + return F("murmurHash3_32", *args) @staticmethod def murmurHash3_64(*args): - return F('murmurHash3_64', *args) + return F("murmurHash3_64", *args) @staticmethod def murmurHash3_128(s): - return F('murmurHash3_128', s) + return F("murmurHash3_128", s) @staticmethod def xxHash32(*args): - return F('xxHash32', *args) + return F("xxHash32", *args) @staticmethod def xxHash64(*args): - return F('xxHash64', *args) + return F("xxHash64", *args) # Functions for generating pseudo-random numbers @staticmethod def rand(dummy=None): - return F('rand') if dummy is None else F('rand', dummy) + return F("rand") if dummy is None else F("rand", dummy) @staticmethod def rand64(dummy=None): - return F('rand64') if dummy is None else F('rand64', dummy) + return F("rand64") if dummy is None else F("rand64", dummy) @staticmethod def randConstant(dummy=None): - return F('randConstant') if dummy is None else F('randConstant', dummy) + return F("randConstant") if dummy is None else F("randConstant", dummy) # Encoding functions @staticmethod def hex(x): - return F('hex', x) + return F("hex", x) @staticmethod def unhex(x): - return F('unhex', x) + return F("unhex", x) @staticmethod def bitmaskToArray(x): - return F('bitmaskToArray', x) + return F("bitmaskToArray", x) @staticmethod def bitmaskToList(x): - return F('bitmaskToList', x) + return F("bitmaskToList", x) # Functions for working with UUID @staticmethod def generateUUIDv4(): - return F('generateUUIDv4') + return F("generateUUIDv4") @staticmethod def toUUID(s): - return F('toUUID', s) + return F("toUUID", s) @staticmethod def UUIDNumToString(s): - return F('UUIDNumToString', s) + return F("UUIDNumToString", s) @staticmethod def UUIDStringToNum(s): - return F('UUIDStringToNum', s) + return F("UUIDStringToNum", s) # Functions for working with IP addresses @staticmethod def IPv4CIDRToRange(ipv4, cidr): - return F('IPv4CIDRToRange', ipv4, cidr) + return F("IPv4CIDRToRange", ipv4, cidr) @staticmethod def IPv4NumToString(num): - return F('IPv4NumToString', num) + return F("IPv4NumToString", num) @staticmethod def IPv4NumToStringClassC(num): - return F('IPv4NumToStringClassC', num) + return F("IPv4NumToStringClassC", num) @staticmethod def IPv4StringToNum(s): - return F('IPv4StringToNum', s) + return F("IPv4StringToNum", s) @staticmethod def IPv4ToIPv6(ipv4): - return F('IPv4ToIPv6', ipv4) + return F("IPv4ToIPv6", ipv4) @staticmethod def IPv6CIDRToRange(ipv6, cidr): - return F('IPv6CIDRToRange', ipv6, cidr) + return F("IPv6CIDRToRange", ipv6, cidr) @staticmethod def IPv6NumToString(num): - return F('IPv6NumToString', num) + return F("IPv6NumToString", num) @staticmethod def IPv6StringToNum(s): - return F('IPv6StringToNum', s) + return F("IPv6StringToNum", s) @staticmethod def toIPv4(ipv4): - return F('toIPv4', ipv4) + return F("toIPv4", ipv4) @staticmethod def toIPv6(ipv6): - return F('toIPv6', ipv6) + return F("toIPv6", ipv6) # Aggregate functions @staticmethod @aggregate def any(x): - return F('any', x) + return F("any", x) @staticmethod @aggregate def anyHeavy(x): - return F('anyHeavy', x) + return F("anyHeavy", x) @staticmethod @aggregate def anyLast(x): - return F('anyLast', x) + return F("anyLast", x) @staticmethod @aggregate def argMax(x, y): - return F('argMax', x, y) + return F("argMax", x, y) @staticmethod @aggregate def argMin(x, y): - return F('argMin', x, y) + return F("argMin", x, y) @staticmethod @aggregate def avg(x): - return F('avg', x) + return F("avg", x) @staticmethod @aggregate def corr(x, y): - return F('corr', x, y) + return F("corr", x, y) @staticmethod @aggregate def count(): - return F('count') + return F("count") @staticmethod @aggregate def covarPop(x, y): - return F('covarPop', x, y) + return F("covarPop", x, y) @staticmethod @aggregate def covarSamp(x, y): - return F('covarSamp', x, y) + return F("covarSamp", x, y) @staticmethod @aggregate def kurtPop(x): - return F('kurtPop', x) + return F("kurtPop", x) @staticmethod @aggregate def kurtSamp(x): - return F('kurtSamp', x) + return F("kurtSamp", x) @staticmethod @aggregate def min(x): - return F('min', x) + return F("min", x) @staticmethod @aggregate def max(x): - return F('max', x) + return F("max", x) @staticmethod @aggregate def skewPop(x): - return F('skewPop', x) + return F("skewPop", x) @staticmethod @aggregate def skewSamp(x): - return F('skewSamp', x) + return F("skewSamp", x) @staticmethod @aggregate def sum(x): - return F('sum', x) + return F("sum", x) @staticmethod @aggregate def uniq(*args): - return F('uniq', *args) + return F("uniq", *args) @staticmethod @aggregate def uniqExact(*args): - return F('uniqExact', *args) + return F("uniqExact", *args) @staticmethod @aggregate def uniqHLL12(*args): - return F('uniqHLL12', *args) + return F("uniqHLL12", *args) @staticmethod @aggregate def varPop(x): - return F('varPop', x) + return F("varPop", x) @staticmethod @aggregate def varSamp(x): - return F('varSamp', x) + return F("varSamp", x) @staticmethod @aggregate @parametric def quantile(expr): - return F('quantile', expr) + return F("quantile", expr) @staticmethod @aggregate @parametric def quantileDeterministic(expr, determinator): - return F('quantileDeterministic', expr, determinator) + return F("quantileDeterministic", expr, determinator) @staticmethod @aggregate @parametric def quantileExact(expr): - return F('quantileExact', expr) + return F("quantileExact", expr) @staticmethod @aggregate @parametric def quantileExactWeighted(expr, weight): - return F('quantileExactWeighted', expr, weight) + return F("quantileExactWeighted", expr, weight) @staticmethod @aggregate @parametric def quantileTiming(expr): - return F('quantileTiming', expr) + return F("quantileTiming", expr) @staticmethod @aggregate @parametric def quantileTimingWeighted(expr, weight): - return F('quantileTimingWeighted', expr, weight) + return F("quantileTimingWeighted", expr, weight) @staticmethod @aggregate @parametric def quantileTDigest(expr): - return F('quantileTDigest', expr) + return F("quantileTDigest", expr) @staticmethod @aggregate @parametric def quantileTDigestWeighted(expr, weight): - return F('quantileTDigestWeighted', expr, weight) + return F("quantileTDigestWeighted", expr, weight) @staticmethod @aggregate @parametric def quantiles(expr): - return F('quantiles', expr) + return F("quantiles", expr) @staticmethod @aggregate @parametric def quantilesDeterministic(expr, determinator): - return F('quantilesDeterministic', expr, determinator) + return F("quantilesDeterministic", expr, determinator) @staticmethod @aggregate @parametric def quantilesExact(expr): - return F('quantilesExact', expr) + return F("quantilesExact", expr) @staticmethod @aggregate @parametric def quantilesExactWeighted(expr, weight): - return F('quantilesExactWeighted', expr, weight) + return F("quantilesExactWeighted", expr, weight) @staticmethod @aggregate @parametric def quantilesTiming(expr): - return F('quantilesTiming', expr) + return F("quantilesTiming", expr) @staticmethod @aggregate @parametric def quantilesTimingWeighted(expr, weight): - return F('quantilesTimingWeighted', expr, weight) + return F("quantilesTimingWeighted", expr, weight) @staticmethod @aggregate @parametric def quantilesTDigest(expr): - return F('quantilesTDigest', expr) + return F("quantilesTDigest", expr) @staticmethod @aggregate @parametric def quantilesTDigestWeighted(expr, weight): - return F('quantilesTDigestWeighted', expr, weight) + return F("quantilesTDigestWeighted", expr, weight) @staticmethod @aggregate @parametric def topK(expr): - return F('topK', expr) + return F("topK", expr) @staticmethod @aggregate @parametric def topKWeighted(expr, weight): - return F('topKWeighted', expr, weight) + return F("topKWeighted", expr, weight) # Null handling functions @staticmethod def ifNull(x, y): - return F('ifNull', x, y) + return F("ifNull", x, y) @staticmethod def nullIf(x, y): - return F('nullIf', x, y) + return F("nullIf", x, y) @staticmethod def isNotNull(x): - return F('isNotNull', x) + return F("isNotNull", x) @staticmethod def isNull(x): - return F('isNull', x) + return F("isNull", x) @staticmethod def coalesce(*args): - return F('coalesce', *args) + return F("coalesce", *args) # Misc functions @staticmethod def ifNotFinite(x, y): - return F('ifNotFinite', x, y) + return F("ifNotFinite", x, y) @staticmethod def isFinite(x): - return F('isFinite', x) + return F("isFinite", x) @staticmethod def isInfinite(x): - return F('isInfinite', x) + return F("isInfinite", x) @staticmethod def isNaN(x): - return F('isNaN', x) + return F("isNaN", x) @staticmethod def least(x, y): - return F('least', x, y) + return F("least", x, y) @staticmethod def greatest(x, y): - return F('greatest', x, y) + return F("greatest", x, y) # Dictionary functions @staticmethod def dictGet(dict_name, attr_name, id_expr): - return F('dictGet', dict_name, attr_name, id_expr) + return F("dictGet", dict_name, attr_name, id_expr) @staticmethod def dictGetOrDefault(dict_name, attr_name, id_expr, default): - return F('dictGetOrDefault', dict_name, attr_name, id_expr, default) + return F("dictGetOrDefault", dict_name, attr_name, id_expr, default) @staticmethod def dictHas(dict_name, id_expr): - return F('dictHas', dict_name, id_expr) + return F("dictHas", dict_name, id_expr) @staticmethod def dictGetHierarchy(dict_name, id_expr): - return F('dictGetHierarchy', dict_name, id_expr) + return F("dictGetHierarchy", dict_name, id_expr) @staticmethod def dictIsIn(dict_name, child_id_expr, ancestor_id_expr): - return F('dictIsIn', dict_name, child_id_expr, ancestor_id_expr) + return F("dictIsIn", dict_name, child_id_expr, ancestor_id_expr) # Expose only relevant classes in import * -__all__ = ['F'] - +__all__ = ["F"] From 4e49da3b19599c03d33fa361e4447b6989ed3b67 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:12:23 +0100 Subject: [PATCH 10/51] Chore: fix linting on majority of modules --- clickhouse_orm/funcs.py | 4 +-- clickhouse_orm/migrations.py | 9 ++++--- clickhouse_orm/models.py | 47 ++++++++++++++++----------------- clickhouse_orm/system_models.py | 2 +- clickhouse_orm/utils.py | 8 +++--- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/clickhouse_orm/funcs.py b/clickhouse_orm/funcs.py index 192b0b3..52be002 100644 --- a/clickhouse_orm/funcs.py +++ b/clickhouse_orm/funcs.py @@ -1,9 +1,9 @@ from functools import wraps -from inspect import signature, Parameter +from inspect import Parameter, signature from types import FunctionType -from .utils import is_iterable, comma_join, NO_VALUE, arg_to_sql from .query import Cond, QuerySet +from .utils import NO_VALUE, arg_to_sql, comma_join, is_iterable def binary_operator(func): diff --git a/clickhouse_orm/migrations.py b/clickhouse_orm/migrations.py index dc37f20..722bf8c 100644 --- a/clickhouse_orm/migrations.py +++ b/clickhouse_orm/migrations.py @@ -1,9 +1,10 @@ -from .models import Model, BufferModel -from .fields import DateField, StringField +import logging + from .engines import MergeTree -from .utils import escape, get_subclass_names +from .fields import DateField, StringField +from .models import BufferModel, Model +from .utils import get_subclass_names -import logging logger = logging.getLogger('migrations') diff --git a/clickhouse_orm/models.py b/clickhouse_orm/models.py index 0fc5e7f..3caa311 100644 --- a/clickhouse_orm/models.py +++ b/clickhouse_orm/models.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals import sys from collections import OrderedDict from itertools import chain @@ -6,23 +5,22 @@ import pytz +from .engines import Distributed, Merge from .fields import Field, StringField -from .utils import parse_tsv, NO_VALUE, get_subclass_names, arg_to_sql, unescape -from .query import QuerySet from .funcs import F -from .engines import Merge, Distributed +from .query import QuerySet +from .utils import NO_VALUE, arg_to_sql, get_subclass_names, parse_tsv logger = getLogger('clickhouse_orm') - class Constraint: ''' Defines a model constraint. ''' - name = None # this is set by the parent model - parent = None # this is set by the parent model + name = None # this is set by the parent model + parent = None # this is set by the parent model def __init__(self, expr): ''' @@ -42,8 +40,8 @@ class Index: Defines a data-skipping index. ''' - name = None # this is set by the parent model - parent = None # this is set by the parent model + name = None # this is set by the parent model + parent = None # this is set by the parent model def __init__(self, expr, type, granularity): ''' @@ -126,7 +124,7 @@ class ModelBase(type): ad_hoc_model_cache = {} - def __new__(cls, name, bases, attrs): + def __new__(metacls, name, bases, attrs): # Collect fields, constraints and indexes from parent classes fields = {} @@ -172,35 +170,36 @@ def __new__(cls, name, bases, attrs): _defaults=defaults, _has_funcs_as_defaults=has_funcs_as_defaults ) - model = super(ModelBase, cls).__new__(cls, str(name), bases, attrs) + model = super(ModelBase, metacls).__new__(metacls, str(name), bases, attrs) # Let each field, constraint and index know its parent and its own name for n, obj in chain(fields, constraints.items(), indexes.items()): - setattr(obj, 'parent', model) - setattr(obj, 'name', n) + obj.parent = model + obj.name = n return model @classmethod - def create_ad_hoc_model(cls, fields, model_name='AdHocModel'): + def create_ad_hoc_model(metacls, fields, model_name='AdHocModel'): # fields is a list of tuples (name, db_type) # Check if model exists in cache fields = list(fields) cache_key = model_name + ' ' + str(fields) - if cache_key in cls.ad_hoc_model_cache: - return cls.ad_hoc_model_cache[cache_key] + if cache_key in metacls.ad_hoc_model_cache: + return metacls.ad_hoc_model_cache[cache_key] # Create an ad hoc model class attrs = {} for name, db_type in fields: - attrs[name] = cls.create_ad_hoc_field(db_type) - model_class = cls.__new__(cls, model_name, (Model,), attrs) + attrs[name] = metacls.create_ad_hoc_field(db_type) + model_class = metacls.__new__(metacls, model_name, (Model,), attrs) # Add the model class to the cache - cls.ad_hoc_model_cache[cache_key] = model_class + metacls.ad_hoc_model_cache[cache_key] = model_class return model_class @classmethod - def create_ad_hoc_field(cls, db_type): + def create_ad_hoc_field(metacls, db_type): import clickhouse_orm.fields as orm_fields + # Enums if db_type.startswith('Enum'): return orm_fields.BaseEnumField.create_ad_hoc_field(db_type) @@ -219,13 +218,13 @@ def create_ad_hoc_field(cls, db_type): ) # Arrays if db_type.startswith('Array'): - inner_field = cls.create_ad_hoc_field(db_type[6 : -1]) + inner_field = metacls.create_ad_hoc_field(db_type[6 : -1]) return orm_fields.ArrayField(inner_field) # Tuples (poor man's version - convert to array) if db_type.startswith('Tuple'): types = [s.strip() for s in db_type[6 : -1].split(',')] assert len(set(types)) == 1, 'No support for mixed types in tuples - ' + db_type - inner_field = cls.create_ad_hoc_field(types[0]) + inner_field = metacls.create_ad_hoc_field(types[0]) return orm_fields.ArrayField(inner_field) # FixedString if db_type.startswith('FixedString'): @@ -239,11 +238,11 @@ def create_ad_hoc_field(cls, db_type): return field_class(*args) # Nullable if db_type.startswith('Nullable'): - inner_field = cls.create_ad_hoc_field(db_type[9 : -1]) + inner_field = metacls.create_ad_hoc_field(db_type[9 : -1]) return orm_fields.NullableField(inner_field) # LowCardinality if db_type.startswith('LowCardinality'): - inner_field = cls.create_ad_hoc_field(db_type[15 : -1]) + inner_field = metacls.create_ad_hoc_field(db_type[15 : -1]) return orm_fields.LowCardinalityField(inner_field) # Simple fields name = db_type + 'Field' diff --git a/clickhouse_orm/system_models.py b/clickhouse_orm/system_models.py index 69b67fa..af280c7 100644 --- a/clickhouse_orm/system_models.py +++ b/clickhouse_orm/system_models.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from .database import Database -from .fields import * +from .fields import DateTimeField, StringField, UInt8Field, UInt32Field, UInt64Field from .models import Model from .utils import comma_join diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index 78701ff..140cbed 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -1,8 +1,9 @@ import codecs +import importlib +import pkgutil import re from datetime import date, datetime, tzinfo, timedelta - SPECIAL_CHARS = { "\b" : "\\b", "\f" : "\\f", @@ -17,7 +18,6 @@ SPECIAL_CHARS_REGEX = re.compile("[" + ''.join(SPECIAL_CHARS.values()) + "]") - def escape(value, quote=True): ''' If the value is a string, escapes any special characters and optionally @@ -48,7 +48,7 @@ def arg_to_sql(arg): Supports functions, model fields, strings, dates, datetimes, timedeltas, booleans, None, numbers, timezones, arrays/iterables. """ - from clickhouse_orm import Field, StringField, DateTimeField, DateField, F, QuerySet + from clickhouse_orm import Field, StringField, DateTimeField, F, QuerySet if isinstance(arg, F): return arg.to_sql() if isinstance(arg, Field): @@ -122,7 +122,6 @@ def import_submodules(package_name): """ Import all submodules of a module. """ - import importlib, pkgutil package = importlib.import_module(package_name) return { name: importlib.import_module(package_name + '.' + name) @@ -164,4 +163,5 @@ class NoValue: def __repr__(self): return 'NO_VALUE' + NO_VALUE = NoValue() From 87e7858a04bea85886042e90fda168e1a7592514 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:14:22 +0100 Subject: [PATCH 11/51] Chore: fix linting on __init__.py --- clickhouse_orm/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/clickhouse_orm/__init__.py b/clickhouse_orm/__init__.py index 292e25c..90f41cf 100644 --- a/clickhouse_orm/__init__.py +++ b/clickhouse_orm/__init__.py @@ -1,13 +1,12 @@ -__import__("pkg_resources").declare_namespace(__name__) +from inspect import isclass -from clickhouse_orm.database import * -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * -from clickhouse_orm.funcs import * -from clickhouse_orm.migrations import * -from clickhouse_orm.models import * -from clickhouse_orm.query import * -from clickhouse_orm.system_models import * +from .database import * # noqa: F401, F403 +from .engines import * # noqa: F401, F403 +from .fields import * # noqa: F401, F403 +from .funcs import * # noqa: F401, F403 +from .migrations import * # noqa: F401, F403 +from .models import * # noqa: F401, F403 +from .query import * # noqa: F401, F403 +from .system_models import * # noqa: F401, F403 -from inspect import isclass __all__ = [c.__name__ for c in locals().values() if isclass(c)] From e60350259fa213b67faeaec0d1d0b652f1f419c2 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:14:56 +0100 Subject: [PATCH 12/51] Chore: blacken --- clickhouse_orm/database.py | 193 ++++++++++++--------- clickhouse_orm/engines.py | 201 ++++++++++++++------- clickhouse_orm/migrations.py | 154 +++++++++-------- clickhouse_orm/models.py | 297 ++++++++++++++++---------------- clickhouse_orm/system_models.py | 29 ++-- clickhouse_orm/utils.py | 57 +++--- 6 files changed, 524 insertions(+), 407 deletions(-) diff --git a/clickhouse_orm/database.py b/clickhouse_orm/database.py index 7d70c6a..3738b62 100644 --- a/clickhouse_orm/database.py +++ b/clickhouse_orm/database.py @@ -11,16 +11,18 @@ import pytz import logging -logger = logging.getLogger('clickhouse_orm') +logger = logging.getLogger("clickhouse_orm") -Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size') + +Page = namedtuple("Page", "objects number_of_objects pages_total number page_size") class DatabaseException(Exception): - ''' + """ Raised when a database operation fails. - ''' + """ + pass @@ -28,6 +30,7 @@ class ServerError(DatabaseException): """ Raised when a server returns an error. """ + def __init__(self, message): self.code = None processed = self.get_error_code_msg(message) @@ -41,16 +44,22 @@ def __init__(self, message): ERROR_PATTERNS = ( # ClickHouse prior to v19.3.3 - re.compile(r''' + re.compile( + r""" Code:\ (?P\d+), \ e\.displayText\(\)\ =\ (?P[^ \n]+):\ (?P.+?), \ e.what\(\)\ =\ (?P[^ \n]+) - ''', re.VERBOSE | re.DOTALL), + """, + re.VERBOSE | re.DOTALL, + ), # ClickHouse v19.3.3+ - re.compile(r''' + re.compile( + r""" Code:\ (?P\d+), \ e\.displayText\(\)\ =\ (?P[^ \n]+):\ (?P.+) - ''', re.VERBOSE | re.DOTALL), + """, + re.VERBOSE | re.DOTALL, + ), ) @classmethod @@ -65,7 +74,7 @@ def get_error_code_msg(cls, full_error_message): match = pattern.match(full_error_message) if match: # assert match.group('type1') == match.group('type2') - return int(match.group('code')), match.group('msg').strip() + return int(match.group("code")), match.group("msg").strip() return 0, full_error_message @@ -75,15 +84,24 @@ def __str__(self): class Database(object): - ''' + """ Database instances connect to a specific ClickHouse database for running queries, inserting data and other operations. - ''' + """ - def __init__(self, db_name, db_url='http://localhost:8123/', - username=None, password=None, readonly=False, autocreate=True, - timeout=60, verify_ssl_cert=True, log_statements=False): - ''' + def __init__( + self, + db_name, + db_url="http://localhost:8123/", + username=None, + password=None, + readonly=False, + autocreate=True, + timeout=60, + verify_ssl_cert=True, + log_statements=False, + ): + """ Initializes a database instance. Unless it's readonly, the database will be created on the ClickHouse server if it does not already exist. @@ -96,7 +114,7 @@ def __init__(self, db_name, db_url='http://localhost:8123/', - `timeout`: the connection timeout in seconds. - `verify_ssl_cert`: whether to verify the server's certificate when connecting via HTTPS. - `log_statements`: when True, all database statements are logged. - ''' + """ self.db_name = db_name self.db_url = db_url self.readonly = False @@ -104,14 +122,14 @@ def __init__(self, db_name, db_url='http://localhost:8123/', self.request_session = requests.Session() self.request_session.verify = verify_ssl_cert if username: - self.request_session.auth = (username, password or '') + self.request_session.auth = (username, password or "") self.log_statements = log_statements self.settings = {} self.db_exists = False # this is required before running _is_existing_database self.db_exists = self._is_existing_database() if readonly: if not self.db_exists: - raise DatabaseException('Database does not exist, and cannot be created under readonly connection') + raise DatabaseException("Database does not exist, and cannot be created under readonly connection") self.connection_readonly = self._is_connection_readonly() self.readonly = True elif autocreate and not self.db_exists: @@ -125,23 +143,23 @@ def __init__(self, db_name, db_url='http://localhost:8123/', self.has_low_cardinality_support = self.server_version >= (19, 0) def create_database(self): - ''' + """ Creates the database on the ClickHouse server if it does not already exist. - ''' - self._send('CREATE DATABASE IF NOT EXISTS `%s`' % self.db_name) + """ + self._send("CREATE DATABASE IF NOT EXISTS `%s`" % self.db_name) self.db_exists = True def drop_database(self): - ''' + """ Deletes the database on the ClickHouse server. - ''' - self._send('DROP DATABASE `%s`' % self.db_name) + """ + self._send("DROP DATABASE `%s`" % self.db_name) self.db_exists = False def create_table(self, model_class): - ''' + """ Creates a table for the given model class, if it does not exist already. - ''' + """ if model_class.is_system_model(): raise DatabaseException("You can't create system table") if model_class.engine is None: @@ -149,32 +167,32 @@ def create_table(self, model_class): self._send(model_class.create_table_sql(self)) def drop_table(self, model_class): - ''' + """ Drops the database table of the given model class, if it exists. - ''' + """ if model_class.is_system_model(): raise DatabaseException("You can't drop system table") self._send(model_class.drop_table_sql(self)) def does_table_exist(self, model_class): - ''' + """ Checks whether a table for the given model class already exists. Note that this only checks for existence of a table with the expected name. - ''' + """ sql = "SELECT count() FROM system.tables WHERE database = '%s' AND name = '%s'" r = self._send(sql % (self.db_name, model_class.table_name())) - return r.text.strip() == '1' + return r.text.strip() == "1" def get_model_for_table(self, table_name, system_table=False): - ''' + """ Generates a model class from an existing table in the database. This can be used for querying tables which don't have a corresponding model class, for example system tables. - `table_name`: the table to create a model for - `system_table`: whether the table is a system table, or belongs to the current database - ''' - db_name = 'system' if system_table else self.db_name + """ + db_name = "system" if system_table else self.db_name sql = "DESCRIBE `%s`.`%s` FORMAT TSV" % (db_name, table_name) lines = self._send(sql).iter_lines() fields = [parse_tsv(line)[:2] for line in lines] @@ -184,27 +202,28 @@ def get_model_for_table(self, table_name, system_table=False): return model def add_setting(self, name, value): - ''' + """ Adds a database setting that will be sent with every request. For example, `db.add_setting("max_execution_time", 10)` will limit query execution time to 10 seconds. The name must be string, and the value is converted to string in case it isn't. To remove a setting, pass `None` as the value. - ''' - assert isinstance(name, str), 'Setting name must be a string' + """ + assert isinstance(name, str), "Setting name must be a string" if value is None: self.settings.pop(name, None) else: self.settings[name] = str(value) def insert(self, model_instances, batch_size=1000): - ''' + """ Insert records into the database. - `model_instances`: any iterable containing instances of a single model class. - `batch_size`: number of records to send per chunk (use a lower number if your records are very large). - ''' + """ from io import BytesIO + i = iter(model_instances) try: first_instance = next(i) @@ -215,14 +234,13 @@ def insert(self, model_instances, batch_size=1000): if first_instance.is_read_only() or first_instance.is_system_model(): raise DatabaseException("You can't insert into read only and system tables") - fields_list = ','.join( - ['`%s`' % name for name in first_instance.fields(writable=True)]) - fmt = 'TSKV' if model_class.has_funcs_as_defaults() else 'TabSeparated' - query = 'INSERT INTO $table (%s) FORMAT %s\n' % (fields_list, fmt) + fields_list = ",".join(["`%s`" % name for name in first_instance.fields(writable=True)]) + fmt = "TSKV" if model_class.has_funcs_as_defaults() else "TabSeparated" + query = "INSERT INTO $table (%s) FORMAT %s\n" % (fields_list, fmt) def gen(): buf = BytesIO() - buf.write(self._substitute(query, model_class).encode('utf-8')) + buf.write(self._substitute(query, model_class).encode("utf-8")) first_instance.set_database(self) buf.write(first_instance.to_db_string()) # Collect lines in batches of batch_size @@ -240,35 +258,37 @@ def gen(): # Return any remaining lines in partial batch if lines: yield buf.getvalue() + self._send(gen()) def count(self, model_class, conditions=None): - ''' + """ Counts the number of records in the model's table. - `model_class`: the model to count. - `conditions`: optional SQL conditions (contents of the WHERE clause). - ''' + """ from clickhouse_orm.query import Q - query = 'SELECT count() FROM $table' + + query = "SELECT count() FROM $table" if conditions: if isinstance(conditions, Q): conditions = conditions.to_sql(model_class) - query += ' WHERE ' + str(conditions) + query += " WHERE " + str(conditions) query = self._substitute(query, model_class) r = self._send(query) return int(r.text) if r.text else 0 def select(self, query, model_class=None, settings=None): - ''' + """ Performs a query and returns a generator of model instances. - `query`: the SQL query to execute. - `model_class`: the model class matching the query's table, or `None` for getting back instances of an ad-hoc model. - `settings`: query settings to send as HTTP GET parameters - ''' - query += ' FORMAT TabSeparatedWithNamesAndTypes' + """ + query += " FORMAT TabSeparatedWithNamesAndTypes" query = self._substitute(query, model_class) r = self._send(query, settings, True) lines = r.iter_lines() @@ -281,18 +301,18 @@ def select(self, query, model_class=None, settings=None): yield model_class.from_tsv(line, field_names, self.server_timezone, self) def raw(self, query, settings=None, stream=False): - ''' + """ Performs a query and returns its output as text. - `query`: the SQL query to execute. - `settings`: query settings to send as HTTP GET parameters - `stream`: if true, the HTTP response from ClickHouse will be streamed. - ''' + """ query = self._substitute(query, None) return self._send(query, settings=settings, stream=stream).text def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None): - ''' + """ Selects records and returns a single page of model instances. - `model_class`: the model class matching the query's table, @@ -305,54 +325,63 @@ def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions= The result is a namedtuple containing `objects` (list), `number_of_objects`, `pages_total`, `number` (of the current page), and `page_size`. - ''' + """ from clickhouse_orm.query import Q + count = self.count(model_class, conditions) pages_total = int(ceil(count / float(page_size))) if page_num == -1: page_num = max(pages_total, 1) elif page_num < 1: - raise ValueError('Invalid page number: %d' % page_num) + raise ValueError("Invalid page number: %d" % page_num) offset = (page_num - 1) * page_size - query = 'SELECT * FROM $table' + query = "SELECT * FROM $table" if conditions: if isinstance(conditions, Q): conditions = conditions.to_sql(model_class) - query += ' WHERE ' + str(conditions) - query += ' ORDER BY %s' % order_by - query += ' LIMIT %d, %d' % (offset, page_size) + query += " WHERE " + str(conditions) + query += " ORDER BY %s" % order_by + query += " LIMIT %d, %d" % (offset, page_size) query = self._substitute(query, model_class) return Page( objects=list(self.select(query, model_class, settings)) if count else [], number_of_objects=count, pages_total=pages_total, number=page_num, - page_size=page_size + page_size=page_size, ) def migrate(self, migrations_package_name, up_to=9999): - ''' + """ Executes schema migrations. - `migrations_package_name` - fully qualified name of the Python package containing the migrations. - `up_to` - number of the last migration to apply. - ''' + """ from .migrations import MigrationHistory - logger = logging.getLogger('migrations') + + logger = logging.getLogger("migrations") applied_migrations = self._get_applied_migrations(migrations_package_name) modules = import_submodules(migrations_package_name) unapplied_migrations = set(modules.keys()) - applied_migrations for name in sorted(unapplied_migrations): - logger.info('Applying migration %s...', name) + logger.info("Applying migration %s...", name) for operation in modules[name].operations: operation.apply(self) - self.insert([MigrationHistory(package_name=migrations_package_name, module_name=name, applied=datetime.date.today())]) + self.insert( + [ + MigrationHistory( + package_name=migrations_package_name, module_name=name, applied=datetime.date.today() + ) + ] + ) if int(name[:4]) >= up_to: break def _get_applied_migrations(self, migrations_package_name): from .migrations import MigrationHistory + self.create_table(MigrationHistory) query = "SELECT module_name from $table WHERE package_name = '%s'" % migrations_package_name query = self._substitute(query, MigrationHistory) @@ -360,7 +389,7 @@ def _get_applied_migrations(self, migrations_package_name): def _send(self, data, settings=None, stream=False): if isinstance(data, str): - data = data.encode('utf-8') + data = data.encode("utf-8") if self.log_statements: logger.info(data) params = self._build_params(settings) @@ -373,50 +402,50 @@ def _build_params(self, settings): params = dict(settings or {}) params.update(self.settings) if self.db_exists: - params['database'] = self.db_name + params["database"] = self.db_name # Send the readonly flag, unless the connection is already readonly (to prevent db error) if self.readonly and not self.connection_readonly: - params['readonly'] = '1' + params["readonly"] = "1" return params def _substitute(self, query, model_class=None): - ''' + """ Replaces $db and $table placeholders in the query. - ''' - if '$' in query: + """ + if "$" in query: mapping = dict(db="`%s`" % self.db_name) if model_class: if model_class.is_system_model(): - mapping['table'] = "`system`.`%s`" % model_class.table_name() + mapping["table"] = "`system`.`%s`" % model_class.table_name() else: - mapping['table'] = "`%s`.`%s`" % (self.db_name, model_class.table_name()) + mapping["table"] = "`%s`.`%s`" % (self.db_name, model_class.table_name()) query = Template(query).safe_substitute(mapping) return query def _get_server_timezone(self): try: - r = self._send('SELECT timezone()') + r = self._send("SELECT timezone()") return pytz.timezone(r.text.strip()) except ServerError as e: - logger.exception('Cannot determine server timezone (%s), assuming UTC', e) + logger.exception("Cannot determine server timezone (%s), assuming UTC", e) return pytz.utc def _get_server_version(self, as_tuple=True): try: - r = self._send('SELECT version();') + r = self._send("SELECT version();") ver = r.text except ServerError as e: - logger.exception('Cannot determine server version (%s), assuming 1.1.0', e) - ver = '1.1.0' - return tuple(int(n) for n in ver.split('.')) if as_tuple else ver + logger.exception("Cannot determine server version (%s), assuming 1.1.0", e) + ver = "1.1.0" + return tuple(int(n) for n in ver.split(".")) if as_tuple else ver def _is_existing_database(self): r = self._send("SELECT count() FROM system.databases WHERE name = '%s'" % self.db_name) - return r.text.strip() == '1' + return r.text.strip() == "1" def _is_connection_readonly(self): r = self._send("SELECT value FROM system.settings WHERE name = 'readonly'") - return r.text.strip() != '0' + return r.text.strip() != "0" # Expose only relevant classes in import * diff --git a/clickhouse_orm/engines.py b/clickhouse_orm/engines.py index 9b13f82..5fef48f 100644 --- a/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -4,51 +4,57 @@ from .utils import comma_join, get_subclass_names -logger = logging.getLogger('clickhouse_orm') +logger = logging.getLogger("clickhouse_orm") class Engine(object): - def create_table_sql(self, db): - raise NotImplementedError() # pragma: no cover + raise NotImplementedError() # pragma: no cover class TinyLog(Engine): - def create_table_sql(self, db): - return 'TinyLog' + return "TinyLog" class Log(Engine): - def create_table_sql(self, db): - return 'Log' + return "Log" class Memory(Engine): - def create_table_sql(self, db): - return 'Memory' + return "Memory" class MergeTree(Engine): - - def __init__(self, date_col=None, order_by=(), sampling_expr=None, - index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, - primary_key=None): - assert type(order_by) in (list, tuple), 'order_by must be a list or tuple' - assert date_col is None or isinstance(date_col, str), 'date_col must be string if present' - assert primary_key is None or type(primary_key) in (list, tuple), 'primary_key must be a list or tuple' - assert partition_key is None or type(partition_key) in (list, tuple),\ - 'partition_key must be tuple or list if present' - assert (replica_table_path is None) == (replica_name is None), \ - 'both replica_table_path and replica_name must be specified' + def __init__( + self, + date_col=None, + order_by=(), + sampling_expr=None, + index_granularity=8192, + replica_table_path=None, + replica_name=None, + partition_key=None, + primary_key=None, + ): + assert type(order_by) in (list, tuple), "order_by must be a list or tuple" + assert date_col is None or isinstance(date_col, str), "date_col must be string if present" + assert primary_key is None or type(primary_key) in (list, tuple), "primary_key must be a list or tuple" + assert partition_key is None or type(partition_key) in ( + list, + tuple, + ), "partition_key must be tuple or list if present" + assert (replica_table_path is None) == ( + replica_name is None + ), "both replica_table_path and replica_name must be specified" # These values conflict with each other (old and new syntax of table engines. # So let's control only one of them is given. assert date_col or partition_key, "You must set either date_col or partition_key" self.date_col = date_col - self.partition_key = partition_key if partition_key else ('toYYYYMM(`%s`)' % date_col,) + self.partition_key = partition_key if partition_key else ("toYYYYMM(`%s`)" % date_col,) self.primary_key = primary_key self.order_by = order_by @@ -60,26 +66,31 @@ def __init__(self, date_col=None, order_by=(), sampling_expr=None, # I changed field name for new reality and syntax @property def key_cols(self): - logger.warning('`key_cols` attribute is deprecated and may be removed in future. Use `order_by` attribute instead') + logger.warning( + "`key_cols` attribute is deprecated and may be removed in future. Use `order_by` attribute instead" + ) return self.order_by @key_cols.setter def key_cols(self, value): - logger.warning('`key_cols` attribute is deprecated and may be removed in future. Use `order_by` attribute instead') + logger.warning( + "`key_cols` attribute is deprecated and may be removed in future. Use `order_by` attribute instead" + ) self.order_by = value def create_table_sql(self, db): name = self.__class__.__name__ if self.replica_name: - name = 'Replicated' + name + name = "Replicated" + name # In ClickHouse 1.1.54310 custom partitioning key was introduced # https://clickhouse.tech/docs/en/table_engines/custom_partitioning_key/ # Let's check version and use new syntax if available if db.server_version >= (1, 1, 54310): - partition_sql = "PARTITION BY (%s) ORDER BY (%s)" \ - % (comma_join(self.partition_key, stringify=True), - comma_join(self.order_by, stringify=True)) + partition_sql = "PARTITION BY (%s) ORDER BY (%s)" % ( + comma_join(self.partition_key, stringify=True), + comma_join(self.order_by, stringify=True), + ) if self.primary_key: partition_sql += " PRIMARY KEY (%s)" % comma_join(self.primary_key, stringify=True) @@ -92,14 +103,17 @@ def create_table_sql(self, db): elif not self.date_col: # Can't import it globally due to circular import from clickhouse_orm.database import DatabaseException - raise DatabaseException("Custom partitioning is not supported before ClickHouse 1.1.54310. " - "Please update your server or use date_col syntax." - "https://clickhouse.tech/docs/en/table_engines/custom_partitioning_key/") + + raise DatabaseException( + "Custom partitioning is not supported before ClickHouse 1.1.54310. " + "Please update your server or use date_col syntax." + "https://clickhouse.tech/docs/en/table_engines/custom_partitioning_key/" + ) else: - partition_sql = '' + partition_sql = "" params = self._build_sql_params(db) - return '%s(%s) %s' % (name, comma_join(params), partition_sql) + return "%s(%s) %s" % (name, comma_join(params), partition_sql) def _build_sql_params(self, db): params = [] @@ -114,19 +128,35 @@ def _build_sql_params(self, db): params.append(self.date_col) if self.sampling_expr: params.append(self.sampling_expr) - params.append('(%s)' % comma_join(self.order_by, stringify=True)) + params.append("(%s)" % comma_join(self.order_by, stringify=True)) params.append(str(self.index_granularity)) return params class CollapsingMergeTree(MergeTree): - - def __init__(self, date_col=None, order_by=(), sign_col='sign', sampling_expr=None, - index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, - primary_key=None): - super(CollapsingMergeTree, self).__init__(date_col, order_by, sampling_expr, index_granularity, - replica_table_path, replica_name, partition_key, primary_key) + def __init__( + self, + date_col=None, + order_by=(), + sign_col="sign", + sampling_expr=None, + index_granularity=8192, + replica_table_path=None, + replica_name=None, + partition_key=None, + primary_key=None, + ): + super(CollapsingMergeTree, self).__init__( + date_col, + order_by, + sampling_expr, + index_granularity, + replica_table_path, + replica_name, + partition_key, + primary_key, + ) self.sign_col = sign_col def _build_sql_params(self, db): @@ -136,29 +166,61 @@ def _build_sql_params(self, db): class SummingMergeTree(MergeTree): - - def __init__(self, date_col=None, order_by=(), summing_cols=None, sampling_expr=None, - index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, - primary_key=None): - super(SummingMergeTree, self).__init__(date_col, order_by, sampling_expr, index_granularity, replica_table_path, - replica_name, partition_key, primary_key) - assert type is None or type(summing_cols) in (list, tuple), 'summing_cols must be a list or tuple' + def __init__( + self, + date_col=None, + order_by=(), + summing_cols=None, + sampling_expr=None, + index_granularity=8192, + replica_table_path=None, + replica_name=None, + partition_key=None, + primary_key=None, + ): + super(SummingMergeTree, self).__init__( + date_col, + order_by, + sampling_expr, + index_granularity, + replica_table_path, + replica_name, + partition_key, + primary_key, + ) + assert type is None or type(summing_cols) in (list, tuple), "summing_cols must be a list or tuple" self.summing_cols = summing_cols def _build_sql_params(self, db): params = super(SummingMergeTree, self)._build_sql_params(db) if self.summing_cols: - params.append('(%s)' % comma_join(self.summing_cols)) + params.append("(%s)" % comma_join(self.summing_cols)) return params class ReplacingMergeTree(MergeTree): - - def __init__(self, date_col=None, order_by=(), ver_col=None, sampling_expr=None, - index_granularity=8192, replica_table_path=None, replica_name=None, partition_key=None, - primary_key=None): - super(ReplacingMergeTree, self).__init__(date_col, order_by, sampling_expr, index_granularity, - replica_table_path, replica_name, partition_key, primary_key) + def __init__( + self, + date_col=None, + order_by=(), + ver_col=None, + sampling_expr=None, + index_granularity=8192, + replica_table_path=None, + replica_name=None, + partition_key=None, + primary_key=None, + ): + super(ReplacingMergeTree, self).__init__( + date_col, + order_by, + sampling_expr, + index_granularity, + replica_table_path, + replica_name, + partition_key, + primary_key, + ) self.ver_col = ver_col def _build_sql_params(self, db): @@ -176,8 +238,17 @@ class Buffer(Engine): """ # Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) - def __init__(self, main_model, num_layers=16, min_time=10, max_time=100, min_rows=10000, max_rows=1000000, - min_bytes=10000000, max_bytes=100000000): + def __init__( + self, + main_model, + num_layers=16, + min_time=10, + max_time=100, + min_rows=10000, + max_rows=1000000, + min_bytes=10000000, + max_bytes=100000000, + ): self.main_model = main_model self.num_layers = num_layers self.min_time = min_time @@ -190,10 +261,16 @@ def __init__(self, main_model, num_layers=16, min_time=10, max_time=100, min_row def create_table_sql(self, db): # Overriden create_table_sql example: # sql = 'ENGINE = Buffer(merge, hits, 16, 10, 100, 10000, 1000000, 10000000, 100000000)' - sql = 'ENGINE = Buffer(`%s`, `%s`, %d, %d, %d, %d, %d, %d, %d)' % ( - db.db_name, self.main_model.table_name(), self.num_layers, - self.min_time, self.max_time, self.min_rows, - self.max_rows, self.min_bytes, self.max_bytes + sql = "ENGINE = Buffer(`%s`, `%s`, %d, %d, %d, %d, %d, %d, %d)" % ( + db.db_name, + self.main_model.table_name(), + self.num_layers, + self.min_time, + self.max_time, + self.min_rows, + self.max_rows, + self.min_bytes, + self.max_bytes, ) return sql @@ -224,6 +301,7 @@ class Distributed(Engine): See full documentation here https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ """ + def __init__(self, cluster, table=None, sharding_key=None): """ - `cluster`: what cluster to access data from @@ -252,12 +330,11 @@ def table_name(self): def create_table_sql(self, db): name = self.__class__.__name__ params = self._build_sql_params(db) - return '%s(%s)' % (name, ', '.join(params)) + return "%s(%s)" % (name, ", ".join(params)) def _build_sql_params(self, db): if self.table_name is None: - raise ValueError("Cannot create {} engine: specify an underlying table".format( - self.__class__.__name__)) + raise ValueError("Cannot create {} engine: specify an underlying table".format(self.__class__.__name__)) params = ["`%s`" % p for p in [self.cluster, db.db_name, self.table_name]] if self.sharding_key: diff --git a/clickhouse_orm/migrations.py b/clickhouse_orm/migrations.py index 722bf8c..78db5ec 100644 --- a/clickhouse_orm/migrations.py +++ b/clickhouse_orm/migrations.py @@ -5,67 +5,67 @@ from .models import BufferModel, Model from .utils import get_subclass_names -logger = logging.getLogger('migrations') +logger = logging.getLogger("migrations") -class Operation(): - ''' +class Operation: + """ Base class for migration operations. - ''' + """ def apply(self, database): - raise NotImplementedError() # pragma: no cover + raise NotImplementedError() # pragma: no cover class ModelOperation(Operation): - ''' + """ Base class for migration operations that work on a specific model. - ''' + """ def __init__(self, model_class): - ''' + """ Initializer. - ''' + """ self.model_class = model_class self.table_name = model_class.table_name() def _alter_table(self, database, cmd): - ''' + """ Utility for running ALTER TABLE commands. - ''' + """ cmd = "ALTER TABLE $db.`%s` %s" % (self.table_name, cmd) logger.debug(cmd) database.raw(cmd) class CreateTable(ModelOperation): - ''' + """ A migration operation that creates a table for a given model class. - ''' + """ def apply(self, database): - logger.info(' Create table %s', self.table_name) + logger.info(" Create table %s", self.table_name) if issubclass(self.model_class, BufferModel): database.create_table(self.model_class.engine.main_model) database.create_table(self.model_class) class AlterTable(ModelOperation): - ''' + """ A migration operation that compares the table of a given model class to the model's fields, and alters the table to match the model. The operation can: - add new columns - drop obsolete columns - modify column types Default values are not altered by this operation. - ''' + """ def _get_table_fields(self, database): query = "DESC `%s`.`%s`" % (database.db_name, self.table_name) return [(row.name, row.type) for row in database.select(query)] def apply(self, database): - logger.info(' Alter table %s', self.table_name) + logger.info(" Alter table %s", self.table_name) # Note that MATERIALIZED and ALIAS fields are always at the end of the DESC, # ADD COLUMN ... AFTER doesn't affect it @@ -74,8 +74,8 @@ def apply(self, database): # Identify fields that were deleted from the model deleted_fields = set(table_fields.keys()) - set(self.model_class.fields()) for name in deleted_fields: - logger.info(' Drop column %s', name) - self._alter_table(database, 'DROP COLUMN %s' % name) + logger.info(" Drop column %s", name) + self._alter_table(database, "DROP COLUMN %s" % name) del table_fields[name] # Identify fields that were added to the model @@ -83,11 +83,11 @@ def apply(self, database): for name, field in self.model_class.fields().items(): is_regular_field = not (field.materialized or field.alias) if name not in table_fields: - logger.info(' Add column %s', name) - assert prev_name, 'Cannot add a column to the beginning of the table' - cmd = 'ADD COLUMN %s %s' % (name, field.get_sql(db=database)) + logger.info(" Add column %s", name) + assert prev_name, "Cannot add a column to the beginning of the table" + cmd = "ADD COLUMN %s %s" % (name, field.get_sql(db=database)) if is_regular_field: - cmd += ' AFTER %s' % prev_name + cmd += " AFTER %s" % prev_name self._alter_table(database, cmd) if is_regular_field: @@ -99,24 +99,27 @@ def apply(self, database): # The order of class attributes can be changed any time, so we can't count on it # Secondly, MATERIALIZED and ALIAS fields are always at the end of the DESC, so we can't expect them to save # attribute position. Watch https://github.com/Infinidat/clickhouse_orm/issues/47 - model_fields = {name: field.get_sql(with_default_expression=False, db=database) - for name, field in self.model_class.fields().items()} + model_fields = { + name: field.get_sql(with_default_expression=False, db=database) + for name, field in self.model_class.fields().items() + } for field_name, field_sql in self._get_table_fields(database): # All fields must have been created and dropped by this moment - assert field_name in model_fields, 'Model fields and table columns in disagreement' + assert field_name in model_fields, "Model fields and table columns in disagreement" if field_sql != model_fields[field_name]: - logger.info(' Change type of column %s from %s to %s', field_name, field_sql, - model_fields[field_name]) - self._alter_table(database, 'MODIFY COLUMN %s %s' % (field_name, model_fields[field_name])) + logger.info( + " Change type of column %s from %s to %s", field_name, field_sql, model_fields[field_name] + ) + self._alter_table(database, "MODIFY COLUMN %s %s" % (field_name, model_fields[field_name])) class AlterTableWithBuffer(ModelOperation): - ''' + """ A migration operation for altering a buffer table and its underlying on-disk table. The buffer table is dropped, the on-disk table is altered, and then the buffer table is re-created. - ''' + """ def apply(self, database): if issubclass(self.model_class, BufferModel): @@ -128,149 +131,152 @@ def apply(self, database): class DropTable(ModelOperation): - ''' + """ A migration operation that drops the table of a given model class. - ''' + """ def apply(self, database): - logger.info(' Drop table %s', self.table_name) + logger.info(" Drop table %s", self.table_name) database.drop_table(self.model_class) class AlterConstraints(ModelOperation): - ''' + """ A migration operation that adds new constraints from the model to the database table, and drops obsolete ones. Constraints are identified by their names, so a change in an existing constraint will not be detected unless its name was changed too. ClickHouse does not check that the constraints hold for existing data in the table. - ''' + """ def apply(self, database): - logger.info(' Alter constraints for %s', self.table_name) + logger.info(" Alter constraints for %s", self.table_name) existing = self._get_constraint_names(database) # Go over constraints in the model for constraint in self.model_class._constraints.values(): # Check if it's a new constraint if constraint.name not in existing: - logger.info(' Add constraint %s', constraint.name) - self._alter_table(database, 'ADD %s' % constraint.create_table_sql()) + logger.info(" Add constraint %s", constraint.name) + self._alter_table(database, "ADD %s" % constraint.create_table_sql()) else: existing.remove(constraint.name) # Remaining constraints in `existing` are obsolete for name in existing: - logger.info(' Drop constraint %s', name) - self._alter_table(database, 'DROP CONSTRAINT `%s`' % name) + logger.info(" Drop constraint %s", name) + self._alter_table(database, "DROP CONSTRAINT `%s`" % name) def _get_constraint_names(self, database): - ''' + """ Returns a set containing the names of existing constraints in the table. - ''' + """ import re - table_def = database.raw('SHOW CREATE TABLE $db.`%s`' % self.table_name) - matches = re.findall(r'\sCONSTRAINT\s+`?(.+?)`?\s+CHECK\s', table_def) + + table_def = database.raw("SHOW CREATE TABLE $db.`%s`" % self.table_name) + matches = re.findall(r"\sCONSTRAINT\s+`?(.+?)`?\s+CHECK\s", table_def) return set(matches) class AlterIndexes(ModelOperation): - ''' + """ A migration operation that adds new indexes from the model to the database table, and drops obsolete ones. Indexes are identified by their names, so a change in an existing index will not be detected unless its name was changed too. - ''' + """ def __init__(self, model_class, reindex=False): - ''' + """ Initializer. By default ClickHouse does not build indexes over existing data, only for new data. Passing `reindex=True` will run `OPTIMIZE TABLE` in order to build the indexes over the existing data. - ''' + """ super().__init__(model_class) self.reindex = reindex def apply(self, database): - logger.info(' Alter indexes for %s', self.table_name) + logger.info(" Alter indexes for %s", self.table_name) existing = self._get_index_names(database) logger.info(existing) # Go over indexes in the model for index in self.model_class._indexes.values(): # Check if it's a new index if index.name not in existing: - logger.info(' Add index %s', index.name) - self._alter_table(database, 'ADD %s' % index.create_table_sql()) + logger.info(" Add index %s", index.name) + self._alter_table(database, "ADD %s" % index.create_table_sql()) else: existing.remove(index.name) # Remaining indexes in `existing` are obsolete for name in existing: - logger.info(' Drop index %s', name) - self._alter_table(database, 'DROP INDEX `%s`' % name) + logger.info(" Drop index %s", name) + self._alter_table(database, "DROP INDEX `%s`" % name) # Reindex if self.reindex: - logger.info(' Build indexes on table') - database.raw('OPTIMIZE TABLE $db.`%s` FINAL' % self.table_name) + logger.info(" Build indexes on table") + database.raw("OPTIMIZE TABLE $db.`%s` FINAL" % self.table_name) def _get_index_names(self, database): - ''' + """ Returns a set containing the names of existing indexes in the table. - ''' + """ import re - table_def = database.raw('SHOW CREATE TABLE $db.`%s`' % self.table_name) - matches = re.findall(r'\sINDEX\s+`?(.+?)`?\s+', table_def) + + table_def = database.raw("SHOW CREATE TABLE $db.`%s`" % self.table_name) + matches = re.findall(r"\sINDEX\s+`?(.+?)`?\s+", table_def) return set(matches) class RunPython(Operation): - ''' + """ A migration operation that executes a Python function. - ''' + """ + def __init__(self, func): - ''' + """ Initializer. The given Python function will be called with a single argument - the Database instance to apply the migration to. - ''' + """ assert callable(func), "'func' argument must be function" self._func = func def apply(self, database): - logger.info(' Executing python operation %s', self._func.__name__) + logger.info(" Executing python operation %s", self._func.__name__) self._func(database) class RunSQL(Operation): - ''' + """ A migration operation that executes arbitrary SQL statements. - ''' + """ def __init__(self, sql): - ''' + """ Initializer. The given sql argument must be a valid SQL statement or list of statements. - ''' + """ if isinstance(sql, str): sql = [sql] assert isinstance(sql, list), "'sql' argument must be string or list of strings" self._sql = sql def apply(self, database): - logger.info(' Executing raw SQL operations') + logger.info(" Executing raw SQL operations") for item in self._sql: database.raw(item) class MigrationHistory(Model): - ''' + """ A model for storing which migrations were already applied to the containing database. - ''' + """ package_name = StringField() module_name = StringField() applied = DateField() - engine = MergeTree('applied', ('package_name', 'module_name')) + engine = MergeTree("applied", ("package_name", "module_name")) @classmethod def table_name(cls): - return 'infi_clickhouse_orm_migrations' + return "infi_clickhouse_orm_migrations" # Expose only relevant classes in import * diff --git a/clickhouse_orm/models.py b/clickhouse_orm/models.py index 3caa311..c7b1bb4 100644 --- a/clickhouse_orm/models.py +++ b/clickhouse_orm/models.py @@ -11,77 +11,77 @@ from .query import QuerySet from .utils import NO_VALUE, arg_to_sql, get_subclass_names, parse_tsv -logger = getLogger('clickhouse_orm') +logger = getLogger("clickhouse_orm") class Constraint: - ''' + """ Defines a model constraint. - ''' + """ name = None # this is set by the parent model parent = None # this is set by the parent model def __init__(self, expr): - ''' + """ Initializer. Expects an expression that ClickHouse will verify when inserting data. - ''' + """ self.expr = expr def create_table_sql(self): - ''' + """ Returns the SQL statement for defining this constraint during table creation. - ''' - return 'CONSTRAINT `%s` CHECK %s' % (self.name, arg_to_sql(self.expr)) + """ + return "CONSTRAINT `%s` CHECK %s" % (self.name, arg_to_sql(self.expr)) class Index: - ''' + """ Defines a data-skipping index. - ''' + """ name = None # this is set by the parent model parent = None # this is set by the parent model def __init__(self, expr, type, granularity): - ''' + """ Initializer. - `expr` - a column, expression, or tuple of columns and expressions to index. - `type` - the index type. Use one of the following methods to specify the type: `Index.minmax`, `Index.set`, `Index.ngrambf_v1`, `Index.tokenbf_v1` or `Index.bloom_filter`. - `granularity` - index block size (number of multiples of the `index_granularity` defined by the engine). - ''' + """ self.expr = expr self.type = type self.granularity = granularity def create_table_sql(self): - ''' + """ Returns the SQL statement for defining this index during table creation. - ''' - return 'INDEX `%s` %s TYPE %s GRANULARITY %d' % (self.name, arg_to_sql(self.expr), self.type, self.granularity) + """ + return "INDEX `%s` %s TYPE %s GRANULARITY %d" % (self.name, arg_to_sql(self.expr), self.type, self.granularity) @staticmethod def minmax(): - ''' + """ An index that stores extremes of the specified expression (if the expression is tuple, then it stores extremes for each element of tuple). The stored info is used for skipping blocks of data like the primary key. - ''' - return 'minmax' + """ + return "minmax" @staticmethod def set(max_rows): - ''' + """ An index that stores unique values of the specified expression (no more than max_rows rows, or unlimited if max_rows=0). Uses the values to check if the WHERE expression is not satisfiable on a block of data. - ''' - return 'set(%d)' % max_rows + """ + return "set(%d)" % max_rows @staticmethod def ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed): - ''' + """ An index that stores a Bloom filter containing all ngrams from a block of data. Works only with strings. Can be used for optimization of equals, like and in expressions. @@ -90,12 +90,12 @@ def ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, rando for example 256 or 512, because it can be compressed well). - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - `random_seed` — The seed for Bloom filter hash functions. - ''' - return 'ngrambf_v1(%d, %d, %d, %d)' % (n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed) + """ + return "ngrambf_v1(%d, %d, %d, %d)" % (n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed) @staticmethod def tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed): - ''' + """ An index that stores a Bloom filter containing string tokens. Tokens are sequences separated by non-alphanumeric characters. @@ -103,24 +103,24 @@ def tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_s for example 256 or 512, because it can be compressed well). - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - `random_seed` — The seed for Bloom filter hash functions. - ''' - return 'tokenbf_v1(%d, %d, %d)' % (size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed) + """ + return "tokenbf_v1(%d, %d, %d)" % (size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed) @staticmethod def bloom_filter(false_positive=0.025): - ''' + """ An index that stores a Bloom filter containing values of the index expression. - `false_positive` - the probability (between 0 and 1) of receiving a false positive response from the filter - ''' - return 'bloom_filter(%f)' % false_positive + """ + return "bloom_filter(%f)" % false_positive class ModelBase(type): - ''' + """ A metaclass for ORM models. It adds the _fields list to model classes. - ''' + """ ad_hoc_model_cache = {} @@ -168,7 +168,7 @@ def __new__(metacls, name, bases, attrs): _indexes=indexes, _writable_fields=OrderedDict([f for f in fields if not f[1].readonly]), _defaults=defaults, - _has_funcs_as_defaults=has_funcs_as_defaults + _has_funcs_as_defaults=has_funcs_as_defaults, ) model = super(ModelBase, metacls).__new__(metacls, str(name), bases, attrs) @@ -180,11 +180,11 @@ def __new__(metacls, name, bases, attrs): return model @classmethod - def create_ad_hoc_model(metacls, fields, model_name='AdHocModel'): + def create_ad_hoc_model(metacls, fields, model_name="AdHocModel"): # fields is a list of tuples (name, db_type) # Check if model exists in cache fields = list(fields) - cache_key = model_name + ' ' + str(fields) + cache_key = model_name + " " + str(fields) if cache_key in metacls.ad_hoc_model_cache: return metacls.ad_hoc_model_cache[cache_key] # Create an ad hoc model class @@ -201,58 +201,55 @@ def create_ad_hoc_field(metacls, db_type): import clickhouse_orm.fields as orm_fields # Enums - if db_type.startswith('Enum'): + if db_type.startswith("Enum"): return orm_fields.BaseEnumField.create_ad_hoc_field(db_type) # DateTime with timezone - if db_type.startswith('DateTime('): + if db_type.startswith("DateTime("): timezone = db_type[9:-1] - return orm_fields.DateTimeField( - timezone=timezone[1:-1] if timezone else None - ) + return orm_fields.DateTimeField(timezone=timezone[1:-1] if timezone else None) # DateTime64 - if db_type.startswith('DateTime64('): - precision, *timezone = [s.strip() for s in db_type[11:-1].split(',')] + if db_type.startswith("DateTime64("): + precision, *timezone = [s.strip() for s in db_type[11:-1].split(",")] return orm_fields.DateTime64Field( - precision=int(precision), - timezone=timezone[0][1:-1] if timezone else None + precision=int(precision), timezone=timezone[0][1:-1] if timezone else None ) # Arrays - if db_type.startswith('Array'): - inner_field = metacls.create_ad_hoc_field(db_type[6 : -1]) + if db_type.startswith("Array"): + inner_field = metacls.create_ad_hoc_field(db_type[6:-1]) return orm_fields.ArrayField(inner_field) # Tuples (poor man's version - convert to array) - if db_type.startswith('Tuple'): - types = [s.strip() for s in db_type[6 : -1].split(',')] - assert len(set(types)) == 1, 'No support for mixed types in tuples - ' + db_type + if db_type.startswith("Tuple"): + types = [s.strip() for s in db_type[6:-1].split(",")] + assert len(set(types)) == 1, "No support for mixed types in tuples - " + db_type inner_field = metacls.create_ad_hoc_field(types[0]) return orm_fields.ArrayField(inner_field) # FixedString - if db_type.startswith('FixedString'): - length = int(db_type[12 : -1]) + if db_type.startswith("FixedString"): + length = int(db_type[12:-1]) return orm_fields.FixedStringField(length) # Decimal / Decimal32 / Decimal64 / Decimal128 - if db_type.startswith('Decimal'): - p = db_type.index('(') - args = [int(n.strip()) for n in db_type[p + 1 : -1].split(',')] - field_class = getattr(orm_fields, db_type[:p] + 'Field') + if db_type.startswith("Decimal"): + p = db_type.index("(") + args = [int(n.strip()) for n in db_type[p + 1 : -1].split(",")] + field_class = getattr(orm_fields, db_type[:p] + "Field") return field_class(*args) # Nullable - if db_type.startswith('Nullable'): - inner_field = metacls.create_ad_hoc_field(db_type[9 : -1]) + if db_type.startswith("Nullable"): + inner_field = metacls.create_ad_hoc_field(db_type[9:-1]) return orm_fields.NullableField(inner_field) # LowCardinality - if db_type.startswith('LowCardinality'): - inner_field = metacls.create_ad_hoc_field(db_type[15 : -1]) + if db_type.startswith("LowCardinality"): + inner_field = metacls.create_ad_hoc_field(db_type[15:-1]) return orm_fields.LowCardinalityField(inner_field) # Simple fields - name = db_type + 'Field' + name = db_type + "Field" if not hasattr(orm_fields, name): - raise NotImplementedError('No field class for %s' % db_type) + raise NotImplementedError("No field class for %s" % db_type) return getattr(orm_fields, name)() class Model(metaclass=ModelBase): - ''' + """ A base class for ORM models. Each model class represent a ClickHouse table. For example: class CPUStats(Model): @@ -260,7 +257,7 @@ class CPUStats(Model): cpu_id = UInt16Field() cpu_percent = Float32Field() engine = Memory() - ''' + """ engine = None @@ -273,12 +270,12 @@ class CPUStats(Model): _database = None def __init__(self, **kwargs): - ''' + """ Creates a model instance, using keyword arguments as field values. Since values are immediately converted to their Pythonic type, invalid values will cause a `ValueError` to be raised. Unrecognized field names will cause an `AttributeError`. - ''' + """ super(Model, self).__init__() # Assign default values self.__dict__.update(self._defaults) @@ -288,13 +285,13 @@ def __init__(self, **kwargs): if field: setattr(self, name, value) else: - raise AttributeError('%s does not have a field called %s' % (self.__class__.__name__, name)) + raise AttributeError("%s does not have a field called %s" % (self.__class__.__name__, name)) def __setattr__(self, name, value): - ''' + """ When setting a field value, converts the value to its Pythonic type and validates it. This may raise a `ValueError`. - ''' + """ field = self.get_field(name) if field and (value != NO_VALUE): try: @@ -307,77 +304,78 @@ def __setattr__(self, name, value): super(Model, self).__setattr__(name, value) def set_database(self, db): - ''' + """ Sets the `Database` that this model instance belongs to. This is done automatically when the instance is read from the database or written to it. - ''' + """ # This can not be imported globally due to circular import from .database import Database + assert isinstance(db, Database), "database must be database.Database instance" self._database = db def get_database(self): - ''' + """ Gets the `Database` that this model instance belongs to. Returns `None` unless the instance was read from the database or written to it. - ''' + """ return self._database def get_field(self, name): - ''' + """ Gets a `Field` instance given its name, or `None` if not found. - ''' + """ return self._fields.get(name) @classmethod def table_name(cls): - ''' + """ Returns the model's database table name. By default this is the class name converted to lowercase. Override this if you want to use a different table name. - ''' + """ return cls.__name__.lower() @classmethod def has_funcs_as_defaults(cls): - ''' + """ Return True if some of the model's fields use a function expression as a default value. This requires special handling when inserting instances. - ''' + """ return cls._has_funcs_as_defaults @classmethod def create_table_sql(cls, db): - ''' + """ Returns the SQL statement for creating a table for this model. - ''' - parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())] + """ + parts = ["CREATE TABLE IF NOT EXISTS `%s`.`%s` (" % (db.db_name, cls.table_name())] # Fields items = [] for name, field in cls.fields().items(): - items.append(' %s %s' % (name, field.get_sql(db=db))) + items.append(" %s %s" % (name, field.get_sql(db=db))) # Constraints for c in cls._constraints.values(): - items.append(' %s' % c.create_table_sql()) + items.append(" %s" % c.create_table_sql()) # Indexes for i in cls._indexes.values(): - items.append(' %s' % i.create_table_sql()) - parts.append(',\n'.join(items)) + items.append(" %s" % i.create_table_sql()) + parts.append(",\n".join(items)) # Engine - parts.append(')') - parts.append('ENGINE = ' + cls.engine.create_table_sql(db)) - return '\n'.join(parts) + parts.append(")") + parts.append("ENGINE = " + cls.engine.create_table_sql(db)) + return "\n".join(parts) @classmethod def drop_table_sql(cls, db): - ''' + """ Returns the SQL command for deleting this model's table. - ''' - return 'DROP TABLE IF EXISTS `%s`.`%s`' % (db.db_name, cls.table_name()) + """ + return "DROP TABLE IF EXISTS `%s`.`%s`" % (db.db_name, cls.table_name()) @classmethod def from_tsv(cls, line, field_names, timezone_in_use=pytz.utc, database=None): - ''' + """ Create a model instance from a tab-separated line. The line may or may not include a newline. The `field_names` list must match the fields defined in the model, but does not have to include all of them. @@ -385,12 +383,12 @@ def from_tsv(cls, line, field_names, timezone_in_use=pytz.utc, database=None): - `field_names`: names of the model fields in the data. - `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones. - `database`: if given, sets the database that this instance belongs to. - ''' + """ values = iter(parse_tsv(line)) kwargs = {} for name in field_names: field = getattr(cls, name) - field_timezone = getattr(field, 'timezone', None) or timezone_in_use + field_timezone = getattr(field, "timezone", None) or timezone_in_use kwargs[name] = field.to_python(next(values), field_timezone) obj = cls(**kwargs) @@ -400,45 +398,45 @@ def from_tsv(cls, line, field_names, timezone_in_use=pytz.utc, database=None): return obj def to_tsv(self, include_readonly=True): - ''' + """ Returns the instance's column values as a tab-separated line. A newline is not included. - `include_readonly`: if false, returns only fields that can be inserted into database. - ''' + """ data = self.__dict__ fields = self.fields(writable=not include_readonly) - return '\t'.join(field.to_db_string(data[name], quote=False) for name, field in fields.items()) + return "\t".join(field.to_db_string(data[name], quote=False) for name, field in fields.items()) def to_tskv(self, include_readonly=True): - ''' + """ Returns the instance's column keys and values as a tab-separated line. A newline is not included. Fields that were not assigned a value are omitted. - `include_readonly`: if false, returns only fields that can be inserted into database. - ''' + """ data = self.__dict__ fields = self.fields(writable=not include_readonly) parts = [] for name, field in fields.items(): if data[name] != NO_VALUE: - parts.append(name + '=' + field.to_db_string(data[name], quote=False)) - return '\t'.join(parts) + parts.append(name + "=" + field.to_db_string(data[name], quote=False)) + return "\t".join(parts) def to_db_string(self): - ''' + """ Returns the instance as a bytestring ready to be inserted into the database. - ''' + """ s = self.to_tskv(False) if self._has_funcs_as_defaults else self.to_tsv(False) - s += '\n' - return s.encode('utf-8') + s += "\n" + return s.encode("utf-8") def to_dict(self, include_readonly=True, field_names=None): - ''' + """ Returns the instance's column values as a dict. - `include_readonly`: if false, returns only fields that can be inserted into database. - `field_names`: an iterable of field names to return (optional) - ''' + """ fields = self.fields(writable=not include_readonly) if field_names is not None: @@ -449,56 +447,58 @@ def to_dict(self, include_readonly=True, field_names=None): @classmethod def objects_in(cls, database): - ''' + """ Returns a `QuerySet` for selecting instances of this model class. - ''' + """ return QuerySet(cls, database) @classmethod def fields(cls, writable=False): - ''' + """ Returns an `OrderedDict` of the model's fields (from name to `Field` instance). If `writable` is true, only writable fields are included. Callers should not modify the dictionary. - ''' + """ # noinspection PyProtectedMember,PyUnresolvedReferences return cls._writable_fields if writable else cls._fields @classmethod def is_read_only(cls): - ''' + """ Returns true if the model is marked as read only. - ''' + """ return cls._readonly @classmethod def is_system_model(cls): - ''' + """ Returns true if the model represents a system table. - ''' + """ return cls._system class BufferModel(Model): - @classmethod def create_table_sql(cls, db): - ''' + """ Returns the SQL statement for creating a table for this model. - ''' - parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` AS `%s`.`%s`' % (db.db_name, cls.table_name(), db.db_name, - cls.engine.main_model.table_name())] + """ + parts = [ + "CREATE TABLE IF NOT EXISTS `%s`.`%s` AS `%s`.`%s`" + % (db.db_name, cls.table_name(), db.db_name, cls.engine.main_model.table_name()) + ] engine_str = cls.engine.create_table_sql(db) parts.append(engine_str) - return ' '.join(parts) + return " ".join(parts) class MergeModel(Model): - ''' + """ Model for Merge engine Predefines virtual _table column an controls that rows can't be inserted to this table type https://clickhouse.tech/docs/en/single/index.html#document-table_engines/merge - ''' + """ + readonly = True # Virtual fields can't be inserted into database @@ -506,19 +506,20 @@ class MergeModel(Model): @classmethod def create_table_sql(cls, db): - ''' + """ Returns the SQL statement for creating a table for this model. - ''' + """ assert isinstance(cls.engine, Merge), "engine must be an instance of engines.Merge" - parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db.db_name, cls.table_name())] + parts = ["CREATE TABLE IF NOT EXISTS `%s`.`%s` (" % (db.db_name, cls.table_name())] cols = [] for name, field in cls.fields().items(): - if name != '_table': - cols.append(' %s %s' % (name, field.get_sql(db=db))) - parts.append(',\n'.join(cols)) - parts.append(')') - parts.append('ENGINE = ' + cls.engine.create_table_sql(db)) - return '\n'.join(parts) + if name != "_table": + cols.append(" %s %s" % (name, field.get_sql(db=db))) + parts.append(",\n".join(cols)) + parts.append(")") + parts.append("ENGINE = " + cls.engine.create_table_sql(db)) + return "\n".join(parts) + # TODO: base class for models that require specific engine @@ -529,10 +530,10 @@ class DistributedModel(Model): """ def set_database(self, db): - ''' + """ Sets the `Database` that this model instance belongs to. This is done automatically when the instance is read from the database or written to it. - ''' + """ assert isinstance(self.engine, Distributed), "engine must be an instance of engines.Distributed" res = super(DistributedModel, self).set_database(db) return res @@ -575,33 +576,37 @@ def fix_engine_table(cls): return # find out all the superclasses of the Model that store any data - storage_models = [b for b in cls.__bases__ if issubclass(b, Model) - and not issubclass(b, DistributedModel)] + storage_models = [b for b in cls.__bases__ if issubclass(b, Model) and not issubclass(b, DistributedModel)] if not storage_models: - raise TypeError("When defining Distributed engine without the table_name " - "ensure that your model has a parent model") + raise TypeError( + "When defining Distributed engine without the table_name " "ensure that your model has a parent model" + ) if len(storage_models) > 1: - raise TypeError("When defining Distributed engine without the table_name " - "ensure that your model has exactly one non-distributed superclass") + raise TypeError( + "When defining Distributed engine without the table_name " + "ensure that your model has exactly one non-distributed superclass" + ) # enable correct SQL for engine cls.engine.table = storage_models[0] @classmethod def create_table_sql(cls, db): - ''' + """ Returns the SQL statement for creating a table for this model. - ''' + """ assert isinstance(cls.engine, Distributed), "engine must be engines.Distributed instance" cls.fix_engine_table() parts = [ - 'CREATE TABLE IF NOT EXISTS `{0}`.`{1}` AS `{0}`.`{2}`'.format( - db.db_name, cls.table_name(), cls.engine.table_name), - 'ENGINE = ' + cls.engine.create_table_sql(db)] - return '\n'.join(parts) + "CREATE TABLE IF NOT EXISTS `{0}`.`{1}` AS `{0}`.`{2}`".format( + db.db_name, cls.table_name(), cls.engine.table_name + ), + "ENGINE = " + cls.engine.create_table_sql(db), + ] + return "\n".join(parts) # Expose only relevant classes in import * diff --git a/clickhouse_orm/system_models.py b/clickhouse_orm/system_models.py index af280c7..9028a40 100644 --- a/clickhouse_orm/system_models.py +++ b/clickhouse_orm/system_models.py @@ -16,7 +16,8 @@ class SystemPart(Model): This model operates only fields, described in the reference. Other fields are ignored. https://clickhouse.tech/docs/en/system_tables/system.parts/ """ - OPERATIONS = frozenset({'DETACH', 'DROP', 'ATTACH', 'FREEZE', 'FETCH'}) + + OPERATIONS = frozenset({"DETACH", "DROP", "ATTACH", "FREEZE", "FETCH"}) _readonly = True _system = True @@ -51,12 +52,13 @@ class SystemPart(Model): @classmethod def table_name(cls): - return 'parts' + return "parts" """ Next methods return SQL for some operations, which can be done with partitions https://clickhouse.tech/docs/en/query_language/queries/#manipulations-with-partitions-and-parts """ + def _partition_operation_sql(self, operation, settings=None, from_part=None): """ Performs some operation over partition @@ -83,7 +85,7 @@ def detach(self, settings=None): Returns: SQL Query """ - return self._partition_operation_sql('DETACH', settings=settings) + return self._partition_operation_sql("DETACH", settings=settings) def drop(self, settings=None): """ @@ -93,7 +95,7 @@ def drop(self, settings=None): Returns: SQL Query """ - return self._partition_operation_sql('DROP', settings=settings) + return self._partition_operation_sql("DROP", settings=settings) def attach(self, settings=None): """ @@ -103,7 +105,7 @@ def attach(self, settings=None): Returns: SQL Query """ - return self._partition_operation_sql('ATTACH', settings=settings) + return self._partition_operation_sql("ATTACH", settings=settings) def freeze(self, settings=None): """ @@ -113,7 +115,7 @@ def freeze(self, settings=None): Returns: SQL Query """ - return self._partition_operation_sql('FREEZE', settings=settings) + return self._partition_operation_sql("FREEZE", settings=settings) def fetch(self, zookeeper_path, settings=None): """ @@ -124,7 +126,7 @@ def fetch(self, zookeeper_path, settings=None): Returns: SQL Query """ - return self._partition_operation_sql('FETCH', settings=settings, from_part=zookeeper_path) + return self._partition_operation_sql("FETCH", settings=settings, from_part=zookeeper_path) @classmethod def get(cls, database, conditions=""): @@ -140,9 +142,12 @@ def get(cls, database, conditions=""): assert isinstance(conditions, str), "conditions must be a string" if conditions: conditions += " AND" - field_names = ','.join(cls.fields()) - return database.select("SELECT %s FROM `system`.%s WHERE %s database='%s'" % - (field_names, cls.table_name(), conditions, database.db_name), model_class=cls) + field_names = ",".join(cls.fields()) + return database.select( + "SELECT %s FROM `system`.%s WHERE %s database='%s'" + % (field_names, cls.table_name(), conditions, database.db_name), + model_class=cls, + ) @classmethod def get_active(cls, database, conditions=""): @@ -155,8 +160,8 @@ def get_active(cls, database, conditions=""): Returns: A list of SystemPart objects """ if conditions: - conditions += ' AND ' - conditions += 'active' + conditions += " AND " + conditions += "active" return SystemPart.get(database, conditions=conditions) diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index 140cbed..cf9c67d 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -4,26 +4,18 @@ import re from datetime import date, datetime, tzinfo, timedelta -SPECIAL_CHARS = { - "\b" : "\\b", - "\f" : "\\f", - "\r" : "\\r", - "\n" : "\\n", - "\t" : "\\t", - "\0" : "\\0", - "\\" : "\\\\", - "'" : "\\'" -} +SPECIAL_CHARS = {"\b": "\\b", "\f": "\\f", "\r": "\\r", "\n": "\\n", "\t": "\\t", "\0": "\\0", "\\": "\\\\", "'": "\\'"} -SPECIAL_CHARS_REGEX = re.compile("[" + ''.join(SPECIAL_CHARS.values()) + "]") +SPECIAL_CHARS_REGEX = re.compile("[" + "".join(SPECIAL_CHARS.values()) + "]") def escape(value, quote=True): - ''' + """ If the value is a string, escapes any special characters and optionally surrounds it with single quotes. If the value is not a string (e.g. a number), converts it to one. - ''' + """ + def escape_one(match): return SPECIAL_CHARS[match.group(0)] @@ -35,11 +27,11 @@ def escape_one(match): def unescape(value): - return codecs.escape_decode(value)[0].decode('utf-8') + return codecs.escape_decode(value)[0].decode("utf-8") def string_or_func(obj): - return obj.to_sql() if hasattr(obj, 'to_sql') else obj + return obj.to_sql() if hasattr(obj, "to_sql") else obj def arg_to_sql(arg): @@ -49,6 +41,7 @@ def arg_to_sql(arg): None, numbers, timezones, arrays/iterables. """ from clickhouse_orm import Field, StringField, DateTimeField, F, QuerySet + if isinstance(arg, F): return arg.to_sql() if isinstance(arg, Field): @@ -66,22 +59,22 @@ def arg_to_sql(arg): if isinstance(arg, tzinfo): return StringField().to_db_string(arg.tzname(None)) if arg is None: - return 'NULL' + return "NULL" if isinstance(arg, QuerySet): return "(%s)" % arg if isinstance(arg, tuple): - return '(' + comma_join(arg_to_sql(x) for x in arg) + ')' + return "(" + comma_join(arg_to_sql(x) for x in arg) + ")" if is_iterable(arg): - return '[' + comma_join(arg_to_sql(x) for x in arg) + ']' + return "[" + comma_join(arg_to_sql(x) for x in arg) + "]" return str(arg) def parse_tsv(line): if isinstance(line, bytes): line = line.decode() - if line and line[-1] == '\n': + if line and line[-1] == "\n": line = line[:-1] - return [unescape(value) for value in line.split(str('\t'))] + return [unescape(value) for value in line.split(str("\t"))] def parse_array(array_string): @@ -91,17 +84,17 @@ def parse_array(array_string): "(1,2,3)" ==> [1, 2, 3] """ # Sanity check - if len(array_string) < 2 or array_string[0] not in '[(' or array_string[-1] not in '])': + if len(array_string) < 2 or array_string[0] not in "[(" or array_string[-1] not in "])": raise ValueError('Invalid array string: "%s"' % array_string) # Drop opening brace array_string = array_string[1:] # Go over the string, lopping off each value at the beginning until nothing is left values = [] while True: - if array_string in '])': + if array_string in "])": # End of array return values - elif array_string[0] in ', ': + elif array_string[0] in ", ": # In between values array_string = array_string[1:] elif array_string[0] == "'": @@ -110,12 +103,12 @@ def parse_array(array_string): if match is None: raise ValueError('Missing closing quote: "%s"' % array_string) values.append(array_string[1 : match.start() + 1]) - array_string = array_string[match.end():] + array_string = array_string[match.end() :] else: # Start of non-quoted value, find its end match = re.search(r",|\]", array_string) values.append(array_string[0 : match.start()]) - array_string = array_string[match.end() - 1:] + array_string = array_string[match.end() - 1 :] def import_submodules(package_name): @@ -124,7 +117,7 @@ def import_submodules(package_name): """ package = importlib.import_module(package_name) return { - name: importlib.import_module(package_name + '.' + name) + name: importlib.import_module(package_name + "." + name) for _, name, _ in pkgutil.iter_modules(package.__path__) } @@ -134,9 +127,9 @@ def comma_join(items, stringify=False): Joins an iterable of strings with commas. """ if stringify: - return ', '.join(str(item) for item in items) + return ", ".join(str(item) for item in items) else: - return ', '.join(items) + return ", ".join(items) def is_iterable(obj): @@ -152,16 +145,18 @@ def is_iterable(obj): def get_subclass_names(locals, base_class): from inspect import isclass + return [c.__name__ for c in locals.values() if isclass(c) and issubclass(c, base_class)] class NoValue: - ''' + """ A sentinel for fields with an expression for a default value, that were not assigned a value yet. - ''' + """ + def __repr__(self): - return 'NO_VALUE' + return "NO_VALUE" NO_VALUE = NoValue() From f2eb81371abe7e3815c717ae8874c3551118f2d1 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:18:38 +0100 Subject: [PATCH 13/51] Chore: isort --- clickhouse_orm/database.py | 13 ++++++------- clickhouse_orm/engines.py | 2 -- clickhouse_orm/fields.py | 2 -- clickhouse_orm/query.py | 7 +++---- clickhouse_orm/system_models.py | 2 -- clickhouse_orm/utils.py | 4 ++-- pyproject.toml | 3 +++ 7 files changed, 14 insertions(+), 19 deletions(-) diff --git a/clickhouse_orm/database.py b/clickhouse_orm/database.py index 3738b62..ec9407c 100644 --- a/clickhouse_orm/database.py +++ b/clickhouse_orm/database.py @@ -1,16 +1,15 @@ -from __future__ import unicode_literals - +import datetime +import logging import re -import requests from collections import namedtuple -from .models import ModelBase -from .utils import parse_tsv, import_submodules from math import ceil -import datetime from string import Template + import pytz +import requests -import logging +from .models import ModelBase +from .utils import import_submodules, parse_tsv logger = logging.getLogger("clickhouse_orm") diff --git a/clickhouse_orm/engines.py b/clickhouse_orm/engines.py index 5fef48f..8dfda54 100644 --- a/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import logging from .utils import comma_join, get_subclass_names diff --git a/clickhouse_orm/fields.py b/clickhouse_orm/fields.py index a890385..31c36eb 100644 --- a/clickhouse_orm/fields.py +++ b/clickhouse_orm/fields.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import datetime from calendar import timegm from decimal import Decimal, localcontext diff --git a/clickhouse_orm/query.py b/clickhouse_orm/query.py index d9be1be..885feb1 100644 --- a/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -1,10 +1,9 @@ -from __future__ import unicode_literals - -import pytz from copy import copy, deepcopy from math import ceil -from .utils import comma_join, string_or_func, arg_to_sql +import pytz + +from .utils import arg_to_sql, comma_join, string_or_func # TODO # - check that field names are valid diff --git a/clickhouse_orm/system_models.py b/clickhouse_orm/system_models.py index 9028a40..dbd2952 100644 --- a/clickhouse_orm/system_models.py +++ b/clickhouse_orm/system_models.py @@ -2,8 +2,6 @@ This file contains system readonly models that can be got from the database https://clickhouse.tech/docs/en/system_tables/ """ -from __future__ import unicode_literals - from .database import Database from .fields import DateTimeField, StringField, UInt8Field, UInt32Field, UInt64Field from .models import Model diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index cf9c67d..8be51f3 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -2,7 +2,7 @@ import importlib import pkgutil import re -from datetime import date, datetime, tzinfo, timedelta +from datetime import date, datetime, timedelta, tzinfo SPECIAL_CHARS = {"\b": "\\b", "\f": "\\f", "\r": "\\r", "\n": "\\n", "\t": "\\t", "\0": "\\0", "\\": "\\\\", "'": "\\'"} @@ -40,7 +40,7 @@ def arg_to_sql(arg): Supports functions, model fields, strings, dates, datetimes, timedeltas, booleans, None, numbers, timezones, arrays/iterables. """ - from clickhouse_orm import Field, StringField, DateTimeField, F, QuerySet + from clickhouse_orm import DateTimeField, F, Field, QuerySet, StringField if isinstance(arg, F): return arg.to_sql() diff --git a/pyproject.toml b/pyproject.toml index e5ea5a8..397130e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,9 @@ flake8 = "^3.9.2" flake8-bugbear = "^21.4.3" pep8-naming = "^0.12.0" pytest = "^6.2.4" +flake8-isort = "^4.0.0" +black = "^21.7b0" +isort = "^5.9.2" [build-system] requires = ["poetry-core>=1.0.0"] From 9ccaacc6409350603c9d311b32a20d169732d607 Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:19:15 +0100 Subject: [PATCH 14/51] Chore: blacken tests --- tests/base_test_with_data.py | 17 +- tests/sample_migrations/0001_initial.py | 4 +- tests/sample_migrations/0002.py | 4 +- tests/sample_migrations/0003.py | 4 +- tests/sample_migrations/0004.py | 4 +- tests/sample_migrations/0005.py | 4 +- tests/sample_migrations/0006.py | 4 +- tests/sample_migrations/0007.py | 4 +- tests/sample_migrations/0008.py | 4 +- tests/sample_migrations/0009.py | 4 +- tests/sample_migrations/0010.py | 4 +- tests/sample_migrations/0011.py | 4 +- tests/sample_migrations/0012.py | 10 +- tests/sample_migrations/0013.py | 8 +- tests/sample_migrations/0014.py | 5 +- tests/sample_migrations/0015.py | 5 +- tests/sample_migrations/0016.py | 4 +- tests/sample_migrations/0017.py | 4 +- tests/sample_migrations/0018.py | 4 +- tests/sample_migrations/0019.py | 4 +- tests/test_alias_fields.py | 29 +- tests/test_array_fields.py | 28 +- tests/test_buffer.py | 1 - tests/test_compressed_fields.py | 125 ++++--- tests/test_constraints.py | 29 +- tests/test_custom_fields.py | 25 +- tests/test_database.py | 92 ++--- tests/test_datetime_fields.py | 103 +++--- tests/test_decimal_fields.py | 79 ++--- tests/test_dictionaries.py | 67 ++-- tests/test_engines.py | 254 ++++++++------ tests/test_enum_fields.py | 46 +-- tests/test_fixed_string_fields.py | 35 +- tests/test_funcs.py | 429 +++++++++++++----------- tests/test_indexes.py | 7 +- tests/test_inheritance.py | 21 +- tests/test_ip_fields.py | 27 +- tests/test_join.py | 16 +- tests/test_materialized_fields.py | 29 +- tests/test_migrations.py | 263 ++++++++------- tests/test_models.py | 78 ++--- tests/test_mutations.py | 16 +- tests/test_nullable_fields.py | 69 ++-- tests/test_querysets.py | 348 +++++++++++-------- tests/test_readonly.py | 27 +- tests/test_server_errors.py | 22 +- tests/test_simple_fields.py | 63 ++-- tests/test_system_models.py | 15 +- tests/test_uuid_fields.py | 19 +- 49 files changed, 1327 insertions(+), 1140 deletions(-) diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index f48d11b..6530681 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -7,13 +7,13 @@ from clickhouse_orm.engines import * import logging + logging.getLogger("requests").setLevel(logging.WARNING) class TestCaseWithData(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(Person) def tearDown(self): @@ -35,7 +35,6 @@ def _sample_data(self): yield Person(**entry) - class Person(Model): first_name = StringField() @@ -44,16 +43,12 @@ class Person(Model): height = Float32Field() passport = NullableField(UInt32Field()) - engine = MergeTree('birthday', ('first_name', 'last_name', 'birthday')) + engine = MergeTree("birthday", ("first_name", "last_name", "birthday")) data = [ - {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63", - "passport": 35052255}, - - {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74", - "passport": 36052255}, - + {"first_name": "Abdul", "last_name": "Hester", "birthday": "1970-12-02", "height": "1.63", "passport": 35052255}, + {"first_name": "Adam", "last_name": "Goodman", "birthday": "1986-01-07", "height": "1.74", "passport": 36052255}, {"first_name": "Adena", "last_name": "Norman", "birthday": "1979-05-14", "height": "1.66"}, {"first_name": "Aline", "last_name": "Crane", "birthday": "1988-05-01", "height": "1.62"}, {"first_name": "Althea", "last_name": "Barrett", "birthday": "2004-07-28", "height": "1.71"}, @@ -151,5 +146,5 @@ class Person(Model): {"first_name": "Whitney", "last_name": "Durham", "birthday": "1977-09-15", "height": "1.72"}, {"first_name": "Whitney", "last_name": "Scott", "birthday": "1971-07-04", "height": "1.70"}, {"first_name": "Wynter", "last_name": "Garcia", "birthday": "1975-01-10", "height": "1.69"}, - {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1997-02-25", "height": "1.74"} + {"first_name": "Yolanda", "last_name": "Duke", "birthday": "1997-02-25", "height": "1.74"}, ] diff --git a/tests/sample_migrations/0001_initial.py b/tests/sample_migrations/0001_initial.py index c06bac9..920fa16 100644 --- a/tests/sample_migrations/0001_initial.py +++ b/tests/sample_migrations/0001_initial.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(Model1) -] +operations = [migrations.CreateTable(Model1)] diff --git a/tests/sample_migrations/0002.py b/tests/sample_migrations/0002.py index 2904c12..d289805 100644 --- a/tests/sample_migrations/0002.py +++ b/tests/sample_migrations/0002.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.DropTable(Model1) -] +operations = [migrations.DropTable(Model1)] diff --git a/tests/sample_migrations/0003.py b/tests/sample_migrations/0003.py index c06bac9..920fa16 100644 --- a/tests/sample_migrations/0003.py +++ b/tests/sample_migrations/0003.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(Model1) -] +operations = [migrations.CreateTable(Model1)] diff --git a/tests/sample_migrations/0004.py b/tests/sample_migrations/0004.py index 7a3322d..3ae6701 100644 --- a/tests/sample_migrations/0004.py +++ b/tests/sample_migrations/0004.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTable(Model2) -] +operations = [migrations.AlterTable(Model2)] diff --git a/tests/sample_migrations/0005.py b/tests/sample_migrations/0005.py index 95e1950..e938ee4 100644 --- a/tests/sample_migrations/0005.py +++ b/tests/sample_migrations/0005.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTable(Model3) -] +operations = [migrations.AlterTable(Model3)] diff --git a/tests/sample_migrations/0006.py b/tests/sample_migrations/0006.py index cb8299b..ec1a204 100644 --- a/tests/sample_migrations/0006.py +++ b/tests/sample_migrations/0006.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(EnumModel1) -] +operations = [migrations.CreateTable(EnumModel1)] diff --git a/tests/sample_migrations/0007.py b/tests/sample_migrations/0007.py index 1db30f7..2138fbb 100644 --- a/tests/sample_migrations/0007.py +++ b/tests/sample_migrations/0007.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTable(EnumModel2) -] +operations = [migrations.AlterTable(EnumModel2)] diff --git a/tests/sample_migrations/0008.py b/tests/sample_migrations/0008.py index 5631567..a452642 100644 --- a/tests/sample_migrations/0008.py +++ b/tests/sample_migrations/0008.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(MaterializedModel) -] +operations = [migrations.CreateTable(MaterializedModel)] diff --git a/tests/sample_migrations/0009.py b/tests/sample_migrations/0009.py index 22b53a0..65731ca 100644 --- a/tests/sample_migrations/0009.py +++ b/tests/sample_migrations/0009.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(AliasModel) -] +operations = [migrations.CreateTable(AliasModel)] diff --git a/tests/sample_migrations/0010.py b/tests/sample_migrations/0010.py index c01c00a..81c53c3 100644 --- a/tests/sample_migrations/0010.py +++ b/tests/sample_migrations/0010.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(Model4Buffer) -] +operations = [migrations.CreateTable(Model4Buffer)] diff --git a/tests/sample_migrations/0011.py b/tests/sample_migrations/0011.py index 0d88556..1bee2fe 100644 --- a/tests/sample_migrations/0011.py +++ b/tests/sample_migrations/0011.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTableWithBuffer(Model4Buffer_changed) -] +operations = [migrations.AlterTableWithBuffer(Model4Buffer_changed)] diff --git a/tests/sample_migrations/0012.py b/tests/sample_migrations/0012.py index 0120a5c..66965bb 100644 --- a/tests/sample_migrations/0012.py +++ b/tests/sample_migrations/0012.py @@ -2,8 +2,10 @@ operations = [ migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "), - migrations.RunSQL([ - "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ", - "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ", - ]) + migrations.RunSQL( + [ + "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ", + "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ", + ] + ), ] diff --git a/tests/sample_migrations/0013.py b/tests/sample_migrations/0013.py index 8bb39b9..ade30c5 100644 --- a/tests/sample_migrations/0013.py +++ b/tests/sample_migrations/0013.py @@ -5,11 +5,7 @@ def forward(database): - database.insert([ - Model3(date=datetime.date(2016, 1, 4), f1=4, f3=1, f4='test4') - ]) + database.insert([Model3(date=datetime.date(2016, 1, 4), f1=4, f3=1, f4="test4")]) -operations = [ - migrations.RunPython(forward) -] +operations = [migrations.RunPython(forward)] diff --git a/tests/sample_migrations/0014.py b/tests/sample_migrations/0014.py index 7cbcc32..47b3a12 100644 --- a/tests/sample_migrations/0014.py +++ b/tests/sample_migrations/0014.py @@ -1,7 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTable(MaterializedModel1), - migrations.AlterTable(AliasModel1) -] +operations = [migrations.AlterTable(MaterializedModel1), migrations.AlterTable(AliasModel1)] diff --git a/tests/sample_migrations/0015.py b/tests/sample_migrations/0015.py index de75d76..02c5d15 100644 --- a/tests/sample_migrations/0015.py +++ b/tests/sample_migrations/0015.py @@ -1,7 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterTable(Model4_compressed), - migrations.AlterTable(Model2LowCardinality) -] +operations = [migrations.AlterTable(Model4_compressed), migrations.AlterTable(Model2LowCardinality)] diff --git a/tests/sample_migrations/0016.py b/tests/sample_migrations/0016.py index 21387d1..6c11dfc 100644 --- a/tests/sample_migrations/0016.py +++ b/tests/sample_migrations/0016.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(ModelWithConstraints) -] +operations = [migrations.CreateTable(ModelWithConstraints)] diff --git a/tests/sample_migrations/0017.py b/tests/sample_migrations/0017.py index 1a1089c..4fb2e8f 100644 --- a/tests/sample_migrations/0017.py +++ b/tests/sample_migrations/0017.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterConstraints(ModelWithConstraints2) -] +operations = [migrations.AlterConstraints(ModelWithConstraints2)] diff --git a/tests/sample_migrations/0018.py b/tests/sample_migrations/0018.py index 97f52a5..0c52095 100644 --- a/tests/sample_migrations/0018.py +++ b/tests/sample_migrations/0018.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.CreateTable(ModelWithIndex) -] +operations = [migrations.CreateTable(ModelWithIndex)] diff --git a/tests/sample_migrations/0019.py b/tests/sample_migrations/0019.py index ede957f..328f5e7 100644 --- a/tests/sample_migrations/0019.py +++ b/tests/sample_migrations/0019.py @@ -1,6 +1,4 @@ from clickhouse_orm import migrations from ..test_migrations import * -operations = [ - migrations.AlterIndexes(ModelWithIndex2, reindex=True) -] +operations = [migrations.AlterIndexes(ModelWithIndex2, reindex=True)] diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index 8ddcc3c..52da31e 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -9,24 +9,21 @@ class AliasFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(ModelWithAliasFields) def tearDown(self): self.database.drop_database() def test_insert_and_select(self): - instance = ModelWithAliasFields( - date_field='2016-08-30', - int_field=-10, - str_field='TEST' - ) + instance = ModelWithAliasFields(date_field="2016-08-30", int_field=-10, str_field="TEST") self.database.insert([instance]) # We can't select * from table, as it doesn't select materialized and alias fields - query = 'SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str, alias_func' \ - ' FROM $db.%s ORDER BY alias_date' % ModelWithAliasFields.table_name() + query = ( + "SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str, alias_func" + " FROM $db.%s ORDER BY alias_date" % ModelWithAliasFields.table_name() + ) for model_cls in (ModelWithAliasFields, None): results = list(self.database.select(query, model_cls)) self.assertEqual(len(results), 1) @@ -41,7 +38,7 @@ def test_insert_and_select(self): def test_assignment_error(self): # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. instance = ModelWithAliasFields() - for value in ('x', [date.today()], ['aaa'], [None]): + for value in ("x", [date.today()], ["aaa"], [None]): with self.assertRaises(ValueError): instance.alias_date = value @@ -51,10 +48,10 @@ def test_wrong_field(self): def test_duplicate_default(self): with self.assertRaises(AssertionError): - StringField(alias='str_field', default='with default') + StringField(alias="str_field", default="with default") with self.assertRaises(AssertionError): - StringField(alias='str_field', materialized='str_field') + StringField(alias="str_field", materialized="str_field") def test_default_value(self): instance = ModelWithAliasFields() @@ -70,9 +67,9 @@ class ModelWithAliasFields(Model): date_field = DateField() str_field = StringField() - alias_str = StringField(alias=u'str_field') - alias_int = Int32Field(alias='int_field') - alias_date = DateField(alias='date_field') + alias_str = StringField(alias=u"str_field") + alias_int = Int32Field(alias="int_field") + alias_date = DateField(alias="date_field") alias_func = Int32Field(alias=F.toYYYYMM(date_field)) - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index 20972be..ea92644 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -8,9 +8,8 @@ class ArrayFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(ModelWithArrays) def tearDown(self): @@ -18,12 +17,12 @@ def tearDown(self): def test_insert_and_select(self): instance = ModelWithArrays( - date_field='2016-08-30', - arr_str=['goodbye,', 'cruel', 'world', 'special chars: ,"\\\'` \n\t\\[]'], - arr_date=['2010-01-01'], + date_field="2016-08-30", + arr_str=["goodbye,", "cruel", "world", "special chars: ,\"\\'` \n\t\\[]"], + arr_date=["2010-01-01"], ) self.database.insert([instance]) - query = 'SELECT * from $db.modelwitharrays ORDER BY date_field' + query = "SELECT * from $db.modelwitharrays ORDER BY date_field" for model_cls in (ModelWithArrays, None): results = list(self.database.select(query, model_cls)) self.assertEqual(len(results), 1) @@ -32,32 +31,25 @@ def test_insert_and_select(self): self.assertEqual(results[0].arr_date, instance.arr_date) def test_conversion(self): - instance = ModelWithArrays( - arr_int=('1', '2', '3'), - arr_date=['2010-01-01'] - ) + instance = ModelWithArrays(arr_int=("1", "2", "3"), arr_date=["2010-01-01"]) self.assertEqual(instance.arr_str, []) self.assertEqual(instance.arr_int, [1, 2, 3]) self.assertEqual(instance.arr_date, [date(2010, 1, 1)]) def test_assignment_error(self): instance = ModelWithArrays() - for value in (7, 'x', [date.today()], ['aaa'], [None]): + for value in (7, "x", [date.today()], ["aaa"], [None]): with self.assertRaises(ValueError): instance.arr_int = value def test_parse_array(self): from clickhouse_orm.utils import parse_array, unescape + self.assertEqual(parse_array("[]"), []) self.assertEqual(parse_array("[1, 2, 395, -44]"), ["1", "2", "395", "-44"]) self.assertEqual(parse_array("['big','mouse','','!']"), ["big", "mouse", "", "!"]) self.assertEqual(parse_array(unescape("['\\r\\n\\0\\t\\b']")), ["\r\n\0\t\b"]) - for s in ("", - "[", - "]", - "[1, 2", - "3, 4]", - "['aaa', 'aaa]"): + for s in ("", "[", "]", "[1, 2", "3, 4]", "['aaa', 'aaa]"): with self.assertRaises(ValueError): parse_array(s) @@ -74,4 +66,4 @@ class ModelWithArrays(Model): arr_int = ArrayField(Int32Field()) arr_date = ArrayField(DateField()) - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 71dafea..24be02e 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -7,7 +7,6 @@ class BufferTestCase(TestCaseWithData): - def _insert_and_check_buffer(self, data, count): self.database.insert(data) self.assertEqual(count, self.database.count(PersonBuffer)) diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index 828a57c..b514853 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -10,9 +10,8 @@ class CompressedFieldsTestCase(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(CompressedModel) def tearDown(self): @@ -24,7 +23,7 @@ def test_defaults(self): self.database.insert([instance]) self.assertEqual(instance.date_field, datetime.date(1970, 1, 1)) self.assertEqual(instance.datetime_field, datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)) - self.assertEqual(instance.string_field, 'dozo') + self.assertEqual(instance.string_field, "dozo") self.assertEqual(instance.int64_field, 42) self.assertEqual(instance.float_field, 0) self.assertEqual(instance.nullable_field, None) @@ -36,11 +35,11 @@ def test_assignment(self): uint64_field=217, date_field=datetime.date(1973, 12, 6), datetime_field=datetime.datetime(2000, 5, 24, 10, 22, tzinfo=pytz.utc), - string_field='aloha', + string_field="aloha", int64_field=-50, float_field=3.14, nullable_field=-2.718281, - array_field=['123456789123456','','a'] + array_field=["123456789123456", "", "a"], ) instance = CompressedModel(**kwargs) self.database.insert([instance]) @@ -49,75 +48,91 @@ def test_assignment(self): def test_string_conversion(self): # Check field conversion from string during construction - instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', nullable_field=None, array_field='[a,b,c]') + instance = CompressedModel( + date_field="1973-12-06", int64_field="100", float_field="7", nullable_field=None, array_field="[a,b,c]" + ) self.assertEqual(instance.date_field, datetime.date(1973, 12, 6)) self.assertEqual(instance.int64_field, 100) self.assertEqual(instance.float_field, 7) self.assertEqual(instance.nullable_field, None) - self.assertEqual(instance.array_field, ['a', 'b', 'c']) + self.assertEqual(instance.array_field, ["a", "b", "c"]) # Check field conversion from string during assignment - instance.int64_field = '99' + instance.int64_field = "99" self.assertEqual(instance.int64_field, 99) def test_to_dict(self): - instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', array_field='[a,b,c]') - self.assertDictEqual(instance.to_dict(), { - "date_field": datetime.date(1973, 12, 6), - "int64_field": 100, - "float_field": 7.0, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), - "alias_field": NO_VALUE, - 'string_field': 'dozo', - 'nullable_field': None, - 'uint64_field': 0, - 'array_field': ['a','b','c'] - }) - self.assertDictEqual(instance.to_dict(include_readonly=False), { - "date_field": datetime.date(1973, 12, 6), - "int64_field": 100, - "float_field": 7.0, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), - 'string_field': 'dozo', - 'nullable_field': None, - 'uint64_field': 0, - 'array_field': ['a', 'b', 'c'] - }) + instance = CompressedModel(date_field="1973-12-06", int64_field="100", float_field="7", array_field="[a,b,c]") + self.assertDictEqual( + instance.to_dict(), + { + "date_field": datetime.date(1973, 12, 6), + "int64_field": 100, + "float_field": 7.0, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + "alias_field": NO_VALUE, + "string_field": "dozo", + "nullable_field": None, + "uint64_field": 0, + "array_field": ["a", "b", "c"], + }, + ) self.assertDictEqual( - instance.to_dict(include_readonly=False, field_names=('int64_field', 'alias_field', 'datetime_field')), { + instance.to_dict(include_readonly=False), + { + "date_field": datetime.date(1973, 12, 6), "int64_field": 100, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc) - }) + "float_field": 7.0, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + "string_field": "dozo", + "nullable_field": None, + "uint64_field": 0, + "array_field": ["a", "b", "c"], + }, + ) + self.assertDictEqual( + instance.to_dict(include_readonly=False, field_names=("int64_field", "alias_field", "datetime_field")), + {"int64_field": 100, "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)}, + ) def test_confirm_compression_codec(self): if self.database.server_version < (19, 17): - raise unittest.SkipTest('ClickHouse version too old') - instance = CompressedModel(date_field='1973-12-06', int64_field='100', float_field='7', array_field='[a,b,c]') + raise unittest.SkipTest("ClickHouse version too old") + instance = CompressedModel(date_field="1973-12-06", int64_field="100", float_field="7", array_field="[a,b,c]") self.database.insert([instance]) - r = self.database.raw("select name, compression_codec from system.columns where table = '{}' and database='{}' FORMAT TabSeparatedWithNamesAndTypes".format(instance.table_name(), self.database.db_name)) + r = self.database.raw( + "select name, compression_codec from system.columns where table = '{}' and database='{}' FORMAT TabSeparatedWithNamesAndTypes".format( + instance.table_name(), self.database.db_name + ) + ) lines = r.splitlines() field_names = parse_tsv(lines[0]) field_types = parse_tsv(lines[1]) data = [tuple(parse_tsv(line)) for line in lines[2:]] - self.assertListEqual(data, [('uint64_field', 'CODEC(ZSTD(10))'), - ('datetime_field', 'CODEC(Delta(4), ZSTD(1))'), - ('date_field', 'CODEC(Delta(4), ZSTD(22))'), - ('int64_field', 'CODEC(LZ4)'), - ('string_field', 'CODEC(LZ4HC(10))'), - ('nullable_field', 'CODEC(ZSTD(1))'), - ('array_field', 'CODEC(Delta(2), LZ4HC(0))'), - ('float_field', 'CODEC(NONE)'), - ('alias_field', 'CODEC(ZSTD(4))')]) + self.assertListEqual( + data, + [ + ("uint64_field", "CODEC(ZSTD(10))"), + ("datetime_field", "CODEC(Delta(4), ZSTD(1))"), + ("date_field", "CODEC(Delta(4), ZSTD(22))"), + ("int64_field", "CODEC(LZ4)"), + ("string_field", "CODEC(LZ4HC(10))"), + ("nullable_field", "CODEC(ZSTD(1))"), + ("array_field", "CODEC(Delta(2), LZ4HC(0))"), + ("float_field", "CODEC(NONE)"), + ("alias_field", "CODEC(ZSTD(4))"), + ], + ) class CompressedModel(Model): - uint64_field = UInt64Field(codec='ZSTD(10)') - datetime_field = DateTimeField(codec='Delta,ZSTD') - date_field = DateField(codec='Delta(4),ZSTD(22)') - int64_field = Int64Field(default=42, codec='LZ4') - string_field = StringField(default='dozo', codec='LZ4HC(10)') - nullable_field = NullableField(Float32Field(), codec='ZSTD') - array_field = ArrayField(FixedStringField(length=15), codec='Delta(2),LZ4HC') - float_field = Float32Field(codec='NONE') - alias_field = Float32Field(alias='float_field', codec='ZSTD(4)') + uint64_field = UInt64Field(codec="ZSTD(10)") + datetime_field = DateTimeField(codec="Delta,ZSTD") + date_field = DateField(codec="Delta(4),ZSTD(22)") + int64_field = Int64Field(default=42, codec="LZ4") + string_field = StringField(default="dozo", codec="LZ4HC(10)") + nullable_field = NullableField(Float32Field(), codec="ZSTD") + array_field = ArrayField(FixedStringField(length=15), codec="Delta(2),LZ4HC") + float_field = Float32Field(codec="NONE") + alias_field = Float32Field(alias="float_field", codec="ZSTD(4)") - engine = MergeTree('datetime_field', ('uint64_field', 'datetime_field')) + engine = MergeTree("datetime_field", ("uint64_field", "datetime_field")) diff --git a/tests/test_constraints.py b/tests/test_constraints.py index b66b54a..a849c93 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -5,40 +5,37 @@ class ConstraintsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) if self.database.server_version < (19, 14, 3, 3): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") self.database.create_table(PersonWithConstraints) def tearDown(self): self.database.drop_database() def test_insert_valid_values(self): - self.database.insert([ - PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2000-01-01", height=1.66) - ]) + self.database.insert( + [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2000-01-01", height=1.66)] + ) def test_insert_invalid_values(self): with self.assertRaises(ServerError) as e: - self.database.insert([ - PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2100-01-01", height=1.66) - ]) + self.database.insert( + [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2100-01-01", height=1.66)] + ) self.assertEqual(e.code, 469) - self.assertTrue('Constraint `birthday_in_the_past`' in e.message) + self.assertTrue("Constraint `birthday_in_the_past`" in e.message) with self.assertRaises(ServerError) as e: - self.database.insert([ - PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="1970-01-01", height=3) - ]) + self.database.insert( + [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="1970-01-01", height=3)] + ) self.assertEqual(e.code, 469) - self.assertTrue('Constraint `max_height`' in e.message) + self.assertTrue("Constraint `max_height`" in e.message) class PersonWithConstraints(Person): birthday_in_the_past = Constraint(Person.birthday <= F.today()) max_height = Constraint(Person.height <= 2.75) - - diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py index fee6730..26e82d8 100644 --- a/tests/test_custom_fields.py +++ b/tests/test_custom_fields.py @@ -6,9 +6,8 @@ class CustomFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) def tearDown(self): self.database.drop_database() @@ -19,15 +18,18 @@ class TestModel(Model): i = Int16Field() f = BooleanField() engine = Memory() + self.database.create_table(TestModel) # Check valid values - for index, value in enumerate([1, '1', True, 0, '0', False]): + for index, value in enumerate([1, "1", True, 0, "0", False]): rec = TestModel(i=index, f=value) self.database.insert([rec]) - self.assertEqual([rec.f for rec in TestModel.objects_in(self.database).order_by('i')], - [True, True, True, False, False, False]) + self.assertEqual( + [rec.f for rec in TestModel.objects_in(self.database).order_by("i")], + [True, True, True, False, False, False], + ) # Check invalid values - for value in [None, 'zzz', -5, 7]: + for value in [None, "zzz", -5, 7]: with self.assertRaises(ValueError): TestModel(i=1, f=value) @@ -35,21 +37,20 @@ class TestModel(Model): class BooleanField(Field): # The ClickHouse column type to use - db_type = 'UInt8' + db_type = "UInt8" # The default value if empty class_default = False def to_python(self, value, timezone_in_use): # Convert valid values to bool - if value in (1, '1', True): + if value in (1, "1", True): return True - elif value in (0, '0', False): + elif value in (0, "0", False): return False else: - raise ValueError('Invalid value for BooleanField: %r' % value) + raise ValueError("Invalid value for BooleanField: %r" % value) def to_db_string(self, value, quote=True): # The value was already converted by to_python, so it's a bool - return '1' if value else '0' - + return "1" if value else "0" diff --git a/tests/test_database.py b/tests/test_database.py index d66e23c..b3c1ca6 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -12,7 +12,6 @@ class DatabaseTestCase(TestCaseWithData): - def test_insert__generator(self): self._insert_and_check(self._sample_data(), len(data)) @@ -33,17 +32,19 @@ def test_insert__medium_batches(self): def test_insert__funcs_as_default_values(self): if self.database.server_version < (20, 1, 2, 4): - raise unittest.SkipTest('Buggy in server versions before 20.1.2.4') + raise unittest.SkipTest("Buggy in server versions before 20.1.2.4") + class TestModel(Model): a = DateTimeField(default=datetime.datetime(2020, 1, 1)) b = DateField(default=F.toDate(a)) c = Int32Field(default=7) d = Int32Field(default=c * 5) engine = Memory() + self.database.create_table(TestModel) self.database.insert([TestModel()]) t = TestModel.objects_in(self.database)[0] - self.assertEqual(str(t.b), '2020-01-01') + self.assertEqual(str(t.b), "2020-01-01") self.assertEqual(t.d, 35) def test_count(self): @@ -63,9 +64,9 @@ def test_select(self): query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" results = list(self.database.select(query, Person)) self.assertEqual(len(results), 2) - self.assertEqual(results[0].last_name, 'Durham') + self.assertEqual(results[0].last_name, "Durham") self.assertEqual(results[0].height, 1.72) - self.assertEqual(results[1].last_name, 'Scott') + self.assertEqual(results[1].last_name, "Scott") self.assertEqual(results[1].height, 1.70) self.assertEqual(results[0].get_database(), self.database) self.assertEqual(results[1].get_database(), self.database) @@ -79,10 +80,10 @@ def test_select_partial_fields(self): query = "SELECT first_name, last_name FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" results = list(self.database.select(query, Person)) self.assertEqual(len(results), 2) - self.assertEqual(results[0].last_name, 'Durham') - self.assertEqual(results[0].height, 0) # default value - self.assertEqual(results[1].last_name, 'Scott') - self.assertEqual(results[1].height, 0) # default value + self.assertEqual(results[0].last_name, "Durham") + self.assertEqual(results[0].height, 0) # default value + self.assertEqual(results[1].last_name, "Scott") + self.assertEqual(results[1].height, 0) # default value self.assertEqual(results[0].get_database(), self.database) self.assertEqual(results[1].get_database(), self.database) @@ -91,10 +92,10 @@ def test_select_ad_hoc_model(self): query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" results = list(self.database.select(query)) self.assertEqual(len(results), 2) - self.assertEqual(results[0].__class__.__name__, 'AdHocModel') - self.assertEqual(results[0].last_name, 'Durham') + self.assertEqual(results[0].__class__.__name__, "AdHocModel") + self.assertEqual(results[0].last_name, "Durham") self.assertEqual(results[0].height, 1.72) - self.assertEqual(results[1].last_name, 'Scott') + self.assertEqual(results[1].last_name, "Scott") self.assertEqual(results[1].height, 1.70) self.assertEqual(results[0].get_database(), self.database) self.assertEqual(results[1].get_database(), self.database) @@ -116,7 +117,7 @@ def test_pagination(self): page_num = 1 instances = set() while True: - page = self.database.paginate(Person, 'first_name, last_name', page_num, page_size) + page = self.database.paginate(Person, "first_name, last_name", page_num, page_size) self.assertEqual(page.number_of_objects, len(data)) self.assertGreater(page.pages_total, 0) [instances.add(obj.to_tsv()) for obj in page.objects] @@ -131,15 +132,16 @@ def test_pagination_last_page(self): # Try different page sizes for page_size in (1, 2, 7, 10, 30, 100, 150): # Ask for the last page in two different ways and verify equality - page_a = self.database.paginate(Person, 'first_name, last_name', -1, page_size) - page_b = self.database.paginate(Person, 'first_name, last_name', page_a.pages_total, page_size) + page_a = self.database.paginate(Person, "first_name, last_name", -1, page_size) + page_b = self.database.paginate(Person, "first_name, last_name", page_a.pages_total, page_size) self.assertEqual(page_a[1:], page_b[1:]) - self.assertEqual([obj.to_tsv() for obj in page_a.objects], - [obj.to_tsv() for obj in page_b.objects]) + self.assertEqual([obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects]) def test_pagination_empty_page(self): for page_num in (-1, 1, 2): - page = self.database.paginate(Person, 'first_name, last_name', page_num, 10, conditions="first_name = 'Ziggy'") + page = self.database.paginate( + Person, "first_name, last_name", page_num, 10, conditions="first_name = 'Ziggy'" + ) self.assertEqual(page.number_of_objects, 0) self.assertEqual(page.objects, []) self.assertEqual(page.pages_total, 0) @@ -149,22 +151,22 @@ def test_pagination_invalid_page(self): self._insert_and_check(self._sample_data(), len(data)) for page_num in (0, -2, -100): with self.assertRaises(ValueError): - self.database.paginate(Person, 'first_name, last_name', page_num, 100) + self.database.paginate(Person, "first_name, last_name", page_num, 100) def test_pagination_with_conditions(self): self._insert_and_check(self._sample_data(), len(data)) # Conditions as string - page = self.database.paginate(Person, 'first_name, last_name', 1, 100, conditions="first_name < 'Ava'") + page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions="first_name < 'Ava'") self.assertEqual(page.number_of_objects, 10) # Conditions as expression - page = self.database.paginate(Person, 'first_name, last_name', 1, 100, conditions=Person.first_name < 'Ava') + page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions=Person.first_name < "Ava") self.assertEqual(page.number_of_objects, 10) # Conditions as Q object - page = self.database.paginate(Person, 'first_name, last_name', 1, 100, conditions=Q(first_name__lt='Ava')) + page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions=Q(first_name__lt="Ava")) self.assertEqual(page.number_of_objects, 10) def test_special_chars(self): - s = u'אבגד \\\'"`,.;éåäöšž\n\t\0\b\r' + s = u"אבגד \\'\"`,.;éåäöšž\n\t\0\b\r" p = Person(first_name=s) self.database.insert([p]) p = list(self.database.select("SELECT * from $table", Person))[0] @@ -178,18 +180,18 @@ def test_raw(self): def test_invalid_user(self): with self.assertRaises(ServerError) as cm: - Database(self.database.db_name, username='default', password='wrong') + Database(self.database.db_name, username="default", password="wrong") exc = cm.exception - if exc.code == 193: # ClickHouse version < 20.3 - self.assertTrue(exc.message.startswith('Wrong password for user default')) - elif exc.code == 516: # ClickHouse version >= 20.3 - self.assertTrue(exc.message.startswith('default: Authentication failed')) + if exc.code == 193: # ClickHouse version < 20.3 + self.assertTrue(exc.message.startswith("Wrong password for user default")) + elif exc.code == 516: # ClickHouse version >= 20.3 + self.assertTrue(exc.message.startswith("default: Authentication failed")) else: - raise Exception('Unexpected error code - %s' % exc.code) + raise Exception("Unexpected error code - %s" % exc.code) def test_nonexisting_db(self): - db = Database('db_not_here', autocreate=False) + db = Database("db_not_here", autocreate=False) with self.assertRaises(ServerError) as cm: db.create_table(Person) exc = cm.exception @@ -212,25 +214,28 @@ def test_preexisting_db(self): def test_missing_engine(self): class EnginelessModel(Model): float_field = Float32Field() + with self.assertRaises(DatabaseException) as cm: self.database.create_table(EnginelessModel) - self.assertEqual(str(cm.exception), 'EnginelessModel class must define an engine') + self.assertEqual(str(cm.exception), "EnginelessModel class must define an engine") def test_potentially_problematic_field_names(self): class Model1(Model): system = StringField() readonly = StringField() engine = Memory() - instance = Model1(system='s', readonly='r') - self.assertEqual(instance.to_dict(), dict(system='s', readonly='r')) + + instance = Model1(system="s", readonly="r") + self.assertEqual(instance.to_dict(), dict(system="s", readonly="r")) self.database.create_table(Model1) self.database.insert([instance]) instance = Model1.objects_in(self.database)[0] - self.assertEqual(instance.to_dict(), dict(system='s', readonly='r')) + self.assertEqual(instance.to_dict(), dict(system="s", readonly="r")) def test_does_table_exist(self): class Person2(Person): pass + self.assertTrue(self.database.does_table_exist(Person)) self.assertFalse(self.database.does_table_exist(Person2)) @@ -239,32 +244,31 @@ def test_add_setting(self): with self.assertRaises(AssertionError): self.database.add_setting(0, 1) # Add a setting and see that it makes the query fail - self.database.add_setting('max_columns_to_read', 1) + self.database.add_setting("max_columns_to_read", 1) with self.assertRaises(ServerError): - list(self.database.select('SELECT * from system.tables')) + list(self.database.select("SELECT * from system.tables")) # Remove the setting and see that now it works - self.database.add_setting('max_columns_to_read', None) - list(self.database.select('SELECT * from system.tables')) + self.database.add_setting("max_columns_to_read", None) + list(self.database.select("SELECT * from system.tables")) def test_create_ad_hoc_field(self): # Tests that create_ad_hoc_field works for all column types in the database from clickhouse_orm.models import ModelBase + query = "SELECT DISTINCT type FROM system.columns" for row in self.database.select(query): ModelBase.create_ad_hoc_field(row.type) def test_get_model_for_table(self): # Tests that get_model_for_table works for a non-system model - model = self.database.get_model_for_table('person') + model = self.database.get_model_for_table("person") self.assertFalse(model.is_system_model()) self.assertFalse(model.is_read_only()) - self.assertEqual(model.table_name(), 'person') + self.assertEqual(model.table_name(), "person") # Read a few records list(model.objects_in(self.database)[:10]) # Inserts should work too - self.database.insert([ - model(first_name='aaa', last_name='bbb', height=1.77) - ]) + self.database.insert([model(first_name="aaa", last_name="bbb", height=1.77)]) def test_get_model_for_table__system(self): # Tests that get_model_for_table works for all system tables @@ -279,7 +283,7 @@ def test_get_model_for_table__system(self): try: list(model.objects_in(self.database)[:10]) except ServerError as e: - if 'Not enough privileges' in e.message: + if "Not enough privileges" in e.message: pass else: raise diff --git a/tests/test_datetime_fields.py b/tests/test_datetime_fields.py index a8783bf..9d8f2bd 100644 --- a/tests/test_datetime_fields.py +++ b/tests/test_datetime_fields.py @@ -9,33 +9,35 @@ class DateFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) if self.database.server_version < (20, 1, 2, 4): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") self.database.create_table(ModelWithDate) def tearDown(self): self.database.drop_database() def test_ad_hoc_model(self): - self.database.insert([ - ModelWithDate( - date_field='2016-08-30', - datetime_field='2016-08-30 03:50:00', - datetime64_field='2016-08-30 03:50:00.123456', - datetime64_3_field='2016-08-30 03:50:00.123456' - ), - ModelWithDate( - date_field='2016-08-31', - datetime_field='2016-08-31 01:30:00', - datetime64_field='2016-08-31 01:30:00.123456', - datetime64_3_field='2016-08-31 01:30:00.123456') - ]) + self.database.insert( + [ + ModelWithDate( + date_field="2016-08-30", + datetime_field="2016-08-30 03:50:00", + datetime64_field="2016-08-30 03:50:00.123456", + datetime64_3_field="2016-08-30 03:50:00.123456", + ), + ModelWithDate( + date_field="2016-08-31", + datetime_field="2016-08-31 01:30:00", + datetime64_field="2016-08-31 01:30:00.123456", + datetime64_3_field="2016-08-31 01:30:00.123456", + ), + ] + ) # toStartOfHour returns DateTime('Asia/Yekaterinburg') in my case, so I test it here to - query = 'SELECT toStartOfHour(datetime_field) as hour_start, * from $db.modelwithdate ORDER BY date_field' + query = "SELECT toStartOfHour(datetime_field) as hour_start, * from $db.modelwithdate ORDER BY date_field" results = list(self.database.select(query)) self.assertEqual(len(results), 2) self.assertEqual(results[0].date_field, datetime.date(2016, 8, 30)) @@ -46,11 +48,13 @@ def test_ad_hoc_model(self): self.assertEqual(results[1].hour_start, datetime.datetime(2016, 8, 31, 1, 0, 0, tzinfo=pytz.UTC)) self.assertEqual(results[0].datetime64_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 123456, tzinfo=pytz.UTC)) - self.assertEqual(results[0].datetime64_3_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 123000, - tzinfo=pytz.UTC)) + self.assertEqual( + results[0].datetime64_3_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 123000, tzinfo=pytz.UTC) + ) self.assertEqual(results[1].datetime64_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 123456, tzinfo=pytz.UTC)) - self.assertEqual(results[1].datetime64_3_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 123000, - tzinfo=pytz.UTC)) + self.assertEqual( + results[1].datetime64_3_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 123000, tzinfo=pytz.UTC) + ) class ModelWithDate(Model): @@ -59,45 +63,46 @@ class ModelWithDate(Model): datetime64_field = DateTime64Field() datetime64_3_field = DateTime64Field(precision=3) - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) class ModelWithTz(Model): datetime_no_tz_field = DateTimeField() # server tz - datetime_tz_field = DateTimeField(timezone='Europe/Madrid') - datetime64_tz_field = DateTime64Field(timezone='Europe/Madrid') + datetime_tz_field = DateTimeField(timezone="Europe/Madrid") + datetime64_tz_field = DateTime64Field(timezone="Europe/Madrid") datetime_utc_field = DateTimeField(timezone=pytz.UTC) - engine = MergeTree('datetime_no_tz_field', ('datetime_no_tz_field',)) + engine = MergeTree("datetime_no_tz_field", ("datetime_no_tz_field",)) class DateTimeFieldWithTzTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) if self.database.server_version < (20, 1, 2, 4): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") self.database.create_table(ModelWithTz) def tearDown(self): self.database.drop_database() def test_ad_hoc_model(self): - self.database.insert([ - ModelWithTz( - datetime_no_tz_field='2020-06-11 04:00:00', - datetime_tz_field='2020-06-11 04:00:00', - datetime64_tz_field='2020-06-11 04:00:00', - datetime_utc_field='2020-06-11 04:00:00', - ), - ModelWithTz( - datetime_no_tz_field='2020-06-11 07:00:00+0300', - datetime_tz_field='2020-06-11 07:00:00+0300', - datetime64_tz_field='2020-06-11 07:00:00+0300', - datetime_utc_field='2020-06-11 07:00:00+0300', - ), - ]) - query = 'SELECT * from $db.modelwithtz ORDER BY datetime_no_tz_field' + self.database.insert( + [ + ModelWithTz( + datetime_no_tz_field="2020-06-11 04:00:00", + datetime_tz_field="2020-06-11 04:00:00", + datetime64_tz_field="2020-06-11 04:00:00", + datetime_utc_field="2020-06-11 04:00:00", + ), + ModelWithTz( + datetime_no_tz_field="2020-06-11 07:00:00+0300", + datetime_tz_field="2020-06-11 07:00:00+0300", + datetime64_tz_field="2020-06-11 07:00:00+0300", + datetime_utc_field="2020-06-11 07:00:00+0300", + ), + ] + ) + query = "SELECT * from $db.modelwithtz ORDER BY datetime_no_tz_field" results = list(self.database.select(query)) self.assertEqual(results[0].datetime_no_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) @@ -110,10 +115,10 @@ def test_ad_hoc_model(self): self.assertEqual(results[1].datetime_utc_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) self.assertEqual(results[0].datetime_no_tz_field.tzinfo.zone, self.database.server_timezone.zone) - self.assertEqual(results[0].datetime_tz_field.tzinfo.zone, pytz.timezone('Europe/Madrid').zone) - self.assertEqual(results[0].datetime64_tz_field.tzinfo.zone, pytz.timezone('Europe/Madrid').zone) - self.assertEqual(results[0].datetime_utc_field.tzinfo.zone, pytz.timezone('UTC').zone) + self.assertEqual(results[0].datetime_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual(results[0].datetime64_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual(results[0].datetime_utc_field.tzinfo.zone, pytz.timezone("UTC").zone) self.assertEqual(results[1].datetime_no_tz_field.tzinfo.zone, self.database.server_timezone.zone) - self.assertEqual(results[1].datetime_tz_field.tzinfo.zone, pytz.timezone('Europe/Madrid').zone) - self.assertEqual(results[1].datetime64_tz_field.tzinfo.zone, pytz.timezone('Europe/Madrid').zone) - self.assertEqual(results[1].datetime_utc_field.tzinfo.zone, pytz.timezone('UTC').zone) + self.assertEqual(results[1].datetime_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual(results[1].datetime64_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual(results[1].datetime_utc_field.tzinfo.zone, pytz.timezone("UTC").zone) diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index f26fe26..ec463f8 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -9,9 +9,8 @@ class DecimalFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) try: self.database.create_table(DecimalModel) except ServerError as e: @@ -22,56 +21,58 @@ def tearDown(self): self.database.drop_database() def _insert_sample_data(self): - self.database.insert([ - DecimalModel(date_field='2016-08-20'), - DecimalModel(date_field='2016-08-21', dec=Decimal('1.234')), - DecimalModel(date_field='2016-08-22', dec32=Decimal('12342.2345')), - DecimalModel(date_field='2016-08-23', dec64=Decimal('12342.23456')), - DecimalModel(date_field='2016-08-24', dec128=Decimal('-4545456612342.234567')), - ]) + self.database.insert( + [ + DecimalModel(date_field="2016-08-20"), + DecimalModel(date_field="2016-08-21", dec=Decimal("1.234")), + DecimalModel(date_field="2016-08-22", dec32=Decimal("12342.2345")), + DecimalModel(date_field="2016-08-23", dec64=Decimal("12342.23456")), + DecimalModel(date_field="2016-08-24", dec128=Decimal("-4545456612342.234567")), + ] + ) def _assert_sample_data(self, results): self.assertEqual(len(results), 5) self.assertEqual(results[0].dec, Decimal(0)) self.assertEqual(results[0].dec32, Decimal(17)) - self.assertEqual(results[1].dec, Decimal('1.234')) - self.assertEqual(results[2].dec32, Decimal('12342.2345')) - self.assertEqual(results[3].dec64, Decimal('12342.23456')) - self.assertEqual(results[4].dec128, Decimal('-4545456612342.234567')) + self.assertEqual(results[1].dec, Decimal("1.234")) + self.assertEqual(results[2].dec32, Decimal("12342.2345")) + self.assertEqual(results[3].dec64, Decimal("12342.23456")) + self.assertEqual(results[4].dec128, Decimal("-4545456612342.234567")) def test_insert_and_select(self): self._insert_sample_data() - query = 'SELECT * from $table ORDER BY date_field' + query = "SELECT * from $table ORDER BY date_field" results = list(self.database.select(query, DecimalModel)) self._assert_sample_data(results) def test_ad_hoc_model(self): self._insert_sample_data() - query = 'SELECT * from decimalmodel ORDER BY date_field' + query = "SELECT * from decimalmodel ORDER BY date_field" results = list(self.database.select(query)) self._assert_sample_data(results) def test_rounding(self): - d = Decimal('11111.2340000000000000001') - self.database.insert([DecimalModel(date_field='2016-08-20', dec=d, dec32=d, dec64=d, dec128=d)]) + d = Decimal("11111.2340000000000000001") + self.database.insert([DecimalModel(date_field="2016-08-20", dec=d, dec32=d, dec64=d, dec128=d)]) m = DecimalModel.objects_in(self.database)[0] for val in (m.dec, m.dec32, m.dec64, m.dec128): - self.assertEqual(val, Decimal('11111.234')) + self.assertEqual(val, Decimal("11111.234")) def test_assignment_ok(self): - for value in (True, False, 17, 3.14, '20.5', Decimal('20.5')): + for value in (True, False, 17, 3.14, "20.5", Decimal("20.5")): DecimalModel(dec=value) def test_assignment_error(self): - for value in ('abc', u'זה ארוך', None, float('NaN'), Decimal('-Infinity')): + for value in ("abc", u"זה ארוך", None, float("NaN"), Decimal("-Infinity")): with self.assertRaises(ValueError): DecimalModel(dec=value) def test_aggregation(self): self._insert_sample_data() - result = DecimalModel.objects_in(self.database).aggregate(m='min(dec)', n='max(dec)') + result = DecimalModel.objects_in(self.database).aggregate(m="min(dec)", n="max(dec)") self.assertEqual(result[0].m, Decimal(0)) - self.assertEqual(result[0].n, Decimal('1.234')) + self.assertEqual(result[0].n, Decimal("1.234")) def test_precision_and_scale(self): # Go over all valid combinations @@ -86,36 +87,36 @@ def test_precision_and_scale(self): def test_min_max(self): # In range f = DecimalField(3, 1) - f.validate(f.to_python('99.9', None)) - f.validate(f.to_python('-99.9', None)) + f.validate(f.to_python("99.9", None)) + f.validate(f.to_python("-99.9", None)) # In range after rounding - f.validate(f.to_python('99.94', None)) - f.validate(f.to_python('-99.94', None)) + f.validate(f.to_python("99.94", None)) + f.validate(f.to_python("-99.94", None)) # Out of range with self.assertRaises(ValueError): - f.validate(f.to_python('99.99', None)) + f.validate(f.to_python("99.99", None)) with self.assertRaises(ValueError): - f.validate(f.to_python('-99.99', None)) + f.validate(f.to_python("-99.99", None)) # In range f = Decimal32Field(4) - f.validate(f.to_python('99999.9999', None)) - f.validate(f.to_python('-99999.9999', None)) + f.validate(f.to_python("99999.9999", None)) + f.validate(f.to_python("-99999.9999", None)) # In range after rounding - f.validate(f.to_python('99999.99994', None)) - f.validate(f.to_python('-99999.99994', None)) + f.validate(f.to_python("99999.99994", None)) + f.validate(f.to_python("-99999.99994", None)) # Out of range with self.assertRaises(ValueError): - f.validate(f.to_python('100000', None)) + f.validate(f.to_python("100000", None)) with self.assertRaises(ValueError): - f.validate(f.to_python('-100000', None)) + f.validate(f.to_python("-100000", None)) class DecimalModel(Model): - date_field = DateField() - dec = DecimalField(15, 3) - dec32 = Decimal32Field(4, default=17) - dec64 = Decimal64Field(5) - dec128 = Decimal128Field(6) + date_field = DateField() + dec = DecimalField(15, 3) + dec32 = Decimal32Field(4, default=17) + dec64 = Decimal64Field(5) + dec128 = Decimal128Field(6) engine = Memory() diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 40bead6..15f17aa 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -5,37 +5,36 @@ class DictionaryTestMixin: - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) if self.database.server_version < (20, 1, 11, 73): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") self._create_dictionary() def tearDown(self): self.database.drop_database() def _test_func(self, func, expected_value): - sql = 'SELECT %s AS value' % func.to_sql() + sql = "SELECT %s AS value" % func.to_sql() logging.info(sql) result = list(self.database.select(sql)) - logging.info('\t==> %s', result[0].value if result else '') - print('Comparing %s to %s' % (result[0].value, expected_value)) + logging.info("\t==> %s", result[0].value if result else "") + print("Comparing %s to %s" % (result[0].value, expected_value)) self.assertEqual(result[0].value, expected_value) return result[0].value if result else None class SimpleDictionaryTest(DictionaryTestMixin, unittest.TestCase): - def _create_dictionary(self): # Create a table to be used as source for the dictionary self.database.create_table(NumberName) self.database.insert( NumberName(number=i, name=name) - for i, name in enumerate('Zero One Two Three Four Five Six Seven Eight Nine Ten'.split()) + for i, name in enumerate("Zero One Two Three Four Five Six Seven Eight Nine Ten".split()) ) # Create the dictionary - self.database.raw(""" + self.database.raw( + """ CREATE DICTIONARY numbers_dict( number UInt64, name String DEFAULT '?' @@ -46,16 +45,17 @@ def _create_dictionary(self): )) LIFETIME(100) LAYOUT(HASHED()); - """) - self.dict_name = 'test-db.numbers_dict' + """ + ) + self.dict_name = "test-db.numbers_dict" def test_dictget(self): - self._test_func(F.dictGet(self.dict_name, 'name', F.toUInt64(3)), 'Three') - self._test_func(F.dictGet(self.dict_name, 'name', F.toUInt64(99)), '?') + self._test_func(F.dictGet(self.dict_name, "name", F.toUInt64(3)), "Three") + self._test_func(F.dictGet(self.dict_name, "name", F.toUInt64(99)), "?") def test_dictgetordefault(self): - self._test_func(F.dictGetOrDefault(self.dict_name, 'name', F.toUInt64(3), 'n/a'), 'Three') - self._test_func(F.dictGetOrDefault(self.dict_name, 'name', F.toUInt64(99), 'n/a'), 'n/a') + self._test_func(F.dictGetOrDefault(self.dict_name, "name", F.toUInt64(3), "n/a"), "Three") + self._test_func(F.dictGetOrDefault(self.dict_name, "name", F.toUInt64(99), "n/a"), "n/a") def test_dicthas(self): self._test_func(F.dictHas(self.dict_name, F.toUInt64(3)), 1) @@ -63,19 +63,21 @@ def test_dicthas(self): class HierarchicalDictionaryTest(DictionaryTestMixin, unittest.TestCase): - def _create_dictionary(self): # Create a table to be used as source for the dictionary self.database.create_table(Region) - self.database.insert([ - Region(region_id=1, parent_region=0, region_name='Russia'), - Region(region_id=2, parent_region=1, region_name='Moscow'), - Region(region_id=3, parent_region=2, region_name='Center'), - Region(region_id=4, parent_region=0, region_name='Great Britain'), - Region(region_id=5, parent_region=4, region_name='London'), - ]) + self.database.insert( + [ + Region(region_id=1, parent_region=0, region_name="Russia"), + Region(region_id=2, parent_region=1, region_name="Moscow"), + Region(region_id=3, parent_region=2, region_name="Center"), + Region(region_id=4, parent_region=0, region_name="Great Britain"), + Region(region_id=5, parent_region=4, region_name="London"), + ] + ) # Create the dictionary - self.database.raw(""" + self.database.raw( + """ CREATE DICTIONARY regions_dict( region_id UInt64, parent_region UInt64 HIERARCHICAL, @@ -87,17 +89,18 @@ def _create_dictionary(self): )) LIFETIME(100) LAYOUT(HASHED()); - """) - self.dict_name = 'test-db.regions_dict' + """ + ) + self.dict_name = "test-db.regions_dict" def test_dictget(self): - self._test_func(F.dictGet(self.dict_name, 'region_name', F.toUInt64(3)), 'Center') - self._test_func(F.dictGet(self.dict_name, 'parent_region', F.toUInt64(3)), 2) - self._test_func(F.dictGet(self.dict_name, 'region_name', F.toUInt64(99)), '?') + self._test_func(F.dictGet(self.dict_name, "region_name", F.toUInt64(3)), "Center") + self._test_func(F.dictGet(self.dict_name, "parent_region", F.toUInt64(3)), 2) + self._test_func(F.dictGet(self.dict_name, "region_name", F.toUInt64(99)), "?") def test_dictgetordefault(self): - self._test_func(F.dictGetOrDefault(self.dict_name, 'region_name', F.toUInt64(3), 'n/a'), 'Center') - self._test_func(F.dictGetOrDefault(self.dict_name, 'region_name', F.toUInt64(99), 'n/a'), 'n/a') + self._test_func(F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(3), "n/a"), "Center") + self._test_func(F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(99), "n/a"), "n/a") def test_dicthas(self): self._test_func(F.dictHas(self.dict_name, F.toUInt64(3)), 1) @@ -114,7 +117,7 @@ def test_dictisin(self): class NumberName(Model): - ''' A table to act as a source for the dictionary ''' + """A table to act as a source for the dictionary""" number = UInt64Field() name = StringField() diff --git a/tests/test_engines.py b/tests/test_engines.py index 77781df..a2775fb 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -4,13 +4,13 @@ from clickhouse_orm import * import logging + logging.getLogger("requests").setLevel(logging.WARNING) class _EnginesHelperTestCase(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) def tearDown(self): self.database.drop_database() @@ -19,32 +19,47 @@ def tearDown(self): class EnginesTestCase(_EnginesHelperTestCase): def _create_and_insert(self, model_class): self.database.create_table(model_class) - self.database.insert([ - model_class(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1) - ]) + self.database.insert( + [model_class(date="2017-01-01", event_id=23423, event_group=13, event_count=7, event_version=1)] + ) def test_merge_tree(self): class TestModel(SampleModel): - engine = MergeTree('date', ('date', 'event_id', 'event_group')) + engine = MergeTree("date", ("date", "event_id", "event_group")) + self._create_and_insert(TestModel) def test_merge_tree_with_sampling(self): class TestModel(SampleModel): - engine = MergeTree('date', ('date', 'event_id', 'event_group', 'intHash32(event_id)'), sampling_expr='intHash32(event_id)') + engine = MergeTree( + "date", ("date", "event_id", "event_group", "intHash32(event_id)"), sampling_expr="intHash32(event_id)" + ) + self._create_and_insert(TestModel) def test_merge_tree_with_sampling__funcs(self): class TestModel(SampleModel): - engine = MergeTree('date', ('date', 'event_id', 'event_group', F.intHash32(SampleModel.event_id)), sampling_expr=F.intHash32(SampleModel.event_id)) + engine = MergeTree( + "date", + ("date", "event_id", "event_group", F.intHash32(SampleModel.event_id)), + sampling_expr=F.intHash32(SampleModel.event_id), + ) + self._create_and_insert(TestModel) def test_merge_tree_with_granularity(self): class TestModel(SampleModel): - engine = MergeTree('date', ('date', 'event_id', 'event_group'), index_granularity=4096) + engine = MergeTree("date", ("date", "event_id", "event_group"), index_granularity=4096) + self._create_and_insert(TestModel) def test_replicated_merge_tree(self): - engine = MergeTree('date', ('date', 'event_id', 'event_group'), replica_table_path='/clickhouse/tables/{layer}-{shard}/hits', replica_name='{replica}') + engine = MergeTree( + "date", + ("date", "event_id", "event_group"), + replica_table_path="/clickhouse/tables/{layer}-{shard}/hits", + replica_name="{replica}", + ) # In ClickHouse 1.1.54310 custom partitioning key was introduced and new syntax is used if self.database.server_version >= (1, 1, 54310): expected = "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}') PARTITION BY (toYYYYMM(`date`)) ORDER BY (date, event_id, event_group) SETTINGS index_granularity=8192" @@ -54,38 +69,48 @@ def test_replicated_merge_tree(self): def test_replicated_merge_tree_incomplete(self): with self.assertRaises(AssertionError): - MergeTree('date', ('date', 'event_id', 'event_group'), replica_table_path='/clickhouse/tables/{layer}-{shard}/hits') + MergeTree( + "date", + ("date", "event_id", "event_group"), + replica_table_path="/clickhouse/tables/{layer}-{shard}/hits", + ) with self.assertRaises(AssertionError): - MergeTree('date', ('date', 'event_id', 'event_group'), replica_name='{replica}') + MergeTree("date", ("date", "event_id", "event_group"), replica_name="{replica}") def test_collapsing_merge_tree(self): class TestModel(SampleModel): - engine = CollapsingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_version') + engine = CollapsingMergeTree("date", ("date", "event_id", "event_group"), "event_version") + self._create_and_insert(TestModel) def test_summing_merge_tree(self): class TestModel(SampleModel): - engine = SummingMergeTree('date', ('date', 'event_group'), ('event_count',)) + engine = SummingMergeTree("date", ("date", "event_group"), ("event_count",)) + self._create_and_insert(TestModel) def test_replacing_merge_tree(self): class TestModel(SampleModel): - engine = ReplacingMergeTree('date', ('date', 'event_id', 'event_group'), 'event_uversion') + engine = ReplacingMergeTree("date", ("date", "event_id", "event_group"), "event_uversion") + self._create_and_insert(TestModel) def test_tiny_log(self): class TestModel(SampleModel): engine = TinyLog() + self._create_and_insert(TestModel) def test_log(self): class TestModel(SampleModel): engine = Log() + self._create_and_insert(TestModel) def test_memory(self): class TestModel(SampleModel): engine = Memory() + self._create_and_insert(TestModel) def test_merge(self): @@ -96,7 +121,7 @@ class TestModel2(SampleModel): engine = TinyLog() class TestMergeModel(MergeModel, SampleModel): - engine = Merge('^testmodel') + engine = Merge("^testmodel") self.database.create_table(TestModel1) self.database.create_table(TestModel2) @@ -104,54 +129,57 @@ class TestMergeModel(MergeModel, SampleModel): # Insert operations are restricted for this model type with self.assertRaises(DatabaseException): - self.database.insert([ - TestMergeModel(date='2017-01-01', event_id=23423, event_group=13, event_count=7, event_version=1) - ]) + self.database.insert( + [TestMergeModel(date="2017-01-01", event_id=23423, event_group=13, event_count=7, event_version=1)] + ) # Testing select - self.database.insert([ - TestModel1(date='2017-01-01', event_id=1, event_group=1, event_count=1, event_version=1) - ]) - self.database.insert([ - TestModel2(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2) - ]) + self.database.insert([TestModel1(date="2017-01-01", event_id=1, event_group=1, event_count=1, event_version=1)]) + self.database.insert([TestModel2(date="2017-01-02", event_id=2, event_group=2, event_count=2, event_version=2)]) # event_uversion is materialized field. So * won't select it and it will be zero - res = self.database.select('SELECT *, _table, event_uversion FROM $table ORDER BY event_id', model_class=TestMergeModel) + res = self.database.select( + "SELECT *, _table, event_uversion FROM $table ORDER BY event_id", model_class=TestMergeModel + ) res = list(res) self.assertEqual(2, len(res)) - self.assertDictEqual({ - '_table': 'testmodel1', - 'date': datetime.date(2017, 1, 1), - 'event_id': 1, - 'event_group': 1, - 'event_count': 1, - 'event_version': 1, - 'event_uversion': 1 - }, res[0].to_dict(include_readonly=True)) - self.assertDictEqual({ - '_table': 'testmodel2', - 'date': datetime.date(2017, 1, 2), - 'event_id': 2, - 'event_group': 2, - 'event_count': 2, - 'event_version': 2, - 'event_uversion': 2 - }, res[1].to_dict(include_readonly=True)) + self.assertDictEqual( + { + "_table": "testmodel1", + "date": datetime.date(2017, 1, 1), + "event_id": 1, + "event_group": 1, + "event_count": 1, + "event_version": 1, + "event_uversion": 1, + }, + res[0].to_dict(include_readonly=True), + ) + self.assertDictEqual( + { + "_table": "testmodel2", + "date": datetime.date(2017, 1, 2), + "event_id": 2, + "event_group": 2, + "event_count": 2, + "event_version": 2, + "event_uversion": 2, + }, + res[1].to_dict(include_readonly=True), + ) def test_custom_partitioning(self): class TestModel(SampleModel): engine = MergeTree( - order_by=('date', 'event_id', 'event_group'), - partition_key=('toYYYYMM(date)', 'event_group') + order_by=("date", "event_id", "event_group"), partition_key=("toYYYYMM(date)", "event_group") ) class TestCollapseModel(SampleModel): sign = Int8Field() engine = CollapsingMergeTree( - sign_col='sign', - order_by=('date', 'event_id', 'event_group'), - partition_key=('toYYYYMM(date)', 'event_group') + sign_col="sign", + order_by=("date", "event_id", "event_group"), + partition_key=("toYYYYMM(date)", "event_group"), ) self._create_and_insert(TestModel) @@ -161,30 +189,30 @@ class TestCollapseModel(SampleModel): parts = sorted(list(SystemPart.get(self.database)), key=lambda x: x.table) self.assertEqual(2, len(parts)) - self.assertEqual('testcollapsemodel', parts[0].table) - self.assertEqual('(201701, 13)'.replace(' ', ''), parts[0].partition.replace(' ', '')) - self.assertEqual('testmodel', parts[1].table) - self.assertEqual('(201701, 13)'.replace(' ', ''), parts[1].partition.replace(' ', '')) + self.assertEqual("testcollapsemodel", parts[0].table) + self.assertEqual("(201701, 13)".replace(" ", ""), parts[0].partition.replace(" ", "")) + self.assertEqual("testmodel", parts[1].table) + self.assertEqual("(201701, 13)".replace(" ", ""), parts[1].partition.replace(" ", "")) def test_custom_primary_key(self): if self.database.server_version < (18, 1): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") class TestModel(SampleModel): engine = MergeTree( - order_by=('date', 'event_id', 'event_group'), - partition_key=('toYYYYMM(date)',), - primary_key=('date', 'event_id') + order_by=("date", "event_id", "event_group"), + partition_key=("toYYYYMM(date)",), + primary_key=("date", "event_id"), ) class TestCollapseModel(SampleModel): sign = Int8Field() engine = CollapsingMergeTree( - sign_col='sign', - order_by=('date', 'event_id', 'event_group'), - partition_key=('toYYYYMM(date)',), - primary_key=('date', 'event_id') + sign_col="sign", + order_by=("date", "event_id", "event_group"), + partition_key=("toYYYYMM(date)",), + primary_key=("date", "event_id"), ) self._create_and_insert(TestModel) @@ -195,28 +223,28 @@ class TestCollapseModel(SampleModel): class SampleModel(Model): - date = DateField() - event_id = UInt32Field() - event_group = UInt32Field() - event_count = UInt16Field() - event_version = Int8Field() - event_uversion = UInt8Field(materialized='abs(event_version)') + date = DateField() + event_id = UInt32Field() + event_group = UInt32Field() + event_count = UInt16Field() + event_version = Int8Field() + event_uversion = UInt8Field(materialized="abs(event_version)") class DistributedTestCase(_EnginesHelperTestCase): def test_without_table_name(self): - engine = Distributed('my_cluster') + engine = Distributed("my_cluster") with self.assertRaises(ValueError) as cm: engine.create_table_sql(self.database) exc = cm.exception - self.assertEqual(str(exc), 'Cannot create Distributed engine: specify an underlying table') + self.assertEqual(str(exc), "Cannot create Distributed engine: specify an underlying table") def test_with_table_name(self): - engine = Distributed('my_cluster', 'foo') + engine = Distributed("my_cluster", "foo") sql = engine.create_table_sql(self.database) - self.assertEqual(sql, 'Distributed(`my_cluster`, `test-db`, `foo`)') + self.assertEqual(sql, "Distributed(`my_cluster`, `test-db`, `foo`)") class TestModel(SampleModel): engine = TinyLog() @@ -231,7 +259,7 @@ class TestDistributedModel(DistributedModel, underlying): def test_bad_cluster_name(self): with self.assertRaises(ServerError) as cm: - d_model = self._create_distributed('cluster_name') + d_model = self._create_distributed("cluster_name") self.database.count(d_model) exc = cm.exception @@ -243,7 +271,7 @@ class TestModel2(SampleModel): engine = Log() class TestDistributedModel(DistributedModel, self.TestModel, TestModel2): - engine = Distributed('test_shard_localhost', self.TestModel) + engine = Distributed("test_shard_localhost", self.TestModel) self.database.create_table(self.TestModel) self.database.create_table(TestDistributedModel) @@ -251,7 +279,7 @@ class TestDistributedModel(DistributedModel, self.TestModel, TestModel2): def test_minimal_engine(self): class TestDistributedModel(DistributedModel, self.TestModel): - engine = Distributed('test_shard_localhost') + engine = Distributed("test_shard_localhost") self.database.create_table(self.TestModel) self.database.create_table(TestDistributedModel) @@ -263,64 +291,78 @@ class TestModel2(SampleModel): engine = Log() class TestDistributedModel(DistributedModel, self.TestModel, TestModel2): - engine = Distributed('test_shard_localhost') + engine = Distributed("test_shard_localhost") self.database.create_table(self.TestModel) with self.assertRaises(TypeError) as cm: self.database.create_table(TestDistributedModel) exc = cm.exception - self.assertEqual(str(exc), 'When defining Distributed engine without the table_name ensure ' - 'that your model has exactly one non-distributed superclass') + self.assertEqual( + str(exc), + "When defining Distributed engine without the table_name ensure " + "that your model has exactly one non-distributed superclass", + ) def test_minimal_engine_no_superclasses(self): class TestDistributedModel(DistributedModel): - engine = Distributed('test_shard_localhost') + engine = Distributed("test_shard_localhost") self.database.create_table(self.TestModel) with self.assertRaises(TypeError) as cm: self.database.create_table(TestDistributedModel) exc = cm.exception - self.assertEqual(str(exc), 'When defining Distributed engine without the table_name ensure ' - 'that your model has a parent model') + self.assertEqual( + str(exc), + "When defining Distributed engine without the table_name ensure " "that your model has a parent model", + ) def _test_insert_select(self, local_to_distributed, test_model=TestModel, include_readonly=True): - d_model = self._create_distributed('test_shard_localhost', underlying=test_model) + d_model = self._create_distributed("test_shard_localhost", underlying=test_model) if local_to_distributed: to_insert, to_select = test_model, d_model else: to_insert, to_select = d_model, test_model - self.database.insert([ - to_insert(date='2017-01-01', event_id=1, event_group=1, event_count=1, event_version=1), - to_insert(date='2017-01-02', event_id=2, event_group=2, event_count=2, event_version=2) - ]) + self.database.insert( + [ + to_insert(date="2017-01-01", event_id=1, event_group=1, event_count=1, event_version=1), + to_insert(date="2017-01-02", event_id=2, event_group=2, event_count=2, event_version=2), + ] + ) # event_uversion is materialized field. So * won't select it and it will be zero - res = self.database.select('SELECT *, event_uversion FROM $table ORDER BY event_id', - model_class=to_select) + res = self.database.select("SELECT *, event_uversion FROM $table ORDER BY event_id", model_class=to_select) res = [row for row in res] self.assertEqual(2, len(res)) - self.assertDictEqual({ - 'date': datetime.date(2017, 1, 1), - 'event_id': 1, - 'event_group': 1, - 'event_count': 1, - 'event_version': 1, - 'event_uversion': 1 - }, res[0].to_dict(include_readonly=include_readonly)) - self.assertDictEqual({ - 'date': datetime.date(2017, 1, 2), - 'event_id': 2, - 'event_group': 2, - 'event_count': 2, - 'event_version': 2, - 'event_uversion': 2 - }, res[1].to_dict(include_readonly=include_readonly)) - - @unittest.skip("Bad support of materialized fields in Distributed tables " - "https://groups.google.com/forum/#!topic/clickhouse/XEYRRwZrsSc") + self.assertDictEqual( + { + "date": datetime.date(2017, 1, 1), + "event_id": 1, + "event_group": 1, + "event_count": 1, + "event_version": 1, + "event_uversion": 1, + }, + res[0].to_dict(include_readonly=include_readonly), + ) + self.assertDictEqual( + { + "date": datetime.date(2017, 1, 2), + "event_id": 2, + "event_group": 2, + "event_count": 2, + "event_version": 2, + "event_uversion": 2, + }, + res[1].to_dict(include_readonly=include_readonly), + ) + + @unittest.skip( + "Bad support of materialized fields in Distributed tables " + "https://groups.google.com/forum/#!topic/clickhouse/XEYRRwZrsSc" + ) def test_insert_distributed_select_local(self): return self._test_insert_select(local_to_distributed=False) diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py index f53ce69..004c0da 100644 --- a/tests/test_enum_fields.py +++ b/tests/test_enum_fields.py @@ -9,9 +9,8 @@ class EnumFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(ModelWithEnum) self.database.create_table(ModelWithEnumArray) @@ -19,12 +18,14 @@ def tearDown(self): self.database.drop_database() def test_insert_and_select(self): - self.database.insert([ - ModelWithEnum(date_field='2016-08-30', enum_field=Fruit.apple), - ModelWithEnum(date_field='2016-08-31', enum_field=Fruit.orange), - ModelWithEnum(date_field='2016-08-31', enum_field=Fruit.cherry) - ]) - query = 'SELECT * from $table ORDER BY date_field' + self.database.insert( + [ + ModelWithEnum(date_field="2016-08-30", enum_field=Fruit.apple), + ModelWithEnum(date_field="2016-08-31", enum_field=Fruit.orange), + ModelWithEnum(date_field="2016-08-31", enum_field=Fruit.cherry), + ] + ) + query = "SELECT * from $table ORDER BY date_field" results = list(self.database.select(query, ModelWithEnum)) self.assertEqual(len(results), 3) self.assertEqual(results[0].enum_field, Fruit.apple) @@ -32,12 +33,14 @@ def test_insert_and_select(self): self.assertEqual(results[2].enum_field, Fruit.cherry) def test_ad_hoc_model(self): - self.database.insert([ - ModelWithEnum(date_field='2016-08-30', enum_field=Fruit.apple), - ModelWithEnum(date_field='2016-08-31', enum_field=Fruit.orange), - ModelWithEnum(date_field='2016-08-31', enum_field=Fruit.cherry) - ]) - query = 'SELECT * from $db.modelwithenum ORDER BY date_field' + self.database.insert( + [ + ModelWithEnum(date_field="2016-08-30", enum_field=Fruit.apple), + ModelWithEnum(date_field="2016-08-31", enum_field=Fruit.orange), + ModelWithEnum(date_field="2016-08-31", enum_field=Fruit.cherry), + ] + ) + query = "SELECT * from $db.modelwithenum ORDER BY date_field" results = list(self.database.select(query)) self.assertEqual(len(results), 3) self.assertEqual(results[0].enum_field.name, Fruit.apple.name) @@ -50,11 +53,11 @@ def test_ad_hoc_model(self): def test_conversion(self): self.assertEqual(ModelWithEnum(enum_field=3).enum_field, Fruit.orange) self.assertEqual(ModelWithEnum(enum_field=-7).enum_field, Fruit.cherry) - self.assertEqual(ModelWithEnum(enum_field='apple').enum_field, Fruit.apple) + self.assertEqual(ModelWithEnum(enum_field="apple").enum_field, Fruit.apple) self.assertEqual(ModelWithEnum(enum_field=Fruit.banana).enum_field, Fruit.banana) def test_assignment_error(self): - for value in (0, 17, 'pear', '', None, 99.9): + for value in (0, 17, "pear", "", None, 99.9): with self.assertRaises(ValueError): ModelWithEnum(enum_field=value) @@ -63,15 +66,15 @@ def test_default_value(self): self.assertEqual(instance.enum_field, Fruit.apple) def test_enum_array(self): - instance = ModelWithEnumArray(date_field='2016-08-30', enum_array=[Fruit.apple, Fruit.apple, Fruit.orange]) + instance = ModelWithEnumArray(date_field="2016-08-30", enum_array=[Fruit.apple, Fruit.apple, Fruit.orange]) self.database.insert([instance]) - query = 'SELECT * from $table ORDER BY date_field' + query = "SELECT * from $table ORDER BY date_field" results = list(self.database.select(query, ModelWithEnumArray)) self.assertEqual(len(results), 1) self.assertEqual(results[0].enum_array, instance.enum_array) -Fruit = Enum('Fruit', [('apple', 1), ('banana', 2), ('orange', 3), ('cherry', -7)]) +Fruit = Enum("Fruit", [("apple", 1), ("banana", 2), ("orange", 3), ("cherry", -7)]) class ModelWithEnum(Model): @@ -79,7 +82,7 @@ class ModelWithEnum(Model): date_field = DateField() enum_field = Enum8Field(Fruit) - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) class ModelWithEnumArray(Model): @@ -87,5 +90,4 @@ class ModelWithEnumArray(Model): date_field = DateField() enum_array = ArrayField(Enum16Field(Fruit)) - engine = MergeTree('date_field', ('date_field',)) - + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py index 6b548a1..fc41478 100644 --- a/tests/test_fixed_string_fields.py +++ b/tests/test_fixed_string_fields.py @@ -8,43 +8,44 @@ class FixedStringFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(FixedStringModel) def tearDown(self): self.database.drop_database() def _insert_sample_data(self): - self.database.insert([ - FixedStringModel(date_field='2016-08-30', fstr_field=''), - FixedStringModel(date_field='2016-08-30'), - FixedStringModel(date_field='2016-08-31', fstr_field='foo'), - FixedStringModel(date_field='2016-08-31', fstr_field=u'לילה') - ]) + self.database.insert( + [ + FixedStringModel(date_field="2016-08-30", fstr_field=""), + FixedStringModel(date_field="2016-08-30"), + FixedStringModel(date_field="2016-08-31", fstr_field="foo"), + FixedStringModel(date_field="2016-08-31", fstr_field=u"לילה"), + ] + ) def _assert_sample_data(self, results): self.assertEqual(len(results), 4) - self.assertEqual(results[0].fstr_field, '') - self.assertEqual(results[1].fstr_field, 'ABCDEFGHIJK') - self.assertEqual(results[2].fstr_field, 'foo') - self.assertEqual(results[3].fstr_field, u'לילה') + self.assertEqual(results[0].fstr_field, "") + self.assertEqual(results[1].fstr_field, "ABCDEFGHIJK") + self.assertEqual(results[2].fstr_field, "foo") + self.assertEqual(results[3].fstr_field, u"לילה") def test_insert_and_select(self): self._insert_sample_data() - query = 'SELECT * from $table ORDER BY date_field' + query = "SELECT * from $table ORDER BY date_field" results = list(self.database.select(query, FixedStringModel)) self._assert_sample_data(results) def test_ad_hoc_model(self): self._insert_sample_data() - query = 'SELECT * from $db.fixedstringmodel ORDER BY date_field' + query = "SELECT * from $db.fixedstringmodel ORDER BY date_field" results = list(self.database.select(query)) self._assert_sample_data(results) def test_assignment_error(self): - for value in (17, 'this is too long', u'זה ארוך', None, 99.9): + for value in (17, "this is too long", u"זה ארוך", None, 99.9): with self.assertRaises(ValueError): FixedStringModel(fstr_field=value) @@ -52,6 +53,6 @@ def test_assignment_error(self): class FixedStringModel(Model): date_field = DateField() - fstr_field = FixedStringField(12, default='ABCDEFGHIJK') + fstr_field = FixedStringField(12, default="ABCDEFGHIJK") - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_funcs.py b/tests/test_funcs.py index fb250e6..87ca724 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -13,7 +13,6 @@ class FuncsTestCase(TestCaseWithData): - def setUp(self): super(FuncsTestCase, self).setUp() self.database.insert(self._sample_data()) @@ -23,24 +22,24 @@ def _test_qs(self, qs, expected_count): count = 0 for instance in qs: count += 1 - logging.info('\t[%d]\t%s' % (count, instance.to_dict())) + logging.info("\t[%d]\t%s" % (count, instance.to_dict())) self.assertEqual(count, expected_count) self.assertEqual(qs.count(), expected_count) def _test_func(self, func, expected_value=NO_VALUE): - sql = 'SELECT %s AS value' % func.to_sql() + sql = "SELECT %s AS value" % func.to_sql() logging.info(sql) try: result = list(self.database.select(sql)) - logging.info('\t==> %s', result[0].value if result else '') + logging.info("\t==> %s", result[0].value if result else "") if expected_value != NO_VALUE: - print('Comparing %s to %s' % (result[0].value, expected_value)) + print("Comparing %s to %s" % (result[0].value, expected_value)) self.assertEqual(result[0].value, expected_value) return result[0].value if result else None except ServerError as e: - if 'Unknown function' in e.message: + if "Unknown function" in e.message: logging.warning(e.message) - return # ignore functions that don't exist in the used ClickHouse version + return # ignore functions that don't exist in the used ClickHouse version raise def _test_aggr(self, func, expected_value=NO_VALUE): @@ -48,45 +47,45 @@ def _test_aggr(self, func, expected_value=NO_VALUE): logging.info(qs.as_sql()) try: result = list(qs) - logging.info('\t==> %s', result[0].value if result else '') + logging.info("\t==> %s", result[0].value if result else "") if expected_value != NO_VALUE: self.assertEqual(result[0].value, expected_value) return result[0].value if result else None except ServerError as e: - if 'Unknown function' in e.message: + if "Unknown function" in e.message: logging.warning(e.message) - return # ignore functions that don't exist in the used ClickHouse version + return # ignore functions that don't exist in the used ClickHouse version raise def test_func_to_sql(self): # No args - self.assertEqual(F('func').to_sql(), 'func()') + self.assertEqual(F("func").to_sql(), "func()") # String args - self.assertEqual(F('func', "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')") + self.assertEqual(F("func", "Wendy's", u"Wendy's").to_sql(), "func('Wendy\\'s', 'Wendy\\'s')") # Numeric args - self.assertEqual(F('func', 1, 1.1, Decimal('3.3')).to_sql(), "func(1, 1.1, 3.3)") + self.assertEqual(F("func", 1, 1.1, Decimal("3.3")).to_sql(), "func(1, 1.1, 3.3)") # Date args - self.assertEqual(F('func', date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))") + self.assertEqual(F("func", date(2018, 12, 31)).to_sql(), "func(toDate('2018-12-31'))") # Datetime args - self.assertEqual(F('func', datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))") + self.assertEqual(F("func", datetime(2018, 12, 31)).to_sql(), "func(toDateTime('1546214400'))") # Boolean args - self.assertEqual(F('func', True, False).to_sql(), "func(1, 0)") + self.assertEqual(F("func", True, False).to_sql(), "func(1, 0)") # Timezone args - self.assertEqual(F('func', pytz.utc).to_sql(), "func('UTC')") - self.assertEqual(F('func', pytz.timezone('Europe/Athens')).to_sql(), "func('Europe/Athens')") + self.assertEqual(F("func", pytz.utc).to_sql(), "func('UTC')") + self.assertEqual(F("func", pytz.timezone("Europe/Athens")).to_sql(), "func('Europe/Athens')") # Null args - self.assertEqual(F('func', None).to_sql(), "func(NULL)") + self.assertEqual(F("func", None).to_sql(), "func(NULL)") # Fields as args - self.assertEqual(F('func', SampleModel.color).to_sql(), "func(`color`)") + self.assertEqual(F("func", SampleModel.color).to_sql(), "func(`color`)") # Funcs as args - self.assertEqual(F('func', F('sqrt', 25)).to_sql(), 'func(sqrt(25))') + self.assertEqual(F("func", F("sqrt", 25)).to_sql(), "func(sqrt(25))") # Iterables as args - x = [1, 'z', F('foo', 17)] + x = [1, "z", F("foo", 17)] for y in [x, iter(x)]: - self.assertEqual(F('func', y, 5).to_sql(), "func([1, 'z', foo(17)], 5)") + self.assertEqual(F("func", y, 5).to_sql(), "func([1, 'z', foo(17)], 5)") # Tuples as args - self.assertEqual(F('func', [(1, 2), (3, 4)]).to_sql(), "func([(1, 2), (3, 4)])") - self.assertEqual(F('func', tuple(x), 5).to_sql(), "func((1, 'z', foo(17)), 5)") + self.assertEqual(F("func", [(1, 2), (3, 4)]).to_sql(), "func([(1, 2), (3, 4)])") + self.assertEqual(F("func", tuple(x), 5).to_sql(), "func((1, 'z', foo(17)), 5)") # Binary operator functions self.assertEqual(F.plus(1, 2).to_sql(), "(1 + 2)") self.assertEqual(F.lessOrEquals(1, 2).to_sql(), "(1 <= 2)") @@ -106,32 +105,32 @@ def test_filter_float_field(self): def test_filter_date_field(self): qs = Person.objects_in(self.database) # People born on the 30th - self._test_qs(qs.filter(F('equals', F('toDayOfMonth', Person.birthday), 30)), 3) - self._test_qs(qs.filter(F('toDayOfMonth', Person.birthday) == 30), 3) + self._test_qs(qs.filter(F("equals", F("toDayOfMonth", Person.birthday), 30)), 3) + self._test_qs(qs.filter(F("toDayOfMonth", Person.birthday) == 30), 3) self._test_qs(qs.filter(F.toDayOfMonth(Person.birthday) == 30), 3) # People born on Sunday - self._test_qs(qs.filter(F('equals', F('toDayOfWeek', Person.birthday), 7)), 18) - self._test_qs(qs.filter(F('toDayOfWeek', Person.birthday) == 7), 18) + self._test_qs(qs.filter(F("equals", F("toDayOfWeek", Person.birthday), 7)), 18) + self._test_qs(qs.filter(F("toDayOfWeek", Person.birthday) == 7), 18) self._test_qs(qs.filter(F.toDayOfWeek(Person.birthday) == 7), 18) # People born on 1976-10-01 - self._test_qs(qs.filter(F('equals', Person.birthday, '1976-10-01')), 1) - self._test_qs(qs.filter(F('equals', Person.birthday, date(1976, 10, 1))), 1) + self._test_qs(qs.filter(F("equals", Person.birthday, "1976-10-01")), 1) + self._test_qs(qs.filter(F("equals", Person.birthday, date(1976, 10, 1))), 1) self._test_qs(qs.filter(Person.birthday == date(1976, 10, 1)), 1) def test_func_as_field_value(self): qs = Person.objects_in(self.database) self._test_qs(qs.filter(height__gt=F.plus(1, 0.61)), 96) self._test_qs(qs.exclude(birthday=F.today()), 100) - self._test_qs(qs.filter(birthday__between=['1970-01-01', F.today()]), 100) + self._test_qs(qs.filter(birthday__between=["1970-01-01", F.today()]), 100) def test_in_and_not_in(self): qs = Person.objects_in(self.database) - self._test_qs(qs.filter(Person.first_name.isIn(['Ciaran', 'Elton'])), 4) - self._test_qs(qs.filter(~Person.first_name.isIn(['Ciaran', 'Elton'])), 96) - self._test_qs(qs.filter(Person.first_name.isNotIn(['Ciaran', 'Elton'])), 96) - self._test_qs(qs.exclude(Person.first_name.isIn(['Ciaran', 'Elton'])), 96) + self._test_qs(qs.filter(Person.first_name.isIn(["Ciaran", "Elton"])), 4) + self._test_qs(qs.filter(~Person.first_name.isIn(["Ciaran", "Elton"])), 96) + self._test_qs(qs.filter(Person.first_name.isNotIn(["Ciaran", "Elton"])), 96) + self._test_qs(qs.exclude(Person.first_name.isIn(["Ciaran", "Elton"])), 96) # In subquery - subquery = qs.filter(F.startsWith(Person.last_name, 'M')).only(Person.first_name) + subquery = qs.filter(F.startsWith(Person.last_name, "M")).only(Person.first_name) self._test_qs(qs.filter(Person.first_name.isIn(subquery)), 4) def test_comparison_operators(self): @@ -213,14 +212,14 @@ def test_date_functions(self): dt = datetime(2018, 12, 31, 11, 22, 33) self._test_func(F.toYear(d), 2018) self._test_func(F.toYear(dt), 2018) - self._test_func(F.toISOYear(dt, 'Europe/Athens'), 2019) # 2018-12-31 is ISO year 2019, week 1, day 1 + self._test_func(F.toISOYear(dt, "Europe/Athens"), 2019) # 2018-12-31 is ISO year 2019, week 1, day 1 self._test_func(F.toQuarter(d), 4) self._test_func(F.toQuarter(dt), 4) self._test_func(F.toMonth(d), 12) self._test_func(F.toMonth(dt), 12) self._test_func(F.toWeek(d), 52) self._test_func(F.toWeek(dt), 52) - self._test_func(F.toISOWeek(d), 1) # 2018-12-31 is ISO year 2019, week 1, day 1 + self._test_func(F.toISOWeek(d), 1) # 2018-12-31 is ISO year 2019, week 1, day 1 self._test_func(F.toISOWeek(dt), 1) self._test_func(F.toDayOfYear(d), 365) self._test_func(F.toDayOfYear(dt), 365) @@ -246,182 +245,218 @@ def test_date_functions(self): self._test_func(F.toStartOfTenMinutes(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc)) self._test_func(F.toStartOfWeek(dt), date(2018, 12, 30)) self._test_func(F.toTime(dt), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) - self._test_func(F.toUnixTimestamp(dt, 'UTC'), int(dt.replace(tzinfo=pytz.utc).timestamp())) + self._test_func(F.toUnixTimestamp(dt, "UTC"), int(dt.replace(tzinfo=pytz.utc).timestamp())) self._test_func(F.toYYYYMM(d), 201812) self._test_func(F.toYYYYMM(dt), 201812) - self._test_func(F.toYYYYMM(dt, 'Europe/Athens'), 201812) + self._test_func(F.toYYYYMM(dt, "Europe/Athens"), 201812) self._test_func(F.toYYYYMMDD(d), 20181231) self._test_func(F.toYYYYMMDD(dt), 20181231) - self._test_func(F.toYYYYMMDD(dt, 'Europe/Athens'), 20181231) + self._test_func(F.toYYYYMMDD(dt, "Europe/Athens"), 20181231) self._test_func(F.toYYYYMMDDhhmmss(d), 20181231000000) - self._test_func(F.toYYYYMMDDhhmmss(dt, 'Europe/Athens'), 20181231132233) + self._test_func(F.toYYYYMMDDhhmmss(dt, "Europe/Athens"), 20181231132233) self._test_func(F.toRelativeYearNum(dt), 2018) - self._test_func(F.toRelativeYearNum(dt, 'Europe/Athens'), 2018) + self._test_func(F.toRelativeYearNum(dt, "Europe/Athens"), 2018) self._test_func(F.toRelativeMonthNum(dt), 2018 * 12 + 12) - self._test_func(F.toRelativeMonthNum(dt, 'Europe/Athens'), 2018 * 12 + 12) + self._test_func(F.toRelativeMonthNum(dt, "Europe/Athens"), 2018 * 12 + 12) self._test_func(F.toRelativeWeekNum(dt), 2557) - self._test_func(F.toRelativeWeekNum(dt, 'Europe/Athens'), 2557) + self._test_func(F.toRelativeWeekNum(dt, "Europe/Athens"), 2557) self._test_func(F.toRelativeDayNum(dt), 17896) - self._test_func(F.toRelativeDayNum(dt, 'Europe/Athens'), 17896) + self._test_func(F.toRelativeDayNum(dt, "Europe/Athens"), 17896) self._test_func(F.toRelativeHourNum(dt), 429515) - self._test_func(F.toRelativeHourNum(dt, 'Europe/Athens'), 429515) + self._test_func(F.toRelativeHourNum(dt, "Europe/Athens"), 429515) self._test_func(F.toRelativeMinuteNum(dt), 25770922) - self._test_func(F.toRelativeMinuteNum(dt, 'Europe/Athens'), 25770922) + self._test_func(F.toRelativeMinuteNum(dt, "Europe/Athens"), 25770922) self._test_func(F.toRelativeSecondNum(dt), 1546255353) - self._test_func(F.toRelativeSecondNum(dt, 'Europe/Athens'), 1546255353) + self._test_func(F.toRelativeSecondNum(dt, "Europe/Athens"), 1546255353) self._test_func(F.timeSlot(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)) self._test_func(F.timeSlots(dt, 300), [datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)]) - self._test_func(F.formatDateTime(dt, '%D %T', 'Europe/Athens'), '12/31/18 13:22:33') + self._test_func(F.formatDateTime(dt, "%D %T", "Europe/Athens"), "12/31/18 13:22:33") self._test_func(F.addDays(d, 7), date(2019, 1, 7)) - self._test_func(F.addDays(dt, 7, 'Europe/Athens')) - self._test_func(F.addHours(dt, 7, 'Europe/Athens')) - self._test_func(F.addMinutes(dt, 7, 'Europe/Athens')) + self._test_func(F.addDays(dt, 7, "Europe/Athens")) + self._test_func(F.addHours(dt, 7, "Europe/Athens")) + self._test_func(F.addMinutes(dt, 7, "Europe/Athens")) self._test_func(F.addMonths(d, 7), date(2019, 7, 31)) - self._test_func(F.addMonths(dt, 7, 'Europe/Athens')) + self._test_func(F.addMonths(dt, 7, "Europe/Athens")) self._test_func(F.addQuarters(d, 7)) - self._test_func(F.addQuarters(dt, 7, 'Europe/Athens')) + self._test_func(F.addQuarters(dt, 7, "Europe/Athens")) self._test_func(F.addSeconds(d, 7)) - self._test_func(F.addSeconds(dt, 7, 'Europe/Athens')) + self._test_func(F.addSeconds(dt, 7, "Europe/Athens")) self._test_func(F.addWeeks(d, 7)) - self._test_func(F.addWeeks(dt, 7, 'Europe/Athens')) + self._test_func(F.addWeeks(dt, 7, "Europe/Athens")) self._test_func(F.addYears(d, 7)) - self._test_func(F.addYears(dt, 7, 'Europe/Athens')) + self._test_func(F.addYears(dt, 7, "Europe/Athens")) self._test_func(F.subtractDays(d, 3)) - self._test_func(F.subtractDays(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractDays(dt, 3, "Europe/Athens")) self._test_func(F.subtractHours(d, 3)) - self._test_func(F.subtractHours(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractHours(dt, 3, "Europe/Athens")) self._test_func(F.subtractMinutes(d, 3)) - self._test_func(F.subtractMinutes(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractMinutes(dt, 3, "Europe/Athens")) self._test_func(F.subtractMonths(d, 3)) - self._test_func(F.subtractMonths(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractMonths(dt, 3, "Europe/Athens")) self._test_func(F.subtractQuarters(d, 3)) - self._test_func(F.subtractQuarters(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractQuarters(dt, 3, "Europe/Athens")) self._test_func(F.subtractSeconds(d, 3)) - self._test_func(F.subtractSeconds(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractSeconds(dt, 3, "Europe/Athens")) self._test_func(F.subtractWeeks(d, 3)) - self._test_func(F.subtractWeeks(dt, 3, 'Europe/Athens')) + self._test_func(F.subtractWeeks(dt, 3, "Europe/Athens")) self._test_func(F.subtractYears(d, 3)) - self._test_func(F.subtractYears(dt, 3, 'Europe/Athens')) - self._test_func(F.now() + F.toIntervalSecond(3) + F.toIntervalMinute(3) + F.toIntervalHour(3) + F.toIntervalDay(3)) - self._test_func(F.now() + F.toIntervalWeek(3) + F.toIntervalMonth(3) + F.toIntervalQuarter(3) + F.toIntervalYear(3)) - self._test_func(F.now() + F.toIntervalSecond(3000) - F.toIntervalDay(3000) == F.now() + timedelta(seconds=3000, days=-3000)) + self._test_func(F.subtractYears(dt, 3, "Europe/Athens")) + self._test_func( + F.now() + F.toIntervalSecond(3) + F.toIntervalMinute(3) + F.toIntervalHour(3) + F.toIntervalDay(3) + ) + self._test_func( + F.now() + F.toIntervalWeek(3) + F.toIntervalMonth(3) + F.toIntervalQuarter(3) + F.toIntervalYear(3) + ) + self._test_func( + F.now() + F.toIntervalSecond(3000) - F.toIntervalDay(3000) == F.now() + timedelta(seconds=3000, days=-3000) + ) def test_date_functions__utc_only(self): if self.database.server_timezone != pytz.utc: - raise unittest.SkipTest('This test must run with UTC as the server timezone') + raise unittest.SkipTest("This test must run with UTC as the server timezone") d = date(2018, 12, 31) dt = datetime(2018, 12, 31, 11, 22, 33) - athens_tz = pytz.timezone('Europe/Athens') + athens_tz = pytz.timezone("Europe/Athens") self._test_func(F.toHour(dt), 11) self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc)) self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) - self._test_func(F.toTime(dt, 'Europe/Athens'), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) + self._test_func(F.toTime(dt, "Europe/Athens"), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) self._test_func(F.toTime(dt, athens_tz), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) - self._test_func(F.toTimeZone(dt, 'Europe/Athens'), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33))) - self._test_func(F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0)) # FIXME this may fail if the timing is just right + self._test_func(F.toTimeZone(dt, "Europe/Athens"), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33))) + self._test_func( + F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0) + ) # FIXME this may fail if the timing is just right self._test_func(F.today(), datetime.utcnow().date()) self._test_func(F.yesterday(), datetime.utcnow().date() - timedelta(days=1)) self._test_func(F.toYYYYMMDDhhmmss(dt), 20181231112233) - self._test_func(F.formatDateTime(dt, '%D %T'), '12/31/18 11:22:33') + self._test_func(F.formatDateTime(dt, "%D %T"), "12/31/18 11:22:33") self._test_func(F.addHours(d, 7), datetime(2018, 12, 31, 7, 0, 0, tzinfo=pytz.utc)) self._test_func(F.addMinutes(d, 7), datetime(2018, 12, 31, 0, 7, 0, tzinfo=pytz.utc)) def test_type_conversion_functions(self): - for f in (F.toUInt8, F.toUInt16, F.toUInt32, F.toUInt64, F.toInt8, F.toInt16, F.toInt32, F.toInt64, F.toFloat32, F.toFloat64): + for f in ( + F.toUInt8, + F.toUInt16, + F.toUInt32, + F.toUInt64, + F.toInt8, + F.toInt16, + F.toInt32, + F.toInt64, + F.toFloat32, + F.toFloat64, + ): self._test_func(f(17), 17) - self._test_func(f('17'), 17) - for f in (F.toUInt8OrZero, F.toUInt16OrZero, F.toUInt32OrZero, F.toUInt64OrZero, F.toInt8OrZero, F.toInt16OrZero, F.toInt32OrZero, F.toInt64OrZero, F.toFloat32OrZero, F.toFloat64OrZero): - self._test_func(f('17'), 17) - self._test_func(f('a'), 0) + self._test_func(f("17"), 17) + for f in ( + F.toUInt8OrZero, + F.toUInt16OrZero, + F.toUInt32OrZero, + F.toUInt64OrZero, + F.toInt8OrZero, + F.toInt16OrZero, + F.toInt32OrZero, + F.toInt64OrZero, + F.toFloat32OrZero, + F.toFloat64OrZero, + ): + self._test_func(f("17"), 17) + self._test_func(f("a"), 0) for f in (F.toDecimal32, F.toDecimal64, F.toDecimal128): - self._test_func(f(17.17, 2), Decimal('17.17')) - self._test_func(f('17.17', 2), Decimal('17.17')) - self._test_func(F.toDate('2018-12-31'), date(2018, 12, 31)) - self._test_func(F.toString(123), '123') - self._test_func(F.toFixedString('123', 5), '123') - self._test_func(F.toStringCutToZero('123\0'), '123') - self._test_func(F.CAST(17, 'String'), '17') - self._test_func(F.parseDateTimeBestEffort('31/12/2019 10:05AM', 'Europe/Athens')) + self._test_func(f(17.17, 2), Decimal("17.17")) + self._test_func(f("17.17", 2), Decimal("17.17")) + self._test_func(F.toDate("2018-12-31"), date(2018, 12, 31)) + self._test_func(F.toString(123), "123") + self._test_func(F.toFixedString("123", 5), "123") + self._test_func(F.toStringCutToZero("123\0"), "123") + self._test_func(F.CAST(17, "String"), "17") + self._test_func(F.parseDateTimeBestEffort("31/12/2019 10:05AM", "Europe/Athens")) with self.assertRaises(ServerError): - self._test_func(F.parseDateTimeBestEffort('foo')) - self._test_func(F.parseDateTimeBestEffortOrNull('31/12/2019 10:05AM', 'Europe/Athens')) - self._test_func(F.parseDateTimeBestEffortOrNull('foo'), None) - self._test_func(F.parseDateTimeBestEffortOrZero('31/12/2019 10:05AM', 'Europe/Athens')) - self._test_func(F.parseDateTimeBestEffortOrZero('foo'), DateTimeField.class_default) + self._test_func(F.parseDateTimeBestEffort("foo")) + self._test_func(F.parseDateTimeBestEffortOrNull("31/12/2019 10:05AM", "Europe/Athens")) + self._test_func(F.parseDateTimeBestEffortOrNull("foo"), None) + self._test_func(F.parseDateTimeBestEffortOrZero("31/12/2019 10:05AM", "Europe/Athens")) + self._test_func(F.parseDateTimeBestEffortOrZero("foo"), DateTimeField.class_default) def test_type_conversion_functions__utc_only(self): if self.database.server_timezone != pytz.utc: - raise unittest.SkipTest('This test must run with UTC as the server timezone') - self._test_func(F.toDateTime('2018-12-31 11:22:33'), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc)) - self._test_func(F.toDateTime64('2018-12-31 11:22:33.001', 6), datetime(2018, 12, 31, 11, 22, 33, 1000, tzinfo=pytz.utc)) - self._test_func(F.parseDateTimeBestEffort('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc)) - self._test_func(F.parseDateTimeBestEffortOrNull('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc)) - self._test_func(F.parseDateTimeBestEffortOrZero('31/12/2019 10:05AM'), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc)) + raise unittest.SkipTest("This test must run with UTC as the server timezone") + self._test_func(F.toDateTime("2018-12-31 11:22:33"), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc)) + self._test_func( + F.toDateTime64("2018-12-31 11:22:33.001", 6), datetime(2018, 12, 31, 11, 22, 33, 1000, tzinfo=pytz.utc) + ) + self._test_func(F.parseDateTimeBestEffort("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc)) + self._test_func( + F.parseDateTimeBestEffortOrNull("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc) + ) + self._test_func( + F.parseDateTimeBestEffortOrZero("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc) + ) def test_string_functions(self): - self._test_func(F.empty(''), 1) - self._test_func(F.empty('x'), 0) - self._test_func(F.notEmpty(''), 0) - self._test_func(F.notEmpty('x'), 1) - self._test_func(F.length('x'), 1) - self._test_func(F.lengthUTF8('x'), 1) - self._test_func(F.lower('Ab'), 'ab') - self._test_func(F.upper('Ab'), 'AB') - self._test_func(F.lowerUTF8('Ab'), 'ab') - self._test_func(F.upperUTF8('Ab'), 'AB') - self._test_func(F.reverse('Ab'), 'bA') - self._test_func(F.reverseUTF8('Ab'), 'bA') - self._test_func(F.concat('Ab', 'Cd', 'Ef'), 'AbCdEf') - self._test_func(F.substring('123456', 3, 2), '34') - self._test_func(F.substringUTF8('123456', 3, 2), '34') - self._test_func(F.appendTrailingCharIfAbsent('Hello', '!'), 'Hello!') - self._test_func(F.appendTrailingCharIfAbsent('Hello!', '!'), 'Hello!') - self._test_func(F.convertCharset(F.convertCharset('Hello', 'latin1', 'utf16'), 'utf16', 'latin1'), 'Hello') - self._test_func(F.startsWith('aaa', 'aa'), True) - self._test_func(F.startsWith('aaa', 'bb'), False) - self._test_func(F.endsWith('aaa', 'aa'), True) - self._test_func(F.endsWith('aaa', 'bb'), False) - self._test_func(F.trimLeft(' abc '), 'abc ') - self._test_func(F.trimRight(' abc '), ' abc') - self._test_func(F.trimBoth(' abc '), 'abc') - self._test_func(F.CRC32('whoops'), 3361378926) + self._test_func(F.empty(""), 1) + self._test_func(F.empty("x"), 0) + self._test_func(F.notEmpty(""), 0) + self._test_func(F.notEmpty("x"), 1) + self._test_func(F.length("x"), 1) + self._test_func(F.lengthUTF8("x"), 1) + self._test_func(F.lower("Ab"), "ab") + self._test_func(F.upper("Ab"), "AB") + self._test_func(F.lowerUTF8("Ab"), "ab") + self._test_func(F.upperUTF8("Ab"), "AB") + self._test_func(F.reverse("Ab"), "bA") + self._test_func(F.reverseUTF8("Ab"), "bA") + self._test_func(F.concat("Ab", "Cd", "Ef"), "AbCdEf") + self._test_func(F.substring("123456", 3, 2), "34") + self._test_func(F.substringUTF8("123456", 3, 2), "34") + self._test_func(F.appendTrailingCharIfAbsent("Hello", "!"), "Hello!") + self._test_func(F.appendTrailingCharIfAbsent("Hello!", "!"), "Hello!") + self._test_func(F.convertCharset(F.convertCharset("Hello", "latin1", "utf16"), "utf16", "latin1"), "Hello") + self._test_func(F.startsWith("aaa", "aa"), True) + self._test_func(F.startsWith("aaa", "bb"), False) + self._test_func(F.endsWith("aaa", "aa"), True) + self._test_func(F.endsWith("aaa", "bb"), False) + self._test_func(F.trimLeft(" abc "), "abc ") + self._test_func(F.trimRight(" abc "), " abc") + self._test_func(F.trimBoth(" abc "), "abc") + self._test_func(F.CRC32("whoops"), 3361378926) def test_string_search_functions(self): - self._test_func(F.position('Hello, world!', '!'), 13) - self._test_func(F.positionCaseInsensitive('Hello, world!', 'hello'), 1) - self._test_func(F.positionUTF8('Привет, мир!', '!'), 12) - self._test_func(F.positionCaseInsensitiveUTF8('Привет, мир!', 'Мир'), 9) - self._test_func(F.like('Hello, world!', '%ll%'), 1) - self._test_func(F.notLike('Hello, world!', '%ll%'), 0) - self._test_func(F.match('Hello, world!', '[lmnop]{3}'), 1) - self._test_func(F.extract('Hello, world!', '[lmnop]{3}'), 'llo') - self._test_func(F.extractAll('Hello, world!', '[a-z]+'), ['ello', 'world']) - self._test_func(F.ngramDistance('Hello', 'Hello'), 0) - self._test_func(F.ngramDistanceCaseInsensitive('Hello', 'hello'), 0) - self._test_func(F.ngramDistanceUTF8('Hello', 'Hello'), 0) - self._test_func(F.ngramDistanceCaseInsensitiveUTF8('Hello', 'hello'), 0) - self._test_func(F.ngramSearch('Hello', 'Hello'), 1) - self._test_func(F.ngramSearchCaseInsensitive('Hello', 'hello'), 1) - self._test_func(F.ngramSearchUTF8('Hello', 'Hello'), 1) - self._test_func(F.ngramSearchCaseInsensitiveUTF8('Hello', 'hello'), 1) + self._test_func(F.position("Hello, world!", "!"), 13) + self._test_func(F.positionCaseInsensitive("Hello, world!", "hello"), 1) + self._test_func(F.positionUTF8("Привет, мир!", "!"), 12) + self._test_func(F.positionCaseInsensitiveUTF8("Привет, мир!", "Мир"), 9) + self._test_func(F.like("Hello, world!", "%ll%"), 1) + self._test_func(F.notLike("Hello, world!", "%ll%"), 0) + self._test_func(F.match("Hello, world!", "[lmnop]{3}"), 1) + self._test_func(F.extract("Hello, world!", "[lmnop]{3}"), "llo") + self._test_func(F.extractAll("Hello, world!", "[a-z]+"), ["ello", "world"]) + self._test_func(F.ngramDistance("Hello", "Hello"), 0) + self._test_func(F.ngramDistanceCaseInsensitive("Hello", "hello"), 0) + self._test_func(F.ngramDistanceUTF8("Hello", "Hello"), 0) + self._test_func(F.ngramDistanceCaseInsensitiveUTF8("Hello", "hello"), 0) + self._test_func(F.ngramSearch("Hello", "Hello"), 1) + self._test_func(F.ngramSearchCaseInsensitive("Hello", "hello"), 1) + self._test_func(F.ngramSearchUTF8("Hello", "Hello"), 1) + self._test_func(F.ngramSearchCaseInsensitiveUTF8("Hello", "hello"), 1) def test_base64_functions(self): try: - self._test_func(F.base64Decode(F.base64Encode('Hello')), 'Hello') - self._test_func(F.tryBase64Decode(F.base64Encode('Hello')), 'Hello') - self._test_func(F.tryBase64Decode(':-)')) + self._test_func(F.base64Decode(F.base64Encode("Hello")), "Hello") + self._test_func(F.tryBase64Decode(F.base64Encode("Hello")), "Hello") + self._test_func(F.tryBase64Decode(":-)")) except ServerError as e: # ClickHouse version that doesn't support these functions raise unittest.SkipTest(e.message) def test_replace_functions(self): - haystack = 'hello' - self._test_func(F.replace(haystack, 'l', 'L'), 'heLLo') - self._test_func(F.replaceAll(haystack, 'l', 'L'), 'heLLo') - self._test_func(F.replaceOne(haystack, 'l', 'L'), 'heLlo') - self._test_func(F.replaceRegexpAll(haystack, '[eo]', 'X'), 'hXllX') - self._test_func(F.replaceRegexpOne(haystack, '[eo]', 'X'), 'hXllo') - self._test_func(F.regexpQuoteMeta('[eo]'), '\\[eo\\]') + haystack = "hello" + self._test_func(F.replace(haystack, "l", "L"), "heLLo") + self._test_func(F.replaceAll(haystack, "l", "L"), "heLLo") + self._test_func(F.replaceOne(haystack, "l", "L"), "heLlo") + self._test_func(F.replaceRegexpAll(haystack, "[eo]", "X"), "hXllX") + self._test_func(F.replaceRegexpOne(haystack, "[eo]", "X"), "hXllo") + self._test_func(F.regexpQuoteMeta("[eo]"), "\\[eo\\]") def test_math_functions(self): x = 17 @@ -515,15 +550,15 @@ def test_array_functions(self): self._test_func(F.arrayDifference(arr), [0, 1, 1]) self._test_func(F.arrayDistinct(arr + arr), arr) self._test_func(F.arrayIntersect(arr, [3, 4]), [3]) - self._test_func(F.arrayReduce('min', arr), 1) + self._test_func(F.arrayReduce("min", arr), 1) self._test_func(F.arrayReverse(arr), [3, 2, 1]) def test_split_and_merge_functions(self): - self._test_func(F.splitByChar('_', 'a_b_c'), ['a', 'b', 'c']) - self._test_func(F.splitByString('__', 'a__b__c'), ['a', 'b', 'c']) - self._test_func(F.arrayStringConcat(['a', 'b', 'c']), 'abc') - self._test_func(F.arrayStringConcat(['a', 'b', 'c'], '_'), 'a_b_c') - self._test_func(F.alphaTokens('aaa.bbb.111'), ['aaa', 'bbb']) + self._test_func(F.splitByChar("_", "a_b_c"), ["a", "b", "c"]) + self._test_func(F.splitByString("__", "a__b__c"), ["a", "b", "c"]) + self._test_func(F.arrayStringConcat(["a", "b", "c"]), "abc") + self._test_func(F.arrayStringConcat(["a", "b", "c"], "_"), "a_b_c") + self._test_func(F.alphaTokens("aaa.bbb.111"), ["aaa", "bbb"]) def test_bit_functions(self): x = 17 @@ -546,10 +581,12 @@ def test_bit_functions(self): def test_bitmap_functions(self): self._test_func(F.bitmapToArray(F.bitmapBuild([1, 2, 3])), [1, 2, 3]) self._test_func(F.bitmapContains(F.bitmapBuild([1, 5, 7, 9]), F.toUInt32(9)), 1) - self._test_func(F.bitmapHasAny(F.bitmapBuild([1,2,3]), F.bitmapBuild([3,4,5])), 1) - self._test_func(F.bitmapHasAll(F.bitmapBuild([1,2,3]), F.bitmapBuild([3,4,5])), 0) + self._test_func(F.bitmapHasAny(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 1) + self._test_func(F.bitmapHasAll(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 0) self._test_func(F.bitmapToArray(F.bitmapAnd(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [3]) - self._test_func(F.bitmapToArray(F.bitmapOr(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2, 3, 4, 5]) + self._test_func( + F.bitmapToArray(F.bitmapOr(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2, 3, 4, 5] + ) self._test_func(F.bitmapToArray(F.bitmapXor(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2, 4, 5]) self._test_func(F.bitmapToArray(F.bitmapAndnot(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2]) self._test_func(F.bitmapCardinality(F.bitmapBuild([1, 2, 3, 4, 5])), 5) @@ -559,10 +596,10 @@ def test_bitmap_functions(self): self._test_func(F.bitmapAndnotCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 2) def test_hash_functions(self): - args = ['x', 'y', 'z'] + args = ["x", "y", "z"] x = 17 - s = 'hello' - url = 'http://example.com/a/b/c/d' + s = "hello" + url = "http://example.com/a/b/c/d" self._test_func(F.hex(F.MD5(s))) self._test_func(F.hex(F.sipHash128(s))) self._test_func(F.hex(F.cityHash64(*args))) @@ -594,17 +631,18 @@ def test_rand_functions(self): self._test_func(F.rand(17)) self._test_func(F.rand64()) self._test_func(F.rand64(17)) - if self.database.server_version >= (19, 15): # buggy in older versions + if self.database.server_version >= (19, 15): # buggy in older versions self._test_func(F.randConstant()) self._test_func(F.randConstant(17)) def test_encoding_functions(self): - self._test_func(F.hex(F.unhex('0FA1')), '0FA1') + self._test_func(F.hex(F.unhex("0FA1")), "0FA1") self._test_func(F.bitmaskToArray(17)) self._test_func(F.bitmaskToList(18)) def test_uuid_functions(self): from uuid import UUID + uuid = self._test_func(F.generateUUIDv4()) self.assertEqual(type(uuid), UUID) s = str(uuid) @@ -612,17 +650,20 @@ def test_uuid_functions(self): self._test_func(F.UUIDNumToString(F.UUIDStringToNum(s)), s) def test_ip_funcs(self): - self._test_func(F.IPv4NumToString(F.toUInt32(1)), '0.0.0.1') - self._test_func(F.IPv4NumToStringClassC(F.toUInt32(1)), '0.0.0.xxx') - self._test_func(F.IPv4StringToNum('0.0.0.17'), 17) - self._test_func(F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum('192.168.0.1'))), '::ffff:192.168.0.1') - self._test_func(F.IPv6NumToString(F.IPv6StringToNum('2a02:6b8::11')), '2a02:6b8::11') - self._test_func(F.toIPv4('10.20.30.40'), IPv4Address('10.20.30.40')) - self._test_func(F.toIPv6('2001:438:ffff::407d:1bc1'), IPv6Address('2001:438:ffff::407d:1bc1')) - self._test_func(F.IPv4CIDRToRange(F.toIPv4('192.168.5.2'), 16), - [IPv4Address('192.168.0.0'), IPv4Address('192.168.255.255')]) - self._test_func(F.IPv6CIDRToRange(F.toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32), - [IPv6Address('2001:db8::'), IPv6Address('2001:db8:ffff:ffff:ffff:ffff:ffff:ffff')]) + self._test_func(F.IPv4NumToString(F.toUInt32(1)), "0.0.0.1") + self._test_func(F.IPv4NumToStringClassC(F.toUInt32(1)), "0.0.0.xxx") + self._test_func(F.IPv4StringToNum("0.0.0.17"), 17) + self._test_func(F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum("192.168.0.1"))), "::ffff:192.168.0.1") + self._test_func(F.IPv6NumToString(F.IPv6StringToNum("2a02:6b8::11")), "2a02:6b8::11") + self._test_func(F.toIPv4("10.20.30.40"), IPv4Address("10.20.30.40")) + self._test_func(F.toIPv6("2001:438:ffff::407d:1bc1"), IPv6Address("2001:438:ffff::407d:1bc1")) + self._test_func( + F.IPv4CIDRToRange(F.toIPv4("192.168.5.2"), 16), [IPv4Address("192.168.0.0"), IPv4Address("192.168.255.255")] + ) + self._test_func( + F.IPv6CIDRToRange(F.toIPv6("2001:0db8:0000:85a3:0000:0000:ac1f:8001"), 32), + [IPv6Address("2001:db8::"), IPv6Address("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")], + ) def test_aggregate_funcs(self): self._test_aggr(F.any(Person.first_name)) @@ -649,32 +690,32 @@ def test_aggregate_funcs(self): self._test_aggr(F.varSamp(Person.height)) def test_aggregate_funcs__or_default(self): - self.database.raw('TRUNCATE TABLE person') + self.database.raw("TRUNCATE TABLE person") self._test_aggr(F.countOrDefault(), 0) self._test_aggr(F.maxOrDefault(Person.height), 0) def test_aggregate_funcs__or_null(self): - self.database.raw('TRUNCATE TABLE person') + self.database.raw("TRUNCATE TABLE person") self._test_aggr(F.countOrNull(), None) self._test_aggr(F.maxOrNull(Person.height), None) def test_aggregate_funcs__if(self): - self._test_aggr(F.argMinIf(Person.first_name, Person.height, Person.last_name > 'H')) - self._test_aggr(F.countIf(Person.last_name > 'H'), 57) - self._test_aggr(F.minIf(Person.height, Person.last_name > 'H'), 1.6) + self._test_aggr(F.argMinIf(Person.first_name, Person.height, Person.last_name > "H")) + self._test_aggr(F.countIf(Person.last_name > "H"), 57) + self._test_aggr(F.minIf(Person.height, Person.last_name > "H"), 1.6) def test_aggregate_funcs__or_default_if(self): - self._test_aggr(F.argMinOrDefaultIf(Person.first_name, Person.height, Person.last_name > 'Z')) - self._test_aggr(F.countOrDefaultIf(Person.last_name > 'Z'), 0) - self._test_aggr(F.minOrDefaultIf(Person.height, Person.last_name > 'Z'), 0) + self._test_aggr(F.argMinOrDefaultIf(Person.first_name, Person.height, Person.last_name > "Z")) + self._test_aggr(F.countOrDefaultIf(Person.last_name > "Z"), 0) + self._test_aggr(F.minOrDefaultIf(Person.height, Person.last_name > "Z"), 0) def test_aggregate_funcs__or_null_if(self): - self._test_aggr(F.argMinOrNullIf(Person.first_name, Person.height, Person.last_name > 'Z')) - self._test_aggr(F.countOrNullIf(Person.last_name > 'Z'), None) - self._test_aggr(F.minOrNullIf(Person.height, Person.last_name > 'Z'), None) + self._test_aggr(F.argMinOrNullIf(Person.first_name, Person.height, Person.last_name > "Z")) + self._test_aggr(F.countOrNullIf(Person.last_name > "Z"), None) + self._test_aggr(F.minOrNullIf(Person.height, Person.last_name > "Z"), None) def test_quantile_funcs(self): - cond = Person.last_name > 'H' + cond = Person.last_name > "H" weight_expr = F.toUInt32(F.round(Person.height)) # Quantile self._test_aggr(F.quantile(0.9)(Person.height)) @@ -712,13 +753,13 @@ def test_quantile_funcs(self): def test_top_k_funcs(self): self._test_aggr(F.topK(3)(Person.height)) self._test_aggr(F.topKOrDefault(3)(Person.height)) - self._test_aggr(F.topKIf(3)(Person.height, Person.last_name > 'H')) - self._test_aggr(F.topKOrDefaultIf(3)(Person.height, Person.last_name > 'H')) + self._test_aggr(F.topKIf(3)(Person.height, Person.last_name > "H")) + self._test_aggr(F.topKOrDefaultIf(3)(Person.height, Person.last_name > "H")) weight_expr = F.toUInt32(F.round(Person.height)) self._test_aggr(F.topKWeighted(3)(Person.height, weight_expr)) self._test_aggr(F.topKWeightedOrDefault(3)(Person.height, weight_expr)) - self._test_aggr(F.topKWeightedIf(3)(Person.height, weight_expr, Person.last_name > 'H')) - self._test_aggr(F.topKWeightedOrDefaultIf(3)(Person.height, weight_expr, Person.last_name > 'H')) + self._test_aggr(F.topKWeightedIf(3)(Person.height, weight_expr, Person.last_name > "H")) + self._test_aggr(F.topKWeightedOrDefaultIf(3)(Person.height, weight_expr, Person.last_name > "H")) def test_null_funcs(self): self._test_func(F.ifNull(17, 18), 17) diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 63ce7c3..7bf9bba 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -4,11 +4,10 @@ class IndexesTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) if self.database.server_version < (20, 1, 2, 4): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") def tearDown(self): self.database.drop_database() @@ -29,4 +28,4 @@ class ModelWithIndexes(Model): i4 = Index(F.lower(f2), type=Index.tokenbf_v1(256, 2, 0), granularity=2) i5 = Index((F.toQuarter(date), f2), type=Index.bloom_filter(), granularity=3) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index e15899f..ae36be0 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -9,17 +9,16 @@ class InheritanceTestCase(unittest.TestCase): - def assertFieldNames(self, model_class, names): self.assertEqual(names, list(model_class.fields())) def test_field_inheritance(self): - self.assertFieldNames(ParentModel, ['date_field', 'int_field']) - self.assertFieldNames(Model1, ['date_field', 'int_field', 'string_field']) - self.assertFieldNames(Model2, ['date_field', 'int_field', 'float_field']) + self.assertFieldNames(ParentModel, ["date_field", "int_field"]) + self.assertFieldNames(Model1, ["date_field", "int_field", "string_field"]) + self.assertFieldNames(Model2, ["date_field", "int_field", "float_field"]) def test_create_table_sql(self): - default_db = Database('default') + default_db = Database("default") sql1 = ParentModel.create_table_sql(default_db) sql2 = Model1.create_table_sql(default_db) sql3 = Model2.create_table_sql(default_db) @@ -28,11 +27,11 @@ def test_create_table_sql(self): self.assertNotEqual(sql2, sql3) def test_get_field(self): - self.assertIsNotNone(ParentModel().get_field('date_field')) - self.assertIsNone(ParentModel().get_field('string_field')) - self.assertIsNotNone(Model1().get_field('date_field')) - self.assertIsNotNone(Model1().get_field('string_field')) - self.assertIsNone(Model1().get_field('float_field')) + self.assertIsNotNone(ParentModel().get_field("date_field")) + self.assertIsNone(ParentModel().get_field("string_field")) + self.assertIsNotNone(Model1().get_field("date_field")) + self.assertIsNotNone(Model1().get_field("string_field")) + self.assertIsNone(Model1().get_field("float_field")) class ParentModel(Model): @@ -40,7 +39,7 @@ class ParentModel(Model): date_field = DateField() int_field = Int32Field() - engine = MergeTree('date_field', ('int_field', 'date_field')) + engine = MergeTree("date_field", ("int_field", "date_field")) class Model1(ParentModel): diff --git a/tests/test_ip_fields.py b/tests/test_ip_fields.py index 34301df..2f81064 100644 --- a/tests/test_ip_fields.py +++ b/tests/test_ip_fields.py @@ -7,54 +7,50 @@ class IPFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) def tearDown(self): self.database.drop_database() def test_ipv4_field(self): if self.database.server_version < (19, 17): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") # Create a model class TestModel(Model): i = Int16Field() f = IPv4Field() engine = Memory() + self.database.create_table(TestModel) # Check valid values (all values are the same ip) - values = [ - '1.2.3.4', - b'\x01\x02\x03\x04', - 16909060, - IPv4Address('1.2.3.4') - ] + values = ["1.2.3.4", b"\x01\x02\x03\x04", 16909060, IPv4Address("1.2.3.4")] for index, value in enumerate(values): rec = TestModel(i=index, f=value) self.database.insert([rec]) for rec in TestModel.objects_in(self.database): self.assertEqual(rec.f, IPv4Address(values[0])) # Check invalid values - for value in [None, 'zzz', -1, '123']: + for value in [None, "zzz", -1, "123"]: with self.assertRaises(ValueError): TestModel(i=1, f=value) def test_ipv6_field(self): if self.database.server_version < (19, 17): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") # Create a model class TestModel(Model): i = Int16Field() f = IPv6Field() engine = Memory() + self.database.create_table(TestModel) # Check valid values (all values are the same ip) values = [ - '2a02:e980:1e::1', - b'*\x02\xe9\x80\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01', + "2a02:e980:1e::1", + b"*\x02\xe9\x80\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", 55842696359362256756849388082849382401, - IPv6Address('2a02:e980:1e::1') + IPv6Address("2a02:e980:1e::1"), ] for index, value in enumerate(values): rec = TestModel(i=index, f=value) @@ -62,7 +58,6 @@ class TestModel(Model): for rec in TestModel.objects_in(self.database): self.assertEqual(rec.f, IPv6Address(values[0])) # Check invalid values - for value in [None, 'zzz', -1, '123']: + for value in [None, "zzz", -1, "123"]: with self.assertRaises(ValueError): TestModel(i=1, f=value) - diff --git a/tests/test_join.py b/tests/test_join.py index b00c523..2a72b41 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -1,4 +1,3 @@ - import unittest import json @@ -6,9 +5,8 @@ class JoinTest(unittest.TestCase): - def setUp(self): - self.database = database.Database('test-db', log_statements=True) + self.database = database.Database("test-db", log_statements=True) self.database.create_table(Foo) self.database.create_table(Bar) self.database.insert([Foo(id=i) for i in range(3)]) @@ -29,8 +27,16 @@ def test_with_db_name(self): self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN $db.{} USING id".format(Foo.table_name(), Bar.table_name())) def test_with_subquery(self): - self.print_res("SELECT b FROM {} ALL LEFT JOIN (SELECT * from {}) subquery USING id".format(Foo.table_name(), Bar.table_name())) - self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN (SELECT * from $db.{}) subquery USING id".format(Foo.table_name(), Bar.table_name())) + self.print_res( + "SELECT b FROM {} ALL LEFT JOIN (SELECT * from {}) subquery USING id".format( + Foo.table_name(), Bar.table_name() + ) + ) + self.print_res( + "SELECT b FROM $db.{} ALL LEFT JOIN (SELECT * from $db.{}) subquery USING id".format( + Foo.table_name(), Bar.table_name() + ) + ) class Foo(models.Model): diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index 93a9ee0..b03f211 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -9,24 +9,21 @@ class MaterializedFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(ModelWithMaterializedFields) def tearDown(self): self.database.drop_database() def test_insert_and_select(self): - instance = ModelWithMaterializedFields( - date_time_field='2016-08-30 11:00:00', - int_field=-10, - str_field='TEST' - ) + instance = ModelWithMaterializedFields(date_time_field="2016-08-30 11:00:00", int_field=-10, str_field="TEST") self.database.insert([instance]) # We can't select * from table, as it doesn't select materialized and alias fields - query = 'SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str, mat_func' \ - ' FROM $db.%s ORDER BY mat_date' % ModelWithMaterializedFields.table_name() + query = ( + "SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str, mat_func" + " FROM $db.%s ORDER BY mat_date" % ModelWithMaterializedFields.table_name() + ) for model_cls in (ModelWithMaterializedFields, None): results = list(self.database.select(query, model_cls)) self.assertEqual(len(results), 1) @@ -41,7 +38,7 @@ def test_insert_and_select(self): def test_assignment_error(self): # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. instance = ModelWithMaterializedFields() - for value in ('x', [date.today()], ['aaa'], [None]): + for value in ("x", [date.today()], ["aaa"], [None]): with self.assertRaises(ValueError): instance.mat_date = value @@ -51,10 +48,10 @@ def test_wrong_field(self): def test_duplicate_default(self): with self.assertRaises(AssertionError): - StringField(materialized='str_field', default='with default') + StringField(materialized="str_field", default="with default") with self.assertRaises(AssertionError): - StringField(materialized='str_field', alias='str_field') + StringField(materialized="str_field", alias="str_field") def test_default_value(self): instance = ModelWithMaterializedFields() @@ -66,9 +63,9 @@ class ModelWithMaterializedFields(Model): date_time_field = DateTimeField() str_field = StringField() - mat_str = StringField(materialized='lower(str_field)') - mat_int = Int32Field(materialized='abs(int_field)') - mat_date = DateField(materialized=u'toDate(date_time_field)') + mat_str = StringField(materialized="lower(str_field)") + mat_int = Int32Field(materialized="abs(int_field)") + mat_date = DateField(materialized=u"toDate(date_time_field)") mat_func = StringField(materialized=F.lower(str_field)) - engine = MergeTree('mat_date', ('mat_date',)) + engine = MergeTree("mat_date", ("mat_date",)) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index faaf4e5..093fd41 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -7,20 +7,22 @@ from clickhouse_orm.migrations import MigrationHistory from enum import Enum + # Add tests to path so that migrations will be importable import sys, os + sys.path.append(os.path.dirname(__file__)) import logging -logging.basicConfig(level=logging.DEBUG, format='%(message)s') + +logging.basicConfig(level=logging.DEBUG, format="%(message)s") logging.getLogger("requests").setLevel(logging.WARNING) class MigrationsTestCase(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.drop_table(MigrationHistory) def tearDown(self): @@ -35,123 +37,157 @@ def get_table_fields(self, model_class): return [(row.name, row.type) for row in self.database.select(query)] def get_table_def(self, model_class): - return self.database.raw('SHOW CREATE TABLE $db.`%s`' % model_class.table_name()) + return self.database.raw("SHOW CREATE TABLE $db.`%s`" % model_class.table_name()) def test_migrations(self): # Creation and deletion of table - self.database.migrate('tests.sample_migrations', 1) + self.database.migrate("tests.sample_migrations", 1) self.assertTrue(self.table_exists(Model1)) - self.database.migrate('tests.sample_migrations', 2) + self.database.migrate("tests.sample_migrations", 2) self.assertFalse(self.table_exists(Model1)) - self.database.migrate('tests.sample_migrations', 3) + self.database.migrate("tests.sample_migrations", 3) self.assertTrue(self.table_exists(Model1)) # Adding, removing and altering simple fields - self.assertEqual(self.get_table_fields(Model1), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')]) - self.database.migrate('tests.sample_migrations', 4) - self.assertEqual(self.get_table_fields(Model2), [('date', 'Date'), ('f1', 'Int32'), ('f3', 'Float32'), ('f2', 'String'), ('f4', 'String'), ('f5', 'Array(UInt64)')]) - self.database.migrate('tests.sample_migrations', 5) - self.assertEqual(self.get_table_fields(Model3), [('date', 'Date'), ('f1', 'Int64'), ('f3', 'Float64'), ('f4', 'String')]) + self.assertEqual(self.get_table_fields(Model1), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) + self.database.migrate("tests.sample_migrations", 4) + self.assertEqual( + self.get_table_fields(Model2), + [ + ("date", "Date"), + ("f1", "Int32"), + ("f3", "Float32"), + ("f2", "String"), + ("f4", "String"), + ("f5", "Array(UInt64)"), + ], + ) + self.database.migrate("tests.sample_migrations", 5) + self.assertEqual( + self.get_table_fields(Model3), [("date", "Date"), ("f1", "Int64"), ("f3", "Float64"), ("f4", "String")] + ) # Altering enum fields - self.database.migrate('tests.sample_migrations', 6) + self.database.migrate("tests.sample_migrations", 6) self.assertTrue(self.table_exists(EnumModel1)) - self.assertEqual(self.get_table_fields(EnumModel1), - [('date', 'Date'), ('f1', "Enum8('dog' = 1, 'cat' = 2, 'cow' = 3)")]) - self.database.migrate('tests.sample_migrations', 7) + self.assertEqual( + self.get_table_fields(EnumModel1), [("date", "Date"), ("f1", "Enum8('dog' = 1, 'cat' = 2, 'cow' = 3)")] + ) + self.database.migrate("tests.sample_migrations", 7) self.assertTrue(self.table_exists(EnumModel1)) - self.assertEqual(self.get_table_fields(EnumModel2), - [('date', 'Date'), ('f1', "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")]) + self.assertEqual( + self.get_table_fields(EnumModel2), + [("date", "Date"), ("f1", "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")], + ) # Materialized fields and alias fields - self.database.migrate('tests.sample_migrations', 8) + self.database.migrate("tests.sample_migrations", 8) self.assertTrue(self.table_exists(MaterializedModel)) - self.assertEqual(self.get_table_fields(MaterializedModel), - [('date_time', "DateTime"), ('date', 'Date')]) - self.database.migrate('tests.sample_migrations', 9) + self.assertEqual(self.get_table_fields(MaterializedModel), [("date_time", "DateTime"), ("date", "Date")]) + self.database.migrate("tests.sample_migrations", 9) self.assertTrue(self.table_exists(AliasModel)) - self.assertEqual(self.get_table_fields(AliasModel), - [('date', 'Date'), ('date_alias', "Date")]) + self.assertEqual(self.get_table_fields(AliasModel), [("date", "Date"), ("date_alias", "Date")]) # Buffer models creation and alteration - self.database.migrate('tests.sample_migrations', 10) + self.database.migrate("tests.sample_migrations", 10) self.assertTrue(self.table_exists(Model4)) self.assertTrue(self.table_exists(Model4Buffer)) - self.assertEqual(self.get_table_fields(Model4), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')]) - self.assertEqual(self.get_table_fields(Model4Buffer), [('date', 'Date'), ('f1', 'Int32'), ('f2', 'String')]) - self.database.migrate('tests.sample_migrations', 11) - self.assertEqual(self.get_table_fields(Model4), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')]) - self.assertEqual(self.get_table_fields(Model4Buffer), [('date', 'Date'), ('f3', 'DateTime'), ('f2', 'String')]) + self.assertEqual(self.get_table_fields(Model4), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) + self.assertEqual(self.get_table_fields(Model4Buffer), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) + self.database.migrate("tests.sample_migrations", 11) + self.assertEqual(self.get_table_fields(Model4), [("date", "Date"), ("f3", "DateTime"), ("f2", "String")]) + self.assertEqual(self.get_table_fields(Model4Buffer), [("date", "Date"), ("f3", "DateTime"), ("f2", "String")]) - self.database.migrate('tests.sample_migrations', 12) + self.database.migrate("tests.sample_migrations", 12) self.assertEqual(self.database.count(Model3), 3) - data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)] + data = [item.f1 for item in self.database.select("SELECT f1 FROM $table ORDER BY f1", model_class=Model3)] self.assertListEqual(data, [1, 2, 3]) - self.database.migrate('tests.sample_migrations', 13) + self.database.migrate("tests.sample_migrations", 13) self.assertEqual(self.database.count(Model3), 4) - data = [item.f1 for item in self.database.select('SELECT f1 FROM $table ORDER BY f1', model_class=Model3)] + data = [item.f1 for item in self.database.select("SELECT f1 FROM $table ORDER BY f1", model_class=Model3)] self.assertListEqual(data, [1, 2, 3, 4]) - self.database.migrate('tests.sample_migrations', 14) + self.database.migrate("tests.sample_migrations", 14) self.assertTrue(self.table_exists(MaterializedModel1)) - self.assertEqual(self.get_table_fields(MaterializedModel1), - [('date_time', 'DateTime'), ('int_field', 'Int8'), ('date', 'Date'), ('int_field_plus_one', 'Int8')]) + self.assertEqual( + self.get_table_fields(MaterializedModel1), + [("date_time", "DateTime"), ("int_field", "Int8"), ("date", "Date"), ("int_field_plus_one", "Int8")], + ) self.assertTrue(self.table_exists(AliasModel1)) - self.assertEqual(self.get_table_fields(AliasModel1), - [('date', 'Date'), ('int_field', 'Int8'), ('date_alias', 'Date'), ('int_field_plus_one', 'Int8')]) + self.assertEqual( + self.get_table_fields(AliasModel1), + [("date", "Date"), ("int_field", "Int8"), ("date_alias", "Date"), ("int_field_plus_one", "Int8")], + ) # Codecs and low cardinality - self.database.migrate('tests.sample_migrations', 15) + self.database.migrate("tests.sample_migrations", 15) self.assertTrue(self.table_exists(Model4_compressed)) if self.database.has_low_cardinality_support: - self.assertEqual(self.get_table_fields(Model2LowCardinality), - [('date', 'Date'), ('f1', 'LowCardinality(Int32)'), ('f3', 'LowCardinality(Float32)'), - ('f2', 'LowCardinality(String)'), ('f4', 'LowCardinality(Nullable(String))'), ('f5', 'Array(LowCardinality(UInt64))')]) + self.assertEqual( + self.get_table_fields(Model2LowCardinality), + [ + ("date", "Date"), + ("f1", "LowCardinality(Int32)"), + ("f3", "LowCardinality(Float32)"), + ("f2", "LowCardinality(String)"), + ("f4", "LowCardinality(Nullable(String))"), + ("f5", "Array(LowCardinality(UInt64))"), + ], + ) else: - logging.warning('No support for low cardinality') - self.assertEqual(self.get_table_fields(Model2), - [('date', 'Date'), ('f1', 'Int32'), ('f3', 'Float32'), ('f2', 'String'), ('f4', 'Nullable(String)'), - ('f5', 'Array(UInt64)')]) + logging.warning("No support for low cardinality") + self.assertEqual( + self.get_table_fields(Model2), + [ + ("date", "Date"), + ("f1", "Int32"), + ("f3", "Float32"), + ("f2", "String"), + ("f4", "Nullable(String)"), + ("f5", "Array(UInt64)"), + ], + ) if self.database.server_version >= (19, 14, 3, 3): # Creating constraints - self.database.migrate('tests.sample_migrations', 16) + self.database.migrate("tests.sample_migrations", 16) self.assertTrue(self.table_exists(ModelWithConstraints)) - self.database.insert([ModelWithConstraints(f1=101, f2='a')]) + self.database.insert([ModelWithConstraints(f1=101, f2="a")]) with self.assertRaises(ServerError): - self.database.insert([ModelWithConstraints(f1=99, f2='a')]) + self.database.insert([ModelWithConstraints(f1=99, f2="a")]) with self.assertRaises(ServerError): - self.database.insert([ModelWithConstraints(f1=101, f2='x')]) + self.database.insert([ModelWithConstraints(f1=101, f2="x")]) # Modifying constraints - self.database.migrate('tests.sample_migrations', 17) - self.database.insert([ModelWithConstraints(f1=99, f2='a')]) + self.database.migrate("tests.sample_migrations", 17) + self.database.insert([ModelWithConstraints(f1=99, f2="a")]) with self.assertRaises(ServerError): - self.database.insert([ModelWithConstraints(f1=101, f2='a')]) + self.database.insert([ModelWithConstraints(f1=101, f2="a")]) with self.assertRaises(ServerError): - self.database.insert([ModelWithConstraints(f1=99, f2='x')]) + self.database.insert([ModelWithConstraints(f1=99, f2="x")]) if self.database.server_version >= (20, 1, 2, 4): # Creating indexes - self.database.migrate('tests.sample_migrations', 18) + self.database.migrate("tests.sample_migrations", 18) self.assertTrue(self.table_exists(ModelWithIndex)) - self.assertIn('INDEX index ', self.get_table_def(ModelWithIndex)) - self.assertIn('INDEX another_index ', self.get_table_def(ModelWithIndex)) + self.assertIn("INDEX index ", self.get_table_def(ModelWithIndex)) + self.assertIn("INDEX another_index ", self.get_table_def(ModelWithIndex)) # Modifying indexes - self.database.migrate('tests.sample_migrations', 19) - self.assertNotIn('INDEX index ', self.get_table_def(ModelWithIndex)) - self.assertIn('INDEX index2 ', self.get_table_def(ModelWithIndex)) - self.assertIn('INDEX another_index ', self.get_table_def(ModelWithIndex)) + self.database.migrate("tests.sample_migrations", 19) + self.assertNotIn("INDEX index ", self.get_table_def(ModelWithIndex)) + self.assertIn("INDEX index2 ", self.get_table_def(ModelWithIndex)) + self.assertIn("INDEX another_index ", self.get_table_def(ModelWithIndex)) # Several different models with the same table name, to simulate a table that changes over time + class Model1(Model): date = DateField() f1 = Int32Field() f2 = StringField() - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'mig' + return "mig" class Model2(Model): @@ -161,99 +197,99 @@ class Model2(Model): f3 = Float32Field() f2 = StringField() f4 = StringField() - f5 = ArrayField(UInt64Field()) # addition of an array field + f5 = ArrayField(UInt64Field()) # addition of an array field - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'mig' + return "mig" class Model3(Model): date = DateField() - f1 = Int64Field() # changed from Int32 - f3 = Float64Field() # changed from Float32 + f1 = Int64Field() # changed from Int32 + f3 = Float64Field() # changed from Float32 f4 = StringField() - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'mig' + return "mig" class EnumModel1(Model): date = DateField() - f1 = Enum8Field(Enum('SomeEnum1', 'dog cat cow')) + f1 = Enum8Field(Enum("SomeEnum1", "dog cat cow")) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'enum_mig' + return "enum_mig" class EnumModel2(Model): date = DateField() - f1 = Enum16Field(Enum('SomeEnum2', 'dog cat horse pig')) # changed type and values + f1 = Enum16Field(Enum("SomeEnum2", "dog cat horse pig")) # changed type and values - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'enum_mig' + return "enum_mig" class MaterializedModel(Model): date_time = DateTimeField() - date = DateField(materialized='toDate(date_time)') + date = DateField(materialized="toDate(date_time)") - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'materalized_date' + return "materalized_date" class MaterializedModel1(Model): date_time = DateTimeField() - date = DateField(materialized='toDate(date_time)') + date = DateField(materialized="toDate(date_time)") int_field = Int8Field() - int_field_plus_one = Int8Field(materialized='int_field + 1') + int_field_plus_one = Int8Field(materialized="int_field + 1") - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'materalized_date' + return "materalized_date" class AliasModel(Model): date = DateField() - date_alias = DateField(alias='date') + date_alias = DateField(alias="date") - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'alias_date' + return "alias_date" class AliasModel1(Model): date = DateField() - date_alias = DateField(alias='date') + date_alias = DateField(alias="date") int_field = Int8Field() - int_field_plus_one = Int8Field(alias='int_field + 1') + int_field_plus_one = Int8Field(alias="int_field + 1") - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'alias_date' + return "alias_date" class Model4(Model): @@ -262,11 +298,11 @@ class Model4(Model): f1 = Int32Field() f2 = StringField() - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'model4' + return "model4" class Model4Buffer(BufferModel, Model4): @@ -275,7 +311,7 @@ class Model4Buffer(BufferModel, Model4): @classmethod def table_name(cls): - return 'model4buffer' + return "model4buffer" class Model4_changed(Model): @@ -284,11 +320,11 @@ class Model4_changed(Model): f3 = DateTimeField() f2 = StringField() - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'model4' + return "model4" class Model4Buffer_changed(BufferModel, Model4_changed): @@ -297,20 +333,20 @@ class Model4Buffer_changed(BufferModel, Model4_changed): @classmethod def table_name(cls): - return 'model4buffer' + return "model4buffer" class Model4_compressed(Model): date = DateField() - f3 = DateTimeField(codec='Delta,ZSTD(10)') - f2 = StringField(codec='LZ4HC') + f3 = DateTimeField(codec="Delta,ZSTD(10)") + f2 = StringField(codec="LZ4HC") - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'model4' + return "model4" class Model2LowCardinality(Model): @@ -321,11 +357,11 @@ class Model2LowCardinality(Model): f4 = LowCardinalityField(NullableField(StringField())) f5 = ArrayField(LowCardinalityField(UInt64Field())) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'mig' + return "mig" class ModelWithConstraints(Model): @@ -334,14 +370,14 @@ class ModelWithConstraints(Model): f1 = Int32Field() f2 = StringField() - constraint = Constraint(f2.isIn(['a', 'b', 'c'])) # check reserved keyword as constraint name + constraint = Constraint(f2.isIn(["a", "b", "c"])) # check reserved keyword as constraint name f1_constraint = Constraint(f1 > 100) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'modelwithconstraints' + return "modelwithconstraints" class ModelWithConstraints2(Model): @@ -350,14 +386,14 @@ class ModelWithConstraints2(Model): f1 = Int32Field() f2 = StringField() - constraint = Constraint(f2.isIn(['a', 'b', 'c'])) + constraint = Constraint(f2.isIn(["a", "b", "c"])) f1_constraint_new = Constraint(f1 < 100) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'modelwithconstraints' + return "modelwithconstraints" class ModelWithIndex(Model): @@ -369,11 +405,11 @@ class ModelWithIndex(Model): index = Index(f1, type=Index.minmax(), granularity=1) another_index = Index(f2, type=Index.set(0), granularity=1) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'modelwithindex' + return "modelwithindex" class ModelWithIndex2(Model): @@ -385,9 +421,8 @@ class ModelWithIndex2(Model): index2 = Index(f1, type=Index.bloom_filter(), granularity=2) another_index = Index(f2, type=Index.set(0), granularity=1) - engine = MergeTree('date', ('date',)) + engine = MergeTree("date", ("date",)) @classmethod def table_name(cls): - return 'modelwithindex' - + return "modelwithindex" diff --git a/tests/test_models.py b/tests/test_models.py index 10d8b77..fd28034 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -9,13 +9,12 @@ class ModelTestCase(unittest.TestCase): - def test_defaults(self): # Check that all fields have their explicit or implicit defaults instance = SimpleModel() self.assertEqual(instance.date_field, datetime.date(1970, 1, 1)) self.assertEqual(instance.datetime_field, datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)) - self.assertEqual(instance.str_field, 'dozo') + self.assertEqual(instance.str_field, "dozo") self.assertEqual(instance.int_field, 17) self.assertEqual(instance.float_field, 0) self.assertEqual(instance.default_func, NO_VALUE) @@ -25,9 +24,9 @@ def test_assignment(self): kwargs = dict( date_field=datetime.date(1973, 12, 6), datetime_field=datetime.datetime(2000, 5, 24, 10, 22, tzinfo=pytz.utc), - str_field='aloha', + str_field="aloha", int_field=-50, - float_field=3.14 + float_field=3.14, ) instance = SimpleModel(**kwargs) for name, value in kwargs.items(): @@ -36,12 +35,12 @@ def test_assignment(self): def test_assignment_error(self): # Check non-existing field during construction with self.assertRaises(AttributeError): - instance = SimpleModel(int_field=7450, pineapple='tasty') + instance = SimpleModel(int_field=7450, pineapple="tasty") # Check invalid field values during construction with self.assertRaises(ValueError): - instance = SimpleModel(int_field='nope') + instance = SimpleModel(int_field="nope") with self.assertRaises(ValueError): - instance = SimpleModel(date_field='nope') + instance = SimpleModel(date_field="nope") # Check invalid field values during assignment instance = SimpleModel() with self.assertRaises(ValueError): @@ -49,38 +48,43 @@ def test_assignment_error(self): def test_string_conversion(self): # Check field conversion from string during construction - instance = SimpleModel(date_field='1973-12-06', int_field='100', float_field='7') + instance = SimpleModel(date_field="1973-12-06", int_field="100", float_field="7") self.assertEqual(instance.date_field, datetime.date(1973, 12, 6)) self.assertEqual(instance.int_field, 100) self.assertEqual(instance.float_field, 7) # Check field conversion from string during assignment - instance.int_field = '99' + instance.int_field = "99" self.assertEqual(instance.int_field, 99) def test_to_dict(self): - instance = SimpleModel(date_field='1973-12-06', int_field='100', float_field='7') - self.assertDictEqual(instance.to_dict(), { - "date_field": datetime.date(1973, 12, 6), - "int_field": 100, - "float_field": 7.0, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), - "alias_field": NO_VALUE, - "str_field": "dozo", - "default_func": NO_VALUE - }) - self.assertDictEqual(instance.to_dict(include_readonly=False), { - "date_field": datetime.date(1973, 12, 6), - "int_field": 100, - "float_field": 7.0, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), - "str_field": "dozo", - "default_func": NO_VALUE - }) + instance = SimpleModel(date_field="1973-12-06", int_field="100", float_field="7") + self.assertDictEqual( + instance.to_dict(), + { + "date_field": datetime.date(1973, 12, 6), + "int_field": 100, + "float_field": 7.0, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + "alias_field": NO_VALUE, + "str_field": "dozo", + "default_func": NO_VALUE, + }, + ) self.assertDictEqual( - instance.to_dict(include_readonly=False, field_names=('int_field', 'alias_field', 'datetime_field')), { + instance.to_dict(include_readonly=False), + { + "date_field": datetime.date(1973, 12, 6), "int_field": 100, - "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc) - }) + "float_field": 7.0, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + "str_field": "dozo", + "default_func": NO_VALUE, + }, + ) + self.assertDictEqual( + instance.to_dict(include_readonly=False, field_names=("int_field", "alias_field", "datetime_field")), + {"int_field": 100, "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)}, + ) def test_field_name_in_error_message_for_invalid_value_in_constructor(self): bad_value = 1 @@ -88,19 +92,17 @@ def test_field_name_in_error_message_for_invalid_value_in_constructor(self): SimpleModel(str_field=bad_value) self.assertEqual( - "Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)), - str(cm.exception) + "Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)), str(cm.exception) ) def test_field_name_in_error_message_for_invalid_value_in_assignment(self): instance = SimpleModel() - bad_value = 'foo' + bad_value = "foo" with self.assertRaises(ValueError) as cm: instance.float_field = bad_value self.assertEqual( - "Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)), - str(cm.exception) + "Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)), str(cm.exception) ) @@ -108,10 +110,10 @@ class SimpleModel(Model): date_field = DateField() datetime_field = DateTimeField() - str_field = StringField(default='dozo') + str_field = StringField(default="dozo") int_field = Int32Field(default=17) float_field = Float32Field() - alias_field = Float32Field(alias='float_field') + alias_field = Float32Field(alias="float_field") default_func = Float32Field(default=F.sqrt(float_field) + 17) - engine = MergeTree('date_field', ('int_field', 'date_field')) + engine = MergeTree("date_field", ("int_field", "date_field")) diff --git a/tests/test_mutations.py b/tests/test_mutations.py index ec291b9..89b0146 100644 --- a/tests/test_mutations.py +++ b/tests/test_mutations.py @@ -5,15 +5,14 @@ class MutationsTestCase(TestCaseWithData): - def setUp(self): super().setUp() if self.database.server_version < (18,): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") self._insert_all() def _wait_for_mutations(self): - sql = 'SELECT * FROM system.mutations WHERE is_done = 0' + sql = "SELECT * FROM system.mutations WHERE is_done = 0" while list(self.database.raw(sql)): sleep(0.25) @@ -23,7 +22,7 @@ def test_delete_all(self): self.assertFalse(Person.objects_in(self.database)) def test_delete_with_where_cond(self): - cond = Person.first_name == 'Cassady' + cond = Person.first_name == "Cassady" self.assertTrue(Person.objects_in(self.database).filter(cond)) Person.objects_in(self.database).filter(cond).delete() self._wait_for_mutations() @@ -41,11 +40,12 @@ def test_delete_with_prewhere_cond(self): def test_update_all(self): Person.objects_in(self.database).update(height=0) self._wait_for_mutations() - for p in Person.objects_in(self.database): print(p.height) + for p in Person.objects_in(self.database): + print(p.height) self.assertFalse(Person.objects_in(self.database).exclude(height=0)) def test_update_with_where_cond(self): - cond = Person.first_name == 'Cassady' + cond = Person.first_name == "Cassady" Person.objects_in(self.database).filter(cond).update(height=0) self._wait_for_mutations() self.assertFalse(Person.objects_in(self.database).filter(cond).exclude(height=0)) @@ -71,9 +71,9 @@ def test_invalid_state_for_mutations(self): base_query = Person.objects_in(self.database) queries = [ base_query[0:1], - base_query.limit_by(5, 'first_name'), + base_query.limit_by(5, "first_name"), base_query.distinct(), - base_query.aggregate('first_name', count=F.count()) + base_query.aggregate("first_name", count=F.count()), ] for query in queries: print(query) diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py index a0357ac..3c33e21 100644 --- a/tests/test_nullable_fields.py +++ b/tests/test_nullable_fields.py @@ -11,9 +11,8 @@ class NullableFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(ModelWithNullable) def tearDown(self): @@ -23,18 +22,20 @@ def test_nullable_datetime_field(self): f = NullableField(DateTimeField()) epoch = datetime(1970, 1, 1, tzinfo=pytz.utc) # Valid values - for value in (date(1970, 1, 1), - datetime(1970, 1, 1), - epoch, - epoch.astimezone(pytz.timezone('US/Eastern')), - epoch.astimezone(pytz.timezone('Asia/Jerusalem')), - '1970-01-01 00:00:00', - '1970-01-17 00:00:17', - '0000-00-00 00:00:00', - 0, - '\\N'): + for value in ( + date(1970, 1, 1), + datetime(1970, 1, 1), + epoch, + epoch.astimezone(pytz.timezone("US/Eastern")), + epoch.astimezone(pytz.timezone("Asia/Jerusalem")), + "1970-01-01 00:00:00", + "1970-01-17 00:00:17", + "0000-00-00 00:00:00", + 0, + "\\N", + ): dt = f.to_python(value, pytz.utc) - if value == '\\N': + if value == "\\N": self.assertIsNone(dt) else: self.assertTrue(dt.tzinfo) @@ -42,32 +43,32 @@ def test_nullable_datetime_field(self): dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) self.assertEqual(dt, dt2) # Invalid values - for value in ('nope', '21/7/1999', 0.5): + for value in ("nope", "21/7/1999", 0.5): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) def test_nullable_uint8_field(self): f = NullableField(UInt8Field()) # Valid values - for value in (17, '17', 17.0, '\\N'): + for value in (17, "17", 17.0, "\\N"): python_value = f.to_python(value, pytz.utc) - if value == '\\N': + if value == "\\N": self.assertIsNone(python_value) self.assertEqual(value, f.to_db_string(python_value)) else: self.assertEqual(python_value, 17) # Invalid values - for value in ('nope', date.today()): + for value in ("nope", date.today()): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) def test_nullable_string_field(self): f = NullableField(StringField()) # Valid values - for value in ('\\\\N', 'N', 'some text', '\\N'): + for value in ("\\\\N", "N", "some text", "\\N"): python_value = f.to_python(value, pytz.utc) - if value == '\\N': + if value == "\\N": self.assertIsNone(python_value) self.assertEqual(value, f.to_db_string(python_value)) else: @@ -91,12 +92,16 @@ def test_isinstance(self): def _insert_sample_data(self): dt = date(1970, 1, 1) - self.database.insert([ - ModelWithNullable(date_field='2016-08-30', null_str='', null_int=42, null_date=dt), - ModelWithNullable(date_field='2016-08-30', null_str='nothing', null_int=None, null_date=None), - ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=42, null_date=dt), - ModelWithNullable(date_field='2016-08-31', null_str=None, null_int=None, null_date=None, null_default=None) - ]) + self.database.insert( + [ + ModelWithNullable(date_field="2016-08-30", null_str="", null_int=42, null_date=dt), + ModelWithNullable(date_field="2016-08-30", null_str="nothing", null_int=None, null_date=None), + ModelWithNullable(date_field="2016-08-31", null_str=None, null_int=42, null_date=dt), + ModelWithNullable( + date_field="2016-08-31", null_str=None, null_int=None, null_date=None, null_default=None + ), + ] + ) def _assert_sample_data(self, results): for r in results: @@ -110,7 +115,7 @@ def _assert_sample_data(self, results): self.assertEqual(results[0].null_materialized, 420) self.assertEqual(results[0].null_date, dt) self.assertIsNone(results[1].null_date) - self.assertEqual(results[1].null_str, 'nothing') + self.assertEqual(results[1].null_str, "nothing") self.assertIsNone(results[1].null_date) self.assertIsNone(results[2].null_str) self.assertEqual(results[2].null_date, dt) @@ -128,14 +133,14 @@ def _assert_sample_data(self, results): def test_insert_and_select(self): self._insert_sample_data() fields = comma_join(ModelWithNullable.fields().keys()) - query = 'SELECT %s from $table ORDER BY date_field' % fields + query = "SELECT %s from $table ORDER BY date_field" % fields results = list(self.database.select(query, ModelWithNullable)) self._assert_sample_data(results) def test_ad_hoc_model(self): self._insert_sample_data() fields = comma_join(ModelWithNullable.fields().keys()) - query = 'SELECT %s from $db.modelwithnullable ORDER BY date_field' % fields + query = "SELECT %s from $db.modelwithnullable ORDER BY date_field" % fields results = list(self.database.select(query)) self._assert_sample_data(results) @@ -143,11 +148,11 @@ def test_ad_hoc_model(self): class ModelWithNullable(Model): date_field = DateField() - null_str = NullableField(StringField(), extra_null_values={''}) + null_str = NullableField(StringField(), extra_null_values={""}) null_int = NullableField(Int32Field()) null_date = NullableField(DateField()) null_default = NullableField(Int32Field(), default=7) - null_alias = NullableField(Int32Field(), alias='null_int/2') - null_materialized = NullableField(Int32Field(), alias='null_int*10') + null_alias = NullableField(Int32Field(), alias="null_int/2") + null_materialized = NullableField(Int32Field(), alias="null_int*10") - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 6f7116c..719541e 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -9,12 +9,11 @@ from decimal import Decimal from logging import getLogger -logger = getLogger('tests') +logger = getLogger("tests") class QuerySetTestCase(TestCaseWithData): - def setUp(self): super(QuerySetTestCase, self).setUp() self.database.insert(self._sample_data()) @@ -24,7 +23,7 @@ def _test_qs(self, qs, expected_count): count = 0 for instance in qs: count += 1 - logger.info('\t[%d]\t%s' % (count, instance.to_dict())) + logger.info("\t[%d]\t%s" % (count, instance.to_dict())) self.assertEqual(count, expected_count) self.assertEqual(qs.count(), expected_count) @@ -32,8 +31,8 @@ def test_prewhere(self): # We can't distinguish prewhere and where results, it affects performance only. # So let's control prewhere acts like where does qs = Person.objects_in(self.database) - self.assertTrue(qs.filter(first_name='Connor', prewhere=True)) - self.assertFalse(qs.filter(first_name='Willy', prewhere=True)) + self.assertTrue(qs.filter(first_name="Connor", prewhere=True)) + self.assertFalse(qs.filter(first_name="Willy", prewhere=True)) def test_no_filtering(self): qs = Person.objects_in(self.database) @@ -41,8 +40,8 @@ def test_no_filtering(self): def test_truthiness(self): qs = Person.objects_in(self.database) - self.assertTrue(qs.filter(first_name='Connor')) - self.assertFalse(qs.filter(first_name='Willy')) + self.assertTrue(qs.filter(first_name="Connor")) + self.assertFalse(qs.filter(first_name="Willy")) def test_filter_null_value(self): qs = Person.objects_in(self.database) @@ -53,81 +52,96 @@ def test_filter_null_value(self): def test_filter_string_field(self): qs = Person.objects_in(self.database) - self._test_qs(qs.filter(first_name='Ciaran'), 2) - self._test_qs(qs.filter(first_name='ciaran'), 0) # case sensitive - self._test_qs(qs.filter(first_name__iexact='ciaran'), 2) # case insensitive - self._test_qs(qs.filter(first_name__gt='Whilemina'), 4) - self._test_qs(qs.filter(first_name__gte='Whilemina'), 5) - self._test_qs(qs.filter(first_name__lt='Adam'), 1) - self._test_qs(qs.filter(first_name__lte='Adam'), 2) - self._test_qs(qs.filter(first_name__in=('Connor', 'Courtney')), 3) # in tuple - self._test_qs(qs.filter(first_name__in=['Connor', 'Courtney']), 3) # in list - self._test_qs(qs.filter(first_name__in="'Connor', 'Courtney'"), 3) # in string + self._test_qs(qs.filter(first_name="Ciaran"), 2) + self._test_qs(qs.filter(first_name="ciaran"), 0) # case sensitive + self._test_qs(qs.filter(first_name__iexact="ciaran"), 2) # case insensitive + self._test_qs(qs.filter(first_name__gt="Whilemina"), 4) + self._test_qs(qs.filter(first_name__gte="Whilemina"), 5) + self._test_qs(qs.filter(first_name__lt="Adam"), 1) + self._test_qs(qs.filter(first_name__lte="Adam"), 2) + self._test_qs(qs.filter(first_name__in=("Connor", "Courtney")), 3) # in tuple + self._test_qs(qs.filter(first_name__in=["Connor", "Courtney"]), 3) # in list + self._test_qs(qs.filter(first_name__in="'Connor', 'Courtney'"), 3) # in string self._test_qs(qs.filter(first_name__not_in="'Connor', 'Courtney'"), 97) - self._test_qs(qs.filter(first_name__contains='sh'), 3) # case sensitive - self._test_qs(qs.filter(first_name__icontains='sh'), 6) # case insensitive - self._test_qs(qs.filter(first_name__startswith='le'), 0) # case sensitive - self._test_qs(qs.filter(first_name__istartswith='Le'), 2) # case insensitive - self._test_qs(qs.filter(first_name__istartswith=''), 100) # empty prefix - self._test_qs(qs.filter(first_name__endswith='IA'), 0) # case sensitive - self._test_qs(qs.filter(first_name__iendswith='ia'), 3) # case insensitive - self._test_qs(qs.filter(first_name__iendswith=''), 100) # empty suffix + self._test_qs(qs.filter(first_name__contains="sh"), 3) # case sensitive + self._test_qs(qs.filter(first_name__icontains="sh"), 6) # case insensitive + self._test_qs(qs.filter(first_name__startswith="le"), 0) # case sensitive + self._test_qs(qs.filter(first_name__istartswith="Le"), 2) # case insensitive + self._test_qs(qs.filter(first_name__istartswith=""), 100) # empty prefix + self._test_qs(qs.filter(first_name__endswith="IA"), 0) # case sensitive + self._test_qs(qs.filter(first_name__iendswith="ia"), 3) # case insensitive + self._test_qs(qs.filter(first_name__iendswith=""), 100) # empty suffix def test_filter_with_q_objects(self): qs = Person.objects_in(self.database) - self._test_qs(qs.filter(Q(first_name='Ciaran')), 2) - self._test_qs(qs.filter(Q(first_name='Ciaran') | Q(first_name='Chelsea')), 3) - self._test_qs(qs.filter(Q(first_name__in=['Warren', 'Whilemina', 'Whitney']) & Q(height__gte=1.7)), 3) - self._test_qs(qs.filter((Q(first_name__in=['Warren', 'Whilemina', 'Whitney']) & Q(height__gte=1.7) | - (Q(first_name__in=['Victoria', 'Victor', 'Venus']) & Q(height__lt=1.7)))), 4) - self._test_qs(qs.filter(Q(first_name='Elton') & ~Q(last_name='Smith')), 1) + self._test_qs(qs.filter(Q(first_name="Ciaran")), 2) + self._test_qs(qs.filter(Q(first_name="Ciaran") | Q(first_name="Chelsea")), 3) + self._test_qs(qs.filter(Q(first_name__in=["Warren", "Whilemina", "Whitney"]) & Q(height__gte=1.7)), 3) + self._test_qs( + qs.filter( + ( + Q(first_name__in=["Warren", "Whilemina", "Whitney"]) & Q(height__gte=1.7) + | (Q(first_name__in=["Victoria", "Victor", "Venus"]) & Q(height__lt=1.7)) + ) + ), + 4, + ) + self._test_qs(qs.filter(Q(first_name="Elton") & ~Q(last_name="Smith")), 1) # Check operator precendence - self._test_qs(qs.filter(first_name='Cassady').filter(Q(last_name='Knapp') | Q(last_name='Rogers') | Q(last_name='Gregory')), 2) - self._test_qs(qs.filter(Q(first_name='Cassady') & Q(last_name='Knapp') | Q(first_name='Beatrice') & Q(last_name='Gregory')), 2) - self._test_qs(qs.filter(Q(first_name='Courtney') | Q(first_name='Cassady') & Q(last_name='Knapp')), 3) + self._test_qs( + qs.filter(first_name="Cassady").filter( + Q(last_name="Knapp") | Q(last_name="Rogers") | Q(last_name="Gregory") + ), + 2, + ) + self._test_qs( + qs.filter( + Q(first_name="Cassady") & Q(last_name="Knapp") | Q(first_name="Beatrice") & Q(last_name="Gregory") + ), + 2, + ) + self._test_qs(qs.filter(Q(first_name="Courtney") | Q(first_name="Cassady") & Q(last_name="Knapp")), 3) def test_filter_unicode_string(self): - self.database.insert([ - Person(first_name=u'דונלד', last_name=u'דאק') - ]) + self.database.insert([Person(first_name=u"דונלד", last_name=u"דאק")]) qs = Person.objects_in(self.database) - self._test_qs(qs.filter(first_name=u'דונלד'), 1) + self._test_qs(qs.filter(first_name=u"דונלד"), 1) def test_filter_float_field(self): qs = Person.objects_in(self.database) self._test_qs(qs.filter(height__gt=2), 0) self._test_qs(qs.filter(height__lt=1.61), 4) - self._test_qs(qs.filter(height__lt='1.61'), 4) - self._test_qs(qs.exclude(height__lt='1.61'), 96) + self._test_qs(qs.filter(height__lt="1.61"), 4) + self._test_qs(qs.exclude(height__lt="1.61"), 96) self._test_qs(qs.filter(height__gt=0), 100) self._test_qs(qs.exclude(height__gt=0), 0) def test_filter_date_field(self): qs = Person.objects_in(self.database) - self._test_qs(qs.filter(birthday='1970-12-02'), 1) - self._test_qs(qs.filter(birthday__eq='1970-12-02'), 1) - self._test_qs(qs.filter(birthday__ne='1970-12-02'), 99) + self._test_qs(qs.filter(birthday="1970-12-02"), 1) + self._test_qs(qs.filter(birthday__eq="1970-12-02"), 1) + self._test_qs(qs.filter(birthday__ne="1970-12-02"), 99) self._test_qs(qs.filter(birthday=date(1970, 12, 2)), 1) self._test_qs(qs.filter(birthday__lte=date(1970, 12, 2)), 3) def test_mutiple_filter(self): qs = Person.objects_in(self.database) # Single filter call with multiple conditions is ANDed - self._test_qs(qs.filter(first_name='Ciaran', last_name='Carver'), 1) + self._test_qs(qs.filter(first_name="Ciaran", last_name="Carver"), 1) # Separate filter calls are also ANDed - self._test_qs(qs.filter(first_name='Ciaran').filter(last_name='Carver'), 1) - self._test_qs(qs.filter(birthday='1970-12-02').filter(birthday='1986-01-07'), 0) + self._test_qs(qs.filter(first_name="Ciaran").filter(last_name="Carver"), 1) + self._test_qs(qs.filter(birthday="1970-12-02").filter(birthday="1986-01-07"), 0) def test_multiple_exclude(self): qs = Person.objects_in(self.database) # Single exclude call with multiple conditions is ANDed - self._test_qs(qs.exclude(first_name='Ciaran', last_name='Carver'), 99) + self._test_qs(qs.exclude(first_name="Ciaran", last_name="Carver"), 99) # Separate exclude calls are ORed - self._test_qs(qs.exclude(first_name='Ciaran').exclude(last_name='Carver'), 98) - self._test_qs(qs.exclude(birthday='1970-12-02').exclude(birthday='1986-01-07'), 98) + self._test_qs(qs.exclude(first_name="Ciaran").exclude(last_name="Carver"), 98) + self._test_qs(qs.exclude(birthday="1970-12-02").exclude(birthday="1986-01-07"), 98) def test_only(self): - qs = Person.objects_in(self.database).only('first_name', 'last_name') + qs = Person.objects_in(self.database).only("first_name", "last_name") for person in qs: self.assertTrue(person.first_name) self.assertTrue(person.last_name) @@ -136,46 +150,50 @@ def test_only(self): def test_order_by(self): qs = Person.objects_in(self.database) - self.assertFalse('ORDER BY' in qs.as_sql()) + self.assertFalse("ORDER BY" in qs.as_sql()) self.assertFalse(qs.order_by_as_sql()) - person = list(qs.order_by('first_name', 'last_name'))[0] - self.assertEqual(person.first_name, 'Abdul') - person = list(qs.order_by('-first_name', '-last_name'))[0] - self.assertEqual(person.first_name, 'Yolanda') - person = list(qs.order_by('height'))[0] + person = list(qs.order_by("first_name", "last_name"))[0] + self.assertEqual(person.first_name, "Abdul") + person = list(qs.order_by("-first_name", "-last_name"))[0] + self.assertEqual(person.first_name, "Yolanda") + person = list(qs.order_by("height"))[0] self.assertEqual(person.height, 1.59) - person = list(qs.order_by('-height'))[0] + person = list(qs.order_by("-height"))[0] self.assertEqual(person.height, 1.8) def test_in_subquery(self): qs = Person.objects_in(self.database) - self._test_qs(qs.filter(height__in='SELECT max(height) FROM $table'), 2) - self._test_qs(qs.filter(first_name__in=qs.only('last_name')), 2) - self._test_qs(qs.filter(first_name__not_in=qs.only('last_name')), 98) + self._test_qs(qs.filter(height__in="SELECT max(height) FROM $table"), 2) + self._test_qs(qs.filter(first_name__in=qs.only("last_name")), 2) + self._test_qs(qs.filter(first_name__not_in=qs.only("last_name")), 98) def _insert_sample_model(self): self.database.create_table(SampleModel) now = datetime.now() - self.database.insert([ - SampleModel(timestamp=now, num=1, color=Color.red), - SampleModel(timestamp=now, num=2, color=Color.red), - SampleModel(timestamp=now, num=3, color=Color.blue), - SampleModel(timestamp=now, num=4, color=Color.white), - ]) + self.database.insert( + [ + SampleModel(timestamp=now, num=1, color=Color.red), + SampleModel(timestamp=now, num=2, color=Color.red), + SampleModel(timestamp=now, num=3, color=Color.blue), + SampleModel(timestamp=now, num=4, color=Color.white), + ] + ) def _insert_sample_collapsing_model(self): self.database.create_table(SampleCollapsingModel) now = datetime.now() - self.database.insert([ - SampleCollapsingModel(timestamp=now, num=1, color=Color.red), - SampleCollapsingModel(timestamp=now, num=2, color=Color.red), - SampleCollapsingModel(timestamp=now, num=2, color=Color.red, sign=-1), - SampleCollapsingModel(timestamp=now, num=2, color=Color.green), - SampleCollapsingModel(timestamp=now, num=3, color=Color.white), - SampleCollapsingModel(timestamp=now, num=4, color=Color.white, sign=1), - SampleCollapsingModel(timestamp=now, num=4, color=Color.white, sign=-1), - SampleCollapsingModel(timestamp=now, num=4, color=Color.blue, sign=1), - ]) + self.database.insert( + [ + SampleCollapsingModel(timestamp=now, num=1, color=Color.red), + SampleCollapsingModel(timestamp=now, num=2, color=Color.red), + SampleCollapsingModel(timestamp=now, num=2, color=Color.red, sign=-1), + SampleCollapsingModel(timestamp=now, num=2, color=Color.green), + SampleCollapsingModel(timestamp=now, num=3, color=Color.white), + SampleCollapsingModel(timestamp=now, num=4, color=Color.white, sign=1), + SampleCollapsingModel(timestamp=now, num=4, color=Color.white, sign=-1), + SampleCollapsingModel(timestamp=now, num=4, color=Color.blue, sign=1), + ] + ) def test_filter_enum_field(self): self._insert_sample_model() @@ -184,7 +202,7 @@ def test_filter_enum_field(self): self._test_qs(qs.exclude(color=Color.white), 3) # Different ways to specify blue self._test_qs(qs.filter(color__gt=Color.blue), 1) - self._test_qs(qs.filter(color__gt='blue'), 1) + self._test_qs(qs.filter(color__gt="blue"), 1) self._test_qs(qs.filter(color__gt=2), 1) def test_filter_int_field(self): @@ -199,7 +217,7 @@ def test_filter_int_field(self): self._test_qs(qs.filter(num__in=range(1, 4)), 3) def test_slicing(self): - db = Database('system') + db = Database("system") numbers = list(range(100)) qs = Numbers.objects_in(db) self.assertEqual(qs[0].number, numbers[0]) @@ -211,7 +229,7 @@ def test_slicing(self): self.assertEqual([row.number for row in qs[10:10]], numbers[10:10]) def test_invalid_slicing(self): - db = Database('system') + db = Database("system") qs = Numbers.objects_in(db) with self.assertRaises(AssertionError): qs[3:10:2] @@ -223,7 +241,7 @@ def test_invalid_slicing(self): qs[50:1] def test_pagination(self): - qs = Person.objects_in(self.database).order_by('first_name', 'last_name') + qs = Person.objects_in(self.database).order_by("first_name", "last_name") # Try different page sizes for page_size in (1, 2, 7, 10, 30, 100, 150): # Iterate over pages and collect all instances @@ -241,31 +259,30 @@ def test_pagination(self): self.assertEqual(len(instances), len(data)) def test_pagination_last_page(self): - qs = Person.objects_in(self.database).order_by('first_name', 'last_name') + qs = Person.objects_in(self.database).order_by("first_name", "last_name") # Try different page sizes for page_size in (1, 2, 7, 10, 30, 100, 150): # Ask for the last page in two different ways and verify equality page_a = qs.paginate(-1, page_size) page_b = qs.paginate(page_a.pages_total, page_size) self.assertEqual(page_a[1:], page_b[1:]) - self.assertEqual([obj.to_tsv() for obj in page_a.objects], - [obj.to_tsv() for obj in page_b.objects]) + self.assertEqual([obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects]) def test_pagination_invalid_page(self): - qs = Person.objects_in(self.database).order_by('first_name', 'last_name') + qs = Person.objects_in(self.database).order_by("first_name", "last_name") for page_num in (0, -2, -100): with self.assertRaises(ValueError): qs.paginate(page_num, 100) def test_pagination_with_conditions(self): - qs = Person.objects_in(self.database).order_by('first_name', 'last_name').filter(first_name__lt='Ava') + qs = Person.objects_in(self.database).order_by("first_name", "last_name").filter(first_name__lt="Ava") page = qs.paginate(1, 100) self.assertEqual(page.number_of_objects, 10) def test_distinct(self): qs = Person.objects_in(self.database).distinct() self._test_qs(qs, 100) - self._test_qs(qs.only('first_name'), 94) + self._test_qs(qs.only("first_name"), 94) def test_materialized_field(self): self._insert_sample_model() @@ -291,31 +308,31 @@ def test_final(self): Person.objects_in(self.database).final() self._insert_sample_collapsing_model() - res = list(SampleCollapsingModel.objects_in(self.database).final().order_by('num')) + res = list(SampleCollapsingModel.objects_in(self.database).final().order_by("num")) self.assertEqual(4, len(res)) for item, exp_color in zip(res, (Color.red, Color.green, Color.white, Color.blue)): self.assertEqual(exp_color, item.color) def test_mixed_filter(self): qs = Person.objects_in(self.database) - qs = qs.filter(Q(first_name='a'), F('greater', Person.height, 1.7), last_name='b') - self.assertEqual(qs.conditions_as_sql(), - "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')") + qs = qs.filter(Q(first_name="a"), F("greater", Person.height, 1.7), last_name="b") + self.assertEqual( + qs.conditions_as_sql(), "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')" + ) def test_invalid_filter(self): qs = Person.objects_in(self.database) with self.assertRaises(TypeError): - qs.filter('foo') + qs.filter("foo") class AggregateTestCase(TestCaseWithData): - def setUp(self): super(AggregateTestCase, self).setUp() self.database.insert(self._sample_data()) def test_aggregate_no_grouping(self): - qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()') + qs = Person.objects_in(self.database).aggregate(average_height="avg(height)", count="count()") print(qs.as_sql()) self.assertEqual(qs.count(), 1) for row in qs: @@ -331,14 +348,22 @@ def test_aggregate_no_grouping(self): def test_aggregate_with_filter(self): # When filter comes before aggregate - qs = Person.objects_in(self.database).filter(first_name='Warren').aggregate(average_height='avg(height)', count='count()') + qs = ( + Person.objects_in(self.database) + .filter(first_name="Warren") + .aggregate(average_height="avg(height)", count="count()") + ) print(qs.as_sql()) self.assertEqual(qs.count(), 1) for row in qs: self.assertAlmostEqual(row.average_height, 1.675, places=4) self.assertEqual(row.count, 2) # When filter comes after aggregate - qs = Person.objects_in(self.database).aggregate(average_height='avg(height)', count='count()').filter(first_name='Warren') + qs = ( + Person.objects_in(self.database) + .aggregate(average_height="avg(height)", count="count()") + .filter(first_name="Warren") + ) print(qs.as_sql()) self.assertEqual(qs.count(), 1) for row in qs: @@ -347,14 +372,22 @@ def test_aggregate_with_filter(self): def test_aggregate_with_filter__funcs(self): # When filter comes before aggregate - qs = Person.objects_in(self.database).filter(Person.first_name=='Warren').aggregate(average_height=F.avg(Person.height), count=F.count()) + qs = ( + Person.objects_in(self.database) + .filter(Person.first_name == "Warren") + .aggregate(average_height=F.avg(Person.height), count=F.count()) + ) print(qs.as_sql()) self.assertEqual(qs.count(), 1) for row in qs: self.assertAlmostEqual(row.average_height, 1.675, places=4) self.assertEqual(row.count, 2) # When filter comes after aggregate - qs = Person.objects_in(self.database).aggregate(average_height=F.avg(Person.height), count=F.count()).filter(Person.first_name=='Warren') + qs = ( + Person.objects_in(self.database) + .aggregate(average_height=F.avg(Person.height), count=F.count()) + .filter(Person.first_name == "Warren") + ) print(qs.as_sql()) self.assertEqual(qs.count(), 1) for row in qs: @@ -362,7 +395,7 @@ def test_aggregate_with_filter__funcs(self): self.assertEqual(row.count, 2) def test_aggregate_with_implicit_grouping(self): - qs = Person.objects_in(self.database).aggregate('first_name', average_height='avg(height)', count='count()') + qs = Person.objects_in(self.database).aggregate("first_name", average_height="avg(height)", count="count()") print(qs.as_sql()) self.assertEqual(qs.count(), 94) total = 0 @@ -373,7 +406,11 @@ def test_aggregate_with_implicit_grouping(self): self.assertEqual(total, 100) def test_aggregate_with_explicit_grouping(self): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) print(qs.as_sql()) self.assertEqual(qs.count(), 7) total = 0 @@ -382,24 +419,40 @@ def test_aggregate_with_explicit_grouping(self): self.assertEqual(total, 100) def test_aggregate_with_order_by(self): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') - days = [row.weekday for row in qs.order_by('weekday')] + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) + days = [row.weekday for row in qs.order_by("weekday")] self.assertEqual(days, list(range(1, 8))) def test_aggregate_with_indexing(self): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) total = 0 for i in range(7): total += qs[i].count self.assertEqual(total, 100) def test_aggregate_with_slicing(self): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) total = sum(row.count for row in qs[:3]) + sum(row.count for row in qs[3:]) self.assertEqual(total, 100) def test_aggregate_with_pagination(self): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) total = 0 page_num = 1 while True: @@ -413,7 +466,9 @@ def test_aggregate_with_pagination(self): def test_aggregate_with_wrong_grouping(self): with self.assertRaises(AssertionError): - Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('first_name') + Person.objects_in(self.database).aggregate(weekday="toDayOfWeek(birthday)", count="count()").group_by( + "first_name" + ) def test_aggregate_with_no_calculated_fields(self): with self.assertRaises(AssertionError): @@ -422,31 +477,41 @@ def test_aggregate_with_no_calculated_fields(self): def test_aggregate_with_only(self): # Cannot put only() after aggregate() with self.assertRaises(NotImplementedError): - Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').only('weekday') + Person.objects_in(self.database).aggregate(weekday="toDayOfWeek(birthday)", count="count()").only("weekday") # When only() comes before aggregate(), it gets overridden - qs = Person.objects_in(self.database).only('last_name').aggregate(average_height='avg(height)', count='count()') - self.assertTrue('last_name' not in qs.as_sql()) + qs = Person.objects_in(self.database).only("last_name").aggregate(average_height="avg(height)", count="count()") + self.assertTrue("last_name" not in qs.as_sql()) def test_aggregate_on_aggregate(self): with self.assertRaises(NotImplementedError): - Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').aggregate(s='sum(height)') + Person.objects_in(self.database).aggregate(weekday="toDayOfWeek(birthday)", count="count()").aggregate( + s="sum(height)" + ) def test_filter_on_calculated_field(self): # This is currently not supported, so we expect it to fail with self.assertRaises(AttributeError): - qs = Person.objects_in(self.database).aggregate(weekday='toDayOfWeek(birthday)', count='count()').group_by('weekday') + qs = ( + Person.objects_in(self.database) + .aggregate(weekday="toDayOfWeek(birthday)", count="count()") + .group_by("weekday") + ) qs = qs.filter(weekday=1) self.assertEqual(qs.count(), 1) def test_aggregate_with_distinct(self): # In this case distinct has no effect - qs = Person.objects_in(self.database).aggregate(average_height='avg(height)').distinct() + qs = Person.objects_in(self.database).aggregate(average_height="avg(height)").distinct() print(qs.as_sql()) self.assertEqual(qs.count(), 1) def test_aggregate_with_totals(self): - qs = Person.objects_in(self.database).aggregate('first_name', count='count()').\ - with_totals().order_by('-count')[:5] + qs = ( + Person.objects_in(self.database) + .aggregate("first_name", count="count()") + .with_totals() + .order_by("-count")[:5] + ) print(qs.as_sql()) result = list(qs) self.assertEqual(len(result), 6) @@ -460,61 +525,68 @@ class Mdl(Model): the__number = Int32Field() the__next__number = Int32Field() engine = Memory() + qs = Mdl.objects_in(self.database).filter(the__number=1) - self.assertEqual(qs.conditions_as_sql(), 'the__number = 1') + self.assertEqual(qs.conditions_as_sql(), "the__number = 1") qs = Mdl.objects_in(self.database).filter(the__number__gt=1) - self.assertEqual(qs.conditions_as_sql(), 'the__number > 1') + self.assertEqual(qs.conditions_as_sql(), "the__number > 1") qs = Mdl.objects_in(self.database).filter(the__next__number=1) - self.assertEqual(qs.conditions_as_sql(), 'the__next__number = 1') + self.assertEqual(qs.conditions_as_sql(), "the__next__number = 1") qs = Mdl.objects_in(self.database).filter(the__next__number__gt=1) - self.assertEqual(qs.conditions_as_sql(), 'the__next__number > 1') + self.assertEqual(qs.conditions_as_sql(), "the__next__number > 1") def test_limit_by(self): if self.database.server_version < (19, 17): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") # Test without offset - qs = Person.objects_in(self.database).aggregate('first_name', 'last_name', 'height', n='count()').\ - order_by('first_name', '-height').limit_by(1, 'first_name') + qs = ( + Person.objects_in(self.database) + .aggregate("first_name", "last_name", "height", n="count()") + .order_by("first_name", "-height") + .limit_by(1, "first_name") + ) self.assertEqual(qs.count(), 94) - self.assertEqual(list(qs)[89].last_name, 'Bowen') + self.assertEqual(list(qs)[89].last_name, "Bowen") # Test with funcs and fields - qs = Person.objects_in(self.database).aggregate(Person.first_name, Person.last_name, Person.height, n=F.count()).\ - order_by(Person.first_name, '-height').limit_by(1, F.upper(Person.first_name)) + qs = ( + Person.objects_in(self.database) + .aggregate(Person.first_name, Person.last_name, Person.height, n=F.count()) + .order_by(Person.first_name, "-height") + .limit_by(1, F.upper(Person.first_name)) + ) self.assertEqual(qs.count(), 94) - self.assertEqual(list(qs)[89].last_name, 'Bowen') + self.assertEqual(list(qs)[89].last_name, "Bowen") # Test with limit and offset, also mixing LIMIT with LIMIT BY - qs = Person.objects_in(self.database).filter(height__gt=1.67).order_by('height', 'first_name') - limited_qs = qs.limit_by((0, 3), 'height') - self.assertEqual([p.first_name for p in limited_qs[:3]], ['Amanda', 'Buffy', 'Dora']) - limited_qs = qs.limit_by((3, 3), 'height') - self.assertEqual([p.first_name for p in limited_qs[:3]], ['Elton', 'Josiah', 'Macaulay']) - limited_qs = qs.limit_by((6, 3), 'height') - self.assertEqual([p.first_name for p in limited_qs[:3]], ['Norman', 'Octavius', 'Oliver']) + qs = Person.objects_in(self.database).filter(height__gt=1.67).order_by("height", "first_name") + limited_qs = qs.limit_by((0, 3), "height") + self.assertEqual([p.first_name for p in limited_qs[:3]], ["Amanda", "Buffy", "Dora"]) + limited_qs = qs.limit_by((3, 3), "height") + self.assertEqual([p.first_name for p in limited_qs[:3]], ["Elton", "Josiah", "Macaulay"]) + limited_qs = qs.limit_by((6, 3), "height") + self.assertEqual([p.first_name for p in limited_qs[:3]], ["Norman", "Octavius", "Oliver"]) -Color = Enum('Color', u'red blue green yellow brown white black') +Color = Enum("Color", u"red blue green yellow brown white black") class SampleModel(Model): timestamp = DateTimeField() - materialized_date = DateField(materialized='toDate(timestamp)') + materialized_date = DateField(materialized="toDate(timestamp)") num = Int32Field() color = Enum8Field(Color) - num_squared = Int32Field(alias='num*num') + num_squared = Int32Field(alias="num*num") - engine = MergeTree('materialized_date', ('materialized_date',)) + engine = MergeTree("materialized_date", ("materialized_date",)) class SampleCollapsingModel(SampleModel): sign = Int8Field(default=1) - engine = CollapsingMergeTree('materialized_date', ('num',), 'sign') + engine = CollapsingMergeTree("materialized_date", ("num",), "sign") class Numbers(Model): number = UInt64Field() - - diff --git a/tests/test_readonly.py b/tests/test_readonly.py index e136b9a..c085215 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -5,7 +5,6 @@ class ReadonlyTestCase(TestCaseWithData): - def _test_readonly_db(self, username): self._insert_and_check(self._sample_data(), len(data)) orig_database = self.database @@ -16,7 +15,7 @@ def _test_readonly_db(self, username): self._check_db_readonly_err(cm.exception) self.assertEqual(self.database.count(Person), 100) - list(self.database.select('SELECT * from $table', Person)) + list(self.database.select("SELECT * from $table", Person)) with self.assertRaises(ServerError) as cm: self.database.drop_table(Person) self._check_db_readonly_err(cm.exception, drop_table=True) @@ -25,9 +24,11 @@ def _test_readonly_db(self, username): self.database.drop_database() self._check_db_readonly_err(cm.exception, drop_table=True) except ServerError as e: - if e.code == 192 and e.message.startswith('Unknown user'): # ClickHouse version < 20.3 + if e.code == 192 and e.message.startswith("Unknown user"): # ClickHouse version < 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) - elif e.code == 516 and e.message.startswith('readonly: Authentication failed'): # ClickHouse version >= 20.3 + elif e.code == 516 and e.message.startswith( + "readonly: Authentication failed" + ): # ClickHouse version >= 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) else: raise @@ -38,20 +39,20 @@ def _check_db_readonly_err(self, exc, drop_table=None): self.assertEqual(exc.code, 164) print(exc.message) if self.database.server_version >= (20, 3): - self.assertTrue('Cannot execute query in readonly mode' in exc.message) + self.assertTrue("Cannot execute query in readonly mode" in exc.message) elif drop_table: - self.assertTrue(exc.message.startswith('Cannot drop table in readonly mode')) + self.assertTrue(exc.message.startswith("Cannot drop table in readonly mode")) else: - self.assertTrue(exc.message.startswith('Cannot insert into table in readonly mode')) + self.assertTrue(exc.message.startswith("Cannot insert into table in readonly mode")) def test_readonly_db_with_default_user(self): - self._test_readonly_db('default') + self._test_readonly_db("default") def test_readonly_db_with_readonly_user(self): - self._test_readonly_db('readonly') + self._test_readonly_db("readonly") def test_insert_readonly(self): - m = ReadOnlyModel(name='readonly') + m = ReadOnlyModel(name="readonly") self.database.create_table(ReadOnlyModel) with self.assertRaises(DatabaseException): self.database.insert([m]) @@ -64,8 +65,8 @@ def test_drop_readonly_table(self): def test_nonexisting_readonly_database(self): with self.assertRaises(DatabaseException) as cm: - db = Database('dummy', readonly=True) - self.assertEqual(str(cm.exception), 'Database does not exist, and cannot be created under readonly connection') + db = Database("dummy", readonly=True) + self.assertEqual(str(cm.exception), "Database does not exist, and cannot be created under readonly connection") class ReadOnlyModel(Model): @@ -73,4 +74,4 @@ class ReadOnlyModel(Model): name = StringField() date = DateField() - engine = MergeTree('date', ('name',)) + engine = MergeTree("date", ("name",)) diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py index 578ae9d..d4cff3d 100644 --- a/tests/test_server_errors.py +++ b/tests/test_server_errors.py @@ -4,28 +4,36 @@ class ServerErrorTest(unittest.TestCase): - def test_old_format(self): - code, msg = ServerError.get_error_code_msg("Code: 81, e.displayText() = DB::Exception: Database db_not_here doesn't exist, e.what() = DB::Exception (from [::1]:33458)") + code, msg = ServerError.get_error_code_msg( + "Code: 81, e.displayText() = DB::Exception: Database db_not_here doesn't exist, e.what() = DB::Exception (from [::1]:33458)" + ) self.assertEqual(code, 81) self.assertEqual(msg, "Database db_not_here doesn't exist") - code, msg = ServerError.get_error_code_msg("Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n") + code, msg = ServerError.get_error_code_msg( + "Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n" + ) self.assertEqual(code, 161) self.assertEqual(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1") - def test_new_format(self): - code, msg = ServerError.get_error_code_msg("Code: 164, e.displayText() = DB::Exception: Cannot drop table in readonly mode") + code, msg = ServerError.get_error_code_msg( + "Code: 164, e.displayText() = DB::Exception: Cannot drop table in readonly mode" + ) self.assertEqual(code, 164) self.assertEqual(msg, "Cannot drop table in readonly mode") - code, msg = ServerError.get_error_code_msg("Code: 48, e.displayText() = DB::Exception: Method write is not supported by storage Merge") + code, msg = ServerError.get_error_code_msg( + "Code: 48, e.displayText() = DB::Exception: Method write is not supported by storage Merge" + ) self.assertEqual(code, 48) self.assertEqual(msg, "Method write is not supported by storage Merge") - code, msg = ServerError.get_error_code_msg("Code: 60, e.displayText() = DB::Exception: Table default.zuzu doesn't exist.\n") + code, msg = ServerError.get_error_code_msg( + "Code: 60, e.displayText() = DB::Exception: Table default.zuzu doesn't exist.\n" + ) self.assertEqual(code, 60) self.assertEqual(msg, "Table default.zuzu doesn't exist.") diff --git a/tests/test_simple_fields.py b/tests/test_simple_fields.py index d87dcbc..46ebc04 100644 --- a/tests/test_simple_fields.py +++ b/tests/test_simple_fields.py @@ -9,11 +9,20 @@ class SimpleFieldsTest(unittest.TestCase): epoch = datetime(1970, 1, 1, tzinfo=pytz.utc) # Valid values dates = [ - date(1970, 1, 1), datetime(1970, 1, 1), epoch, - epoch.astimezone(pytz.timezone('US/Eastern')), epoch.astimezone(pytz.timezone('Asia/Jerusalem')), - '1970-01-01 00:00:00', '1970-01-17 00:00:17', '0000-00-00 00:00:00', 0, - '2017-07-26T08:31:05', '2017-07-26T08:31:05Z', '2017-07-26 08:31', - '2017-07-26T13:31:05+05', '2017-07-26 13:31:05+0500' + date(1970, 1, 1), + datetime(1970, 1, 1), + epoch, + epoch.astimezone(pytz.timezone("US/Eastern")), + epoch.astimezone(pytz.timezone("Asia/Jerusalem")), + "1970-01-01 00:00:00", + "1970-01-17 00:00:17", + "0000-00-00 00:00:00", + 0, + "2017-07-26T08:31:05", + "2017-07-26T08:31:05Z", + "2017-07-26 08:31", + "2017-07-26T13:31:05+05", + "2017-07-26 13:31:05+0500", ] def test_datetime_field(self): @@ -25,8 +34,7 @@ def test_datetime_field(self): dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) self.assertEqual(dt, dt2) # Invalid values - for value in ('nope', '21/7/1999', 0.5, - '2017-01 15:06:00', '2017-01-01X15:06:00', '2017-13-01T15:06:00'): + for value in ("nope", "21/7/1999", 0.5, "2017-01 15:06:00", "2017-01-01X15:06:00", "2017-13-01T15:06:00"): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) @@ -35,10 +43,16 @@ def test_datetime64_field(self): # Valid values for value in self.dates + [ datetime(1970, 1, 1, microsecond=100000), - pytz.timezone('US/Eastern').localize(datetime(1970, 1, 1, microsecond=100000)), - '1970-01-01 00:00:00.1', '1970-01-17 00:00:17.1', '0000-00-00 00:00:00.1', 0.1, - '2017-07-26T08:31:05.1', '2017-07-26T08:31:05.1Z', '2017-07-26 08:31.1', - '2017-07-26T13:31:05.1+05', '2017-07-26 13:31:05.1+0500' + pytz.timezone("US/Eastern").localize(datetime(1970, 1, 1, microsecond=100000)), + "1970-01-01 00:00:00.1", + "1970-01-17 00:00:17.1", + "0000-00-00 00:00:00.1", + 0.1, + "2017-07-26T08:31:05.1", + "2017-07-26T08:31:05.1Z", + "2017-07-26 08:31.1", + "2017-07-26T13:31:05.1+05", + "2017-07-26 13:31:05.1+0500", ]: dt = f.to_python(value, pytz.utc) self.assertTrue(dt.tzinfo) @@ -46,8 +60,7 @@ def test_datetime64_field(self): dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) self.assertEqual(dt, dt2) # Invalid values - for value in ('nope', '21/7/1999', - '2017-01 15:06:00', '2017-01-01X15:06:00', '2017-13-01T15:06:00'): + for value in ("nope", "21/7/1999", "2017-01 15:06:00", "2017-01-01X15:06:00", "2017-13-01T15:06:00"): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) @@ -56,21 +69,21 @@ def test_datetime64_field_precision(self): f = DateTime64Field(precision=precision, timezone=pytz.utc) dt = f.to_python(datetime(2000, 1, 1, microsecond=123456), pytz.utc) dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) - m = round(123456, precision - 6) # round rightmost microsecond digits according to precision + m = round(123456, precision - 6) # round rightmost microsecond digits according to precision self.assertEqual(dt2, dt.replace(microsecond=m)) def test_date_field(self): f = DateField() epoch = date(1970, 1, 1) # Valid values - for value in (datetime(1970, 1, 1), epoch, '1970-01-01', '0000-00-00', 0): + for value in (datetime(1970, 1, 1), epoch, "1970-01-01", "0000-00-00", 0): d = f.to_python(value, pytz.utc) self.assertEqual(d, epoch) # Verify that conversion to and from db string does not change value d2 = f.to_python(f.to_db_string(d, quote=False), pytz.utc) self.assertEqual(d, d2) # Invalid values - for value in ('nope', '21/7/1999', 0.5): + for value in ("nope", "21/7/1999", 0.5): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) # Range check @@ -81,29 +94,29 @@ def test_date_field(self): def test_date_field_timezone(self): # Verify that conversion of timezone-aware datetime is correct f = DateField() - dt = datetime(2017, 10, 5, tzinfo=pytz.timezone('Asia/Jerusalem')) + dt = datetime(2017, 10, 5, tzinfo=pytz.timezone("Asia/Jerusalem")) self.assertEqual(f.to_python(dt, pytz.utc), date(2017, 10, 4)) def test_datetime_field_timezone(self): # Verify that conversion of timezone-aware datetime is correct f = DateTimeField() utc_value = datetime(2017, 7, 26, 8, 31, 5, tzinfo=pytz.UTC) - for value in ( - '2017-07-26T08:31:05', - '2017-07-26T08:31:05Z', - '2017-07-26T11:31:05+03', - '2017-07-26 11:31:05+0300', - '2017-07-26T03:31:05-0500', + for value in ( + "2017-07-26T08:31:05", + "2017-07-26T08:31:05Z", + "2017-07-26T11:31:05+03", + "2017-07-26 11:31:05+0300", + "2017-07-26T03:31:05-0500", ): self.assertEqual(f.to_python(value, pytz.utc), utc_value) def test_uint8_field(self): f = UInt8Field() # Valid values - for value in (17, '17', 17.0): + for value in (17, "17", 17.0): self.assertEqual(f.to_python(value, pytz.utc), 17) # Invalid values - for value in ('nope', date.today()): + for value in ("nope", date.today()): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) # Range check diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 77a2b78..e295f3c 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -11,9 +11,8 @@ class SystemTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) def tearDown(self): self.database.drop_database() @@ -34,10 +33,10 @@ def test_drop_readonly_table(self): class SystemPartTest(unittest.TestCase): - BACKUP_DIRS = ['/var/lib/clickhouse/shadow', '/opt/clickhouse/shadow/'] + BACKUP_DIRS = ["/var/lib/clickhouse/shadow", "/opt/clickhouse/shadow/"] def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) self.database.create_table(TestTable) self.database.create_table(CustomPartitionedTable) self.database.insert([TestTable(date_field=date.today())]) @@ -51,7 +50,7 @@ def _get_backups(self): if os.path.exists(dir): _, dirnames, _ = next(os.walk(dir)) return dirnames - raise unittest.SkipTest('Cannot find backups dir') + raise unittest.SkipTest("Cannot find backups dir") def test_is_read_only(self): self.assertTrue(SystemPart.is_read_only()) @@ -109,20 +108,20 @@ def test_fetch(self): def test_query(self): SystemPart.objects_in(self.database).count() - list(SystemPart.objects_in(self.database).filter(table='testtable')) + list(SystemPart.objects_in(self.database).filter(table="testtable")) class TestTable(Model): date_field = DateField() - engine = MergeTree('date_field', ('date_field',)) + engine = MergeTree("date_field", ("date_field",)) class CustomPartitionedTable(Model): date_field = DateField() group_field = UInt32Field() - engine = MergeTree(order_by=('date_field', 'group_field'), partition_key=('toYYYYMM(date_field)', 'group_field')) + engine = MergeTree(order_by=("date_field", "group_field"), partition_key=("toYYYYMM(date_field)", "group_field")) class SystemTestModel(Model): diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py index 90e5acd..da5f4f6 100644 --- a/tests/test_uuid_fields.py +++ b/tests/test_uuid_fields.py @@ -7,29 +7,29 @@ class UUIDFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database('test-db', log_statements=True) + self.database = Database("test-db", log_statements=True) def tearDown(self): self.database.drop_database() def test_uuid_field(self): if self.database.server_version < (18, 1): - raise unittest.SkipTest('ClickHouse version too old') + raise unittest.SkipTest("ClickHouse version too old") # Create a model class TestModel(Model): i = Int16Field() f = UUIDField() engine = Memory() + self.database.create_table(TestModel) # Check valid values (all values are the same UUID) values = [ - '12345678-1234-5678-1234-567812345678', - '{12345678-1234-5678-1234-567812345678}', - '12345678123456781234567812345678', - 'urn:uuid:12345678-1234-5678-1234-567812345678', - b'\x12\x34\x56\x78'*4, + "12345678-1234-5678-1234-567812345678", + "{12345678-1234-5678-1234-567812345678}", + "12345678123456781234567812345678", + "urn:uuid:12345678-1234-5678-1234-567812345678", + b"\x12\x34\x56\x78" * 4, (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678), 0x12345678123456781234567812345678, UUID(int=0x12345678123456781234567812345678), @@ -40,7 +40,6 @@ class TestModel(Model): for rec in TestModel.objects_in(self.database): self.assertEqual(rec.f, UUID(values[0])) # Check invalid values - for value in [None, 'zzz', -1, '123']: + for value in [None, "zzz", -1, "123"]: with self.assertRaises(ValueError): TestModel(i=1, f=value) - From bec70d15db5c27dc3b141d729ab2161f56896c8b Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:30:59 +0100 Subject: [PATCH 15/51] Chore: isort tests --- tests/base_test_with_data.py | 7 +- tests/sample_migrations/0001_initial.py | 1 + tests/sample_migrations/0002.py | 1 + tests/sample_migrations/0003.py | 1 + tests/sample_migrations/0004.py | 1 + tests/sample_migrations/0005.py | 1 + tests/sample_migrations/0006.py | 1 + tests/sample_migrations/0007.py | 1 + tests/sample_migrations/0008.py | 1 + tests/sample_migrations/0009.py | 1 + tests/sample_migrations/0010.py | 1 + tests/sample_migrations/0011.py | 1 + tests/sample_migrations/0013.py | 3 +- tests/sample_migrations/0014.py | 1 + tests/sample_migrations/0015.py | 1 + tests/sample_migrations/0016.py | 1 + tests/sample_migrations/0017.py | 1 + tests/sample_migrations/0018.py | 1 + tests/sample_migrations/0019.py | 1 + tests/test_alias_fields.py | 150 +++++++------- tests/test_array_fields.py | 4 +- tests/test_buffer.py | 3 +- tests/test_compressed_fields.py | 7 +- tests/test_constraints.py | 1 + tests/test_custom_fields.py | 3 +- tests/test_database.py | 7 +- tests/test_datetime_fields.py | 7 +- tests/test_decimal_fields.py | 4 +- tests/test_dictionaries.py | 2 +- tests/test_engines.py | 5 +- tests/test_enum_fields.py | 7 +- tests/test_fixed_string_fields.py | 4 +- tests/test_funcs.py | 16 +- tests/test_inheritance.py | 7 +- tests/test_ip_fields.py | 3 +- tests/test_join.py | 96 ++++----- tests/test_materialized_fields.py | 142 ++++++------- tests/test_migrations.py | 14 +- tests/test_models.py | 7 +- tests/test_mutations.py | 4 +- tests/test_nullable_fields.py | 8 +- tests/test_querysets.py | 13 +- tests/test_readonly.py | 1 + tests/test_simple_fields.py | 4 +- tests/test_system_models.py | 255 ++++++++++++------------ tests/test_uuid_fields.py | 3 +- 46 files changed, 418 insertions(+), 386 deletions(-) diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index 6530681..17da512 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +import logging import unittest from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * - -import logging +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/sample_migrations/0001_initial.py b/tests/sample_migrations/0001_initial.py index 920fa16..b3c8066 100644 --- a/tests/sample_migrations/0001_initial.py +++ b/tests/sample_migrations/0001_initial.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(Model1)] diff --git a/tests/sample_migrations/0002.py b/tests/sample_migrations/0002.py index d289805..44855c0 100644 --- a/tests/sample_migrations/0002.py +++ b/tests/sample_migrations/0002.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.DropTable(Model1)] diff --git a/tests/sample_migrations/0003.py b/tests/sample_migrations/0003.py index 920fa16..b3c8066 100644 --- a/tests/sample_migrations/0003.py +++ b/tests/sample_migrations/0003.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(Model1)] diff --git a/tests/sample_migrations/0004.py b/tests/sample_migrations/0004.py index 3ae6701..c945596 100644 --- a/tests/sample_migrations/0004.py +++ b/tests/sample_migrations/0004.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTable(Model2)] diff --git a/tests/sample_migrations/0005.py b/tests/sample_migrations/0005.py index e938ee4..fb4837a 100644 --- a/tests/sample_migrations/0005.py +++ b/tests/sample_migrations/0005.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTable(Model3)] diff --git a/tests/sample_migrations/0006.py b/tests/sample_migrations/0006.py index ec1a204..9b0d410 100644 --- a/tests/sample_migrations/0006.py +++ b/tests/sample_migrations/0006.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(EnumModel1)] diff --git a/tests/sample_migrations/0007.py b/tests/sample_migrations/0007.py index 2138fbb..ee33343 100644 --- a/tests/sample_migrations/0007.py +++ b/tests/sample_migrations/0007.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTable(EnumModel2)] diff --git a/tests/sample_migrations/0008.py b/tests/sample_migrations/0008.py index a452642..912b573 100644 --- a/tests/sample_migrations/0008.py +++ b/tests/sample_migrations/0008.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(MaterializedModel)] diff --git a/tests/sample_migrations/0009.py b/tests/sample_migrations/0009.py index 65731ca..d411f53 100644 --- a/tests/sample_migrations/0009.py +++ b/tests/sample_migrations/0009.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(AliasModel)] diff --git a/tests/sample_migrations/0010.py b/tests/sample_migrations/0010.py index 81c53c3..175e2d5 100644 --- a/tests/sample_migrations/0010.py +++ b/tests/sample_migrations/0010.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(Model4Buffer)] diff --git a/tests/sample_migrations/0011.py b/tests/sample_migrations/0011.py index 1bee2fe..59e3b08 100644 --- a/tests/sample_migrations/0011.py +++ b/tests/sample_migrations/0011.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTableWithBuffer(Model4Buffer_changed)] diff --git a/tests/sample_migrations/0013.py b/tests/sample_migrations/0013.py index ade30c5..2b50fb1 100644 --- a/tests/sample_migrations/0013.py +++ b/tests/sample_migrations/0013.py @@ -1,8 +1,9 @@ import datetime -from clickhouse_orm import migrations from test_migrations import Model3 +from clickhouse_orm import migrations + def forward(database): database.insert([Model3(date=datetime.date(2016, 1, 4), f1=4, f3=1, f4="test4")]) diff --git a/tests/sample_migrations/0014.py b/tests/sample_migrations/0014.py index 47b3a12..1823f77 100644 --- a/tests/sample_migrations/0014.py +++ b/tests/sample_migrations/0014.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTable(MaterializedModel1), migrations.AlterTable(AliasModel1)] diff --git a/tests/sample_migrations/0015.py b/tests/sample_migrations/0015.py index 02c5d15..0d03f6e 100644 --- a/tests/sample_migrations/0015.py +++ b/tests/sample_migrations/0015.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterTable(Model4_compressed), migrations.AlterTable(Model2LowCardinality)] diff --git a/tests/sample_migrations/0016.py b/tests/sample_migrations/0016.py index 6c11dfc..b0da930 100644 --- a/tests/sample_migrations/0016.py +++ b/tests/sample_migrations/0016.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(ModelWithConstraints)] diff --git a/tests/sample_migrations/0017.py b/tests/sample_migrations/0017.py index 4fb2e8f..abe414c 100644 --- a/tests/sample_migrations/0017.py +++ b/tests/sample_migrations/0017.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterConstraints(ModelWithConstraints2)] diff --git a/tests/sample_migrations/0018.py b/tests/sample_migrations/0018.py index 0c52095..a39a44b 100644 --- a/tests/sample_migrations/0018.py +++ b/tests/sample_migrations/0018.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.CreateTable(ModelWithIndex)] diff --git a/tests/sample_migrations/0019.py b/tests/sample_migrations/0019.py index 328f5e7..6221e4b 100644 --- a/tests/sample_migrations/0019.py +++ b/tests/sample_migrations/0019.py @@ -1,4 +1,5 @@ from clickhouse_orm import migrations + from ..test_migrations import * operations = [migrations.AlterIndexes(ModelWithIndex2, reindex=True)] diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index 52da31e..710cddb 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -1,75 +1,75 @@ -import unittest -from datetime import date - -from clickhouse_orm.database import Database -from clickhouse_orm.models import Model, NO_VALUE -from clickhouse_orm.fields import * -from clickhouse_orm.engines import * -from clickhouse_orm.funcs import F - - -class AliasFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database("test-db", log_statements=True) - self.database.create_table(ModelWithAliasFields) - - def tearDown(self): - self.database.drop_database() - - def test_insert_and_select(self): - instance = ModelWithAliasFields(date_field="2016-08-30", int_field=-10, str_field="TEST") - self.database.insert([instance]) - # We can't select * from table, as it doesn't select materialized and alias fields - query = ( - "SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str, alias_func" - " FROM $db.%s ORDER BY alias_date" % ModelWithAliasFields.table_name() - ) - for model_cls in (ModelWithAliasFields, None): - results = list(self.database.select(query, model_cls)) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].date_field, instance.date_field) - self.assertEqual(results[0].int_field, instance.int_field) - self.assertEqual(results[0].str_field, instance.str_field) - self.assertEqual(results[0].alias_int, instance.int_field) - self.assertEqual(results[0].alias_str, instance.str_field) - self.assertEqual(results[0].alias_date, instance.date_field) - self.assertEqual(results[0].alias_func, 201608) - - def test_assignment_error(self): - # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. - instance = ModelWithAliasFields() - for value in ("x", [date.today()], ["aaa"], [None]): - with self.assertRaises(ValueError): - instance.alias_date = value - - def test_wrong_field(self): - with self.assertRaises(AssertionError): - StringField(alias=123) - - def test_duplicate_default(self): - with self.assertRaises(AssertionError): - StringField(alias="str_field", default="with default") - - with self.assertRaises(AssertionError): - StringField(alias="str_field", materialized="str_field") - - def test_default_value(self): - instance = ModelWithAliasFields() - self.assertEqual(instance.alias_str, NO_VALUE) - # Check that NO_VALUE can be assigned to a field - instance.str_field = NO_VALUE - # Check that NO_VALUE can be assigned when creating a new instance - instance2 = ModelWithAliasFields(**instance.to_dict()) - - -class ModelWithAliasFields(Model): - int_field = Int32Field() - date_field = DateField() - str_field = StringField() - - alias_str = StringField(alias=u"str_field") - alias_int = Int32Field(alias="int_field") - alias_date = DateField(alias="date_field") - alias_func = Int32Field(alias=F.toYYYYMM(date_field)) - - engine = MergeTree("date_field", ("date_field",)) +import unittest +from datetime import date + +from clickhouse_orm.database import Database +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import F +from clickhouse_orm.models import NO_VALUE, Model + + +class AliasFieldsTest(unittest.TestCase): + def setUp(self): + self.database = Database("test-db", log_statements=True) + self.database.create_table(ModelWithAliasFields) + + def tearDown(self): + self.database.drop_database() + + def test_insert_and_select(self): + instance = ModelWithAliasFields(date_field="2016-08-30", int_field=-10, str_field="TEST") + self.database.insert([instance]) + # We can't select * from table, as it doesn't select materialized and alias fields + query = ( + "SELECT date_field, int_field, str_field, alias_int, alias_date, alias_str, alias_func" + " FROM $db.%s ORDER BY alias_date" % ModelWithAliasFields.table_name() + ) + for model_cls in (ModelWithAliasFields, None): + results = list(self.database.select(query, model_cls)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].date_field, instance.date_field) + self.assertEqual(results[0].int_field, instance.int_field) + self.assertEqual(results[0].str_field, instance.str_field) + self.assertEqual(results[0].alias_int, instance.int_field) + self.assertEqual(results[0].alias_str, instance.str_field) + self.assertEqual(results[0].alias_date, instance.date_field) + self.assertEqual(results[0].alias_func, 201608) + + def test_assignment_error(self): + # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. + instance = ModelWithAliasFields() + for value in ("x", [date.today()], ["aaa"], [None]): + with self.assertRaises(ValueError): + instance.alias_date = value + + def test_wrong_field(self): + with self.assertRaises(AssertionError): + StringField(alias=123) + + def test_duplicate_default(self): + with self.assertRaises(AssertionError): + StringField(alias="str_field", default="with default") + + with self.assertRaises(AssertionError): + StringField(alias="str_field", materialized="str_field") + + def test_default_value(self): + instance = ModelWithAliasFields() + self.assertEqual(instance.alias_str, NO_VALUE) + # Check that NO_VALUE can be assigned to a field + instance.str_field = NO_VALUE + # Check that NO_VALUE can be assigned when creating a new instance + instance2 = ModelWithAliasFields(**instance.to_dict()) + + +class ModelWithAliasFields(Model): + int_field = Int32Field() + date_field = DateField() + str_field = StringField() + + alias_str = StringField(alias=u"str_field") + alias_int = Int32Field(alias="int_field") + alias_date = DateField(alias="date_field") + alias_func = Int32Field(alias=F.toYYYYMM(date_field)) + + engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_array_fields.py b/tests/test_array_fields.py index ea92644..f9fe301 100644 --- a/tests/test_array_fields.py +++ b/tests/test_array_fields.py @@ -2,9 +2,9 @@ from datetime import date from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class ArrayFieldsTest(unittest.TestCase): diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 24be02e..98d5785 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- import unittest -from clickhouse_orm.models import BufferModel from clickhouse_orm.engines import * +from clickhouse_orm.models import BufferModel + from .base_test_with_data import * diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index b514853..8b26227 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -1,11 +1,12 @@ -import unittest import datetime +import unittest + import pytz from clickhouse_orm.database import Database -from clickhouse_orm.models import Model, NO_VALUE -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import NO_VALUE, Model from clickhouse_orm.utils import parse_tsv diff --git a/tests/test_constraints.py b/tests/test_constraints.py index a849c93..c2dfa0d 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -1,6 +1,7 @@ import unittest from clickhouse_orm import * + from .base_test_with_data import Person diff --git a/tests/test_custom_fields.py b/tests/test_custom_fields.py index 26e82d8..aa1655c 100644 --- a/tests/test_custom_fields.py +++ b/tests/test_custom_fields.py @@ -1,8 +1,9 @@ import unittest + from clickhouse_orm.database import Database +from clickhouse_orm.engines import Memory from clickhouse_orm.fields import Field, Int16Field from clickhouse_orm.models import Model -from clickhouse_orm.engines import Memory class CustomFieldsTest(unittest.TestCase): diff --git a/tests/test_database.py b/tests/test_database.py index b3c1ca6..bb73327 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- -import unittest import datetime +import unittest -from clickhouse_orm.database import ServerError, DatabaseException -from clickhouse_orm.models import Model +from clickhouse_orm.database import DatabaseException, ServerError from clickhouse_orm.engines import Memory from clickhouse_orm.fields import * from clickhouse_orm.funcs import F +from clickhouse_orm.models import Model from clickhouse_orm.query import Q + from .base_test_with_data import * diff --git a/tests/test_datetime_fields.py b/tests/test_datetime_fields.py index 9d8f2bd..7b8e231 100644 --- a/tests/test_datetime_fields.py +++ b/tests/test_datetime_fields.py @@ -1,11 +1,12 @@ -import unittest import datetime +import unittest + import pytz from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class DateFieldsTest(unittest.TestCase): diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index ec463f8..245360f 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -3,9 +3,9 @@ from decimal import Decimal from clickhouse_orm.database import Database, ServerError -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class DecimalFieldsTest(unittest.TestCase): diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 15f17aa..39e1f3b 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -1,5 +1,5 @@ -import unittest import logging +import unittest from clickhouse_orm import * diff --git a/tests/test_engines.py b/tests/test_engines.py index a2775fb..f796a27 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -1,10 +1,9 @@ -import unittest import datetime +import logging +import unittest from clickhouse_orm import * -import logging - logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/test_enum_fields.py b/tests/test_enum_fields.py index 004c0da..05ad366 100644 --- a/tests/test_enum_fields.py +++ b/tests/test_enum_fields.py @@ -1,11 +1,10 @@ import unittest +from enum import Enum from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * - -from enum import Enum +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class EnumFieldsTest(unittest.TestCase): diff --git a/tests/test_fixed_string_fields.py b/tests/test_fixed_string_fields.py index fc41478..d1bce32 100644 --- a/tests/test_fixed_string_fields.py +++ b/tests/test_fixed_string_fields.py @@ -2,9 +2,9 @@ import unittest from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class FixedStringFieldsTest(unittest.TestCase): diff --git a/tests/test_funcs.py b/tests/test_funcs.py index 87ca724..2d46550 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -1,15 +1,17 @@ -import unittest -from .base_test_with_data import * -from .test_querysets import SampleModel -from datetime import date, datetime, tzinfo, timedelta -import pytz -from ipaddress import IPv4Address, IPv6Address import logging +import unittest +from datetime import date, datetime, timedelta, tzinfo from decimal import Decimal +from ipaddress import IPv4Address, IPv6Address + +import pytz from clickhouse_orm.database import ServerError -from clickhouse_orm.utils import NO_VALUE from clickhouse_orm.funcs import F +from clickhouse_orm.utils import NO_VALUE + +from .base_test_with_data import * +from .test_querysets import SampleModel class FuncsTestCase(TestCaseWithData): diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index ae36be0..622167f 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -1,11 +1,12 @@ -import unittest import datetime +import unittest + import pytz from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model class InheritanceTestCase(unittest.TestCase): diff --git a/tests/test_ip_fields.py b/tests/test_ip_fields.py index 2f81064..f5db64e 100644 --- a/tests/test_ip_fields.py +++ b/tests/test_ip_fields.py @@ -1,9 +1,10 @@ import unittest from ipaddress import IPv4Address, IPv6Address + from clickhouse_orm.database import Database +from clickhouse_orm.engines import Memory from clickhouse_orm.fields import Int16Field, IPv4Field, IPv6Field from clickhouse_orm.models import Model -from clickhouse_orm.engines import Memory class IPFieldsTest(unittest.TestCase): diff --git a/tests/test_join.py b/tests/test_join.py index 2a72b41..58d1f73 100644 --- a/tests/test_join.py +++ b/tests/test_join.py @@ -1,48 +1,48 @@ -import unittest -import json - -from clickhouse_orm import database, engines, fields, models - - -class JoinTest(unittest.TestCase): - def setUp(self): - self.database = database.Database("test-db", log_statements=True) - self.database.create_table(Foo) - self.database.create_table(Bar) - self.database.insert([Foo(id=i) for i in range(3)]) - self.database.insert([Bar(id=i, b=i * i) for i in range(3)]) - - def print_res(self, query): - print(query) - print(json.dumps([row.to_dict() for row in self.database.select(query)])) - - def test_without_db_name(self): - self.print_res("SELECT * FROM {}".format(Foo.table_name())) - self.print_res("SELECT * FROM {}".format(Bar.table_name())) - self.print_res("SELECT b FROM {} ALL LEFT JOIN {} USING id".format(Foo.table_name(), Bar.table_name())) - - def test_with_db_name(self): - self.print_res("SELECT * FROM $db.{}".format(Foo.table_name())) - self.print_res("SELECT * FROM $db.{}".format(Bar.table_name())) - self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN $db.{} USING id".format(Foo.table_name(), Bar.table_name())) - - def test_with_subquery(self): - self.print_res( - "SELECT b FROM {} ALL LEFT JOIN (SELECT * from {}) subquery USING id".format( - Foo.table_name(), Bar.table_name() - ) - ) - self.print_res( - "SELECT b FROM $db.{} ALL LEFT JOIN (SELECT * from $db.{}) subquery USING id".format( - Foo.table_name(), Bar.table_name() - ) - ) - - -class Foo(models.Model): - id = fields.UInt8Field() - engine = engines.Memory() - - -class Bar(Foo): - b = fields.UInt8Field() +import json +import unittest + +from clickhouse_orm import database, engines, fields, models + + +class JoinTest(unittest.TestCase): + def setUp(self): + self.database = database.Database("test-db", log_statements=True) + self.database.create_table(Foo) + self.database.create_table(Bar) + self.database.insert([Foo(id=i) for i in range(3)]) + self.database.insert([Bar(id=i, b=i * i) for i in range(3)]) + + def print_res(self, query): + print(query) + print(json.dumps([row.to_dict() for row in self.database.select(query)])) + + def test_without_db_name(self): + self.print_res("SELECT * FROM {}".format(Foo.table_name())) + self.print_res("SELECT * FROM {}".format(Bar.table_name())) + self.print_res("SELECT b FROM {} ALL LEFT JOIN {} USING id".format(Foo.table_name(), Bar.table_name())) + + def test_with_db_name(self): + self.print_res("SELECT * FROM $db.{}".format(Foo.table_name())) + self.print_res("SELECT * FROM $db.{}".format(Bar.table_name())) + self.print_res("SELECT b FROM $db.{} ALL LEFT JOIN $db.{} USING id".format(Foo.table_name(), Bar.table_name())) + + def test_with_subquery(self): + self.print_res( + "SELECT b FROM {} ALL LEFT JOIN (SELECT * from {}) subquery USING id".format( + Foo.table_name(), Bar.table_name() + ) + ) + self.print_res( + "SELECT b FROM $db.{} ALL LEFT JOIN (SELECT * from $db.{}) subquery USING id".format( + Foo.table_name(), Bar.table_name() + ) + ) + + +class Foo(models.Model): + id = fields.UInt8Field() + engine = engines.Memory() + + +class Bar(Foo): + b = fields.UInt8Field() diff --git a/tests/test_materialized_fields.py b/tests/test_materialized_fields.py index b03f211..ab68a09 100644 --- a/tests/test_materialized_fields.py +++ b/tests/test_materialized_fields.py @@ -1,71 +1,71 @@ -import unittest -from datetime import date - -from clickhouse_orm.database import Database -from clickhouse_orm.models import Model, NO_VALUE -from clickhouse_orm.fields import * -from clickhouse_orm.engines import * -from clickhouse_orm.funcs import F - - -class MaterializedFieldsTest(unittest.TestCase): - def setUp(self): - self.database = Database("test-db", log_statements=True) - self.database.create_table(ModelWithMaterializedFields) - - def tearDown(self): - self.database.drop_database() - - def test_insert_and_select(self): - instance = ModelWithMaterializedFields(date_time_field="2016-08-30 11:00:00", int_field=-10, str_field="TEST") - self.database.insert([instance]) - # We can't select * from table, as it doesn't select materialized and alias fields - query = ( - "SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str, mat_func" - " FROM $db.%s ORDER BY mat_date" % ModelWithMaterializedFields.table_name() - ) - for model_cls in (ModelWithMaterializedFields, None): - results = list(self.database.select(query, model_cls)) - self.assertEqual(len(results), 1) - self.assertEqual(results[0].date_time_field, instance.date_time_field) - self.assertEqual(results[0].int_field, instance.int_field) - self.assertEqual(results[0].str_field, instance.str_field) - self.assertEqual(results[0].mat_int, abs(instance.int_field)) - self.assertEqual(results[0].mat_str, instance.str_field.lower()) - self.assertEqual(results[0].mat_date, instance.date_time_field.date()) - self.assertEqual(results[0].mat_func, instance.str_field.lower()) - - def test_assignment_error(self): - # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. - instance = ModelWithMaterializedFields() - for value in ("x", [date.today()], ["aaa"], [None]): - with self.assertRaises(ValueError): - instance.mat_date = value - - def test_wrong_field(self): - with self.assertRaises(AssertionError): - StringField(materialized=123) - - def test_duplicate_default(self): - with self.assertRaises(AssertionError): - StringField(materialized="str_field", default="with default") - - with self.assertRaises(AssertionError): - StringField(materialized="str_field", alias="str_field") - - def test_default_value(self): - instance = ModelWithMaterializedFields() - self.assertEqual(instance.mat_str, NO_VALUE) - - -class ModelWithMaterializedFields(Model): - int_field = Int32Field() - date_time_field = DateTimeField() - str_field = StringField() - - mat_str = StringField(materialized="lower(str_field)") - mat_int = Int32Field(materialized="abs(int_field)") - mat_date = DateField(materialized=u"toDate(date_time_field)") - mat_func = StringField(materialized=F.lower(str_field)) - - engine = MergeTree("mat_date", ("mat_date",)) +import unittest +from datetime import date + +from clickhouse_orm.database import Database +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.funcs import F +from clickhouse_orm.models import NO_VALUE, Model + + +class MaterializedFieldsTest(unittest.TestCase): + def setUp(self): + self.database = Database("test-db", log_statements=True) + self.database.create_table(ModelWithMaterializedFields) + + def tearDown(self): + self.database.drop_database() + + def test_insert_and_select(self): + instance = ModelWithMaterializedFields(date_time_field="2016-08-30 11:00:00", int_field=-10, str_field="TEST") + self.database.insert([instance]) + # We can't select * from table, as it doesn't select materialized and alias fields + query = ( + "SELECT date_time_field, int_field, str_field, mat_int, mat_date, mat_str, mat_func" + " FROM $db.%s ORDER BY mat_date" % ModelWithMaterializedFields.table_name() + ) + for model_cls in (ModelWithMaterializedFields, None): + results = list(self.database.select(query, model_cls)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].date_time_field, instance.date_time_field) + self.assertEqual(results[0].int_field, instance.int_field) + self.assertEqual(results[0].str_field, instance.str_field) + self.assertEqual(results[0].mat_int, abs(instance.int_field)) + self.assertEqual(results[0].mat_str, instance.str_field.lower()) + self.assertEqual(results[0].mat_date, instance.date_time_field.date()) + self.assertEqual(results[0].mat_func, instance.str_field.lower()) + + def test_assignment_error(self): + # I can't prevent assigning at all, in case db.select statements with model provided sets model fields. + instance = ModelWithMaterializedFields() + for value in ("x", [date.today()], ["aaa"], [None]): + with self.assertRaises(ValueError): + instance.mat_date = value + + def test_wrong_field(self): + with self.assertRaises(AssertionError): + StringField(materialized=123) + + def test_duplicate_default(self): + with self.assertRaises(AssertionError): + StringField(materialized="str_field", default="with default") + + with self.assertRaises(AssertionError): + StringField(materialized="str_field", alias="str_field") + + def test_default_value(self): + instance = ModelWithMaterializedFields() + self.assertEqual(instance.mat_str, NO_VALUE) + + +class ModelWithMaterializedFields(Model): + int_field = Int32Field() + date_time_field = DateTimeField() + str_field = StringField() + + mat_str = StringField(materialized="lower(str_field)") + mat_int = Int32Field(materialized="abs(int_field)") + mat_date = DateField(materialized=u"toDate(date_time_field)") + mat_func = StringField(materialized=F.lower(str_field)) + + engine = MergeTree("mat_date", ("mat_date",)) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 093fd41..fd226a1 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,15 +1,15 @@ +import os + +# Add tests to path so that migrations will be importable +import sys import unittest +from enum import Enum from clickhouse_orm.database import Database, ServerError -from clickhouse_orm.models import Model, BufferModel, Constraint, Index -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * from clickhouse_orm.migrations import MigrationHistory - -from enum import Enum - -# Add tests to path so that migrations will be importable -import sys, os +from clickhouse_orm.models import BufferModel, Constraint, Index, Model sys.path.append(os.path.dirname(__file__)) diff --git a/tests/test_models.py b/tests/test_models.py index fd28034..a6a4917 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,11 +1,12 @@ -import unittest import datetime +import unittest + import pytz -from clickhouse_orm.models import Model, NO_VALUE -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * from clickhouse_orm.funcs import F +from clickhouse_orm.models import NO_VALUE, Model class ModelTestCase(unittest.TestCase): diff --git a/tests/test_mutations.py b/tests/test_mutations.py index 89b0146..afbb1ed 100644 --- a/tests/test_mutations.py +++ b/tests/test_mutations.py @@ -1,7 +1,9 @@ import unittest +from time import sleep + from clickhouse_orm import F + from .base_test_with_data import * -from time import sleep class MutationsTestCase(TestCaseWithData): diff --git a/tests/test_nullable_fields.py b/tests/test_nullable_fields.py index 3c33e21..8b81ef2 100644 --- a/tests/test_nullable_fields.py +++ b/tests/test_nullable_fields.py @@ -1,14 +1,14 @@ import unittest +from datetime import date, datetime + import pytz from clickhouse_orm.database import Database -from clickhouse_orm.models import Model -from clickhouse_orm.fields import * from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model from clickhouse_orm.utils import comma_join -from datetime import date, datetime - class NullableFieldsTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 719541e..d021029 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- import unittest -from clickhouse_orm.database import Database -from clickhouse_orm.query import Q -from clickhouse_orm.funcs import F -from .base_test_with_data import * from datetime import date, datetime -from enum import Enum from decimal import Decimal - +from enum import Enum from logging import getLogger +from clickhouse_orm.database import Database +from clickhouse_orm.funcs import F +from clickhouse_orm.query import Q + +from .base_test_with_data import * + logger = getLogger("tests") diff --git a/tests/test_readonly.py b/tests/test_readonly.py index c085215..9d3174a 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from clickhouse_orm.database import DatabaseException, ServerError + from .base_test_with_data import * diff --git a/tests/test_simple_fields.py b/tests/test_simple_fields.py index 46ebc04..3611bd9 100644 --- a/tests/test_simple_fields.py +++ b/tests/test_simple_fields.py @@ -1,8 +1,10 @@ import unittest -from clickhouse_orm.fields import * from datetime import date, datetime + import pytz +from clickhouse_orm.fields import * + class SimpleFieldsTest(unittest.TestCase): diff --git a/tests/test_system_models.py b/tests/test_system_models.py index e295f3c..7f7d82f 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -1,128 +1,127 @@ -import unittest -from datetime import date - -import os - -from clickhouse_orm.database import Database, DatabaseException -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * -from clickhouse_orm.models import Model -from clickhouse_orm.system_models import SystemPart - - -class SystemTest(unittest.TestCase): - def setUp(self): - self.database = Database("test-db", log_statements=True) - - def tearDown(self): - self.database.drop_database() - - def test_insert_system(self): - m = SystemPart() - with self.assertRaises(DatabaseException): - self.database.insert([m]) - - def test_create_readonly_table(self): - with self.assertRaises(DatabaseException): - self.database.create_table(SystemTestModel) - - def test_drop_readonly_table(self): - with self.assertRaises(DatabaseException): - self.database.drop_table(SystemTestModel) - - -class SystemPartTest(unittest.TestCase): - - BACKUP_DIRS = ["/var/lib/clickhouse/shadow", "/opt/clickhouse/shadow/"] - - def setUp(self): - self.database = Database("test-db", log_statements=True) - self.database.create_table(TestTable) - self.database.create_table(CustomPartitionedTable) - self.database.insert([TestTable(date_field=date.today())]) - self.database.insert([CustomPartitionedTable(date_field=date.today(), group_field=13)]) - - def tearDown(self): - self.database.drop_database() - - def _get_backups(self): - for dir in self.BACKUP_DIRS: - if os.path.exists(dir): - _, dirnames, _ = next(os.walk(dir)) - return dirnames - raise unittest.SkipTest("Cannot find backups dir") - - def test_is_read_only(self): - self.assertTrue(SystemPart.is_read_only()) - - def test_is_system_model(self): - self.assertTrue(SystemPart.is_system_model()) - - def test_get_all(self): - parts = SystemPart.get(self.database) - self.assertEqual(len(list(parts)), 2) - - def test_get_active(self): - parts = list(SystemPart.get_active(self.database)) - self.assertEqual(len(parts), 2) - parts[0].detach() - parts = list(SystemPart.get_active(self.database)) - self.assertEqual(len(parts), 1) - - def test_get_conditions(self): - parts = list(SystemPart.get(self.database, conditions="table='testtable'")) - self.assertEqual(len(parts), 1) - parts = list(SystemPart.get(self.database, conditions=u"table='custompartitionedtable'")) - self.assertEqual(len(parts), 1) - parts = list(SystemPart.get(self.database, conditions=u"table='invalidtable'")) - self.assertEqual(len(parts), 0) - - def test_attach_detach(self): - parts = list(SystemPart.get_active(self.database)) - self.assertEqual(len(parts), 2) - for p in parts: - p.detach() - self.assertEqual(len(list(SystemPart.get_active(self.database))), 0) - for p in parts: - p.attach() - self.assertEqual(len(list(SystemPart.get_active(self.database))), 2) - - def test_drop(self): - parts = list(SystemPart.get_active(self.database)) - for p in parts: - p.drop() - self.assertEqual(len(list(SystemPart.get_active(self.database))), 0) - - def test_freeze(self): - parts = list(SystemPart.get(self.database)) - # There can be other backups in the folder - prev_backups = set(self._get_backups()) - for p in parts: - p.freeze() - backups = set(self._get_backups()) - self.assertEqual(len(backups), len(prev_backups) + 2) - - def test_fetch(self): - # TODO Not tested, as I have no replication set - pass - - def test_query(self): - SystemPart.objects_in(self.database).count() - list(SystemPart.objects_in(self.database).filter(table="testtable")) - - -class TestTable(Model): - date_field = DateField() - - engine = MergeTree("date_field", ("date_field",)) - - -class CustomPartitionedTable(Model): - date_field = DateField() - group_field = UInt32Field() - - engine = MergeTree(order_by=("date_field", "group_field"), partition_key=("toYYYYMM(date_field)", "group_field")) - - -class SystemTestModel(Model): - _system = True +import os +import unittest +from datetime import date + +from clickhouse_orm.database import Database, DatabaseException +from clickhouse_orm.engines import * +from clickhouse_orm.fields import * +from clickhouse_orm.models import Model +from clickhouse_orm.system_models import SystemPart + + +class SystemTest(unittest.TestCase): + def setUp(self): + self.database = Database("test-db", log_statements=True) + + def tearDown(self): + self.database.drop_database() + + def test_insert_system(self): + m = SystemPart() + with self.assertRaises(DatabaseException): + self.database.insert([m]) + + def test_create_readonly_table(self): + with self.assertRaises(DatabaseException): + self.database.create_table(SystemTestModel) + + def test_drop_readonly_table(self): + with self.assertRaises(DatabaseException): + self.database.drop_table(SystemTestModel) + + +class SystemPartTest(unittest.TestCase): + + BACKUP_DIRS = ["/var/lib/clickhouse/shadow", "/opt/clickhouse/shadow/"] + + def setUp(self): + self.database = Database("test-db", log_statements=True) + self.database.create_table(TestTable) + self.database.create_table(CustomPartitionedTable) + self.database.insert([TestTable(date_field=date.today())]) + self.database.insert([CustomPartitionedTable(date_field=date.today(), group_field=13)]) + + def tearDown(self): + self.database.drop_database() + + def _get_backups(self): + for dir in self.BACKUP_DIRS: + if os.path.exists(dir): + _, dirnames, _ = next(os.walk(dir)) + return dirnames + raise unittest.SkipTest("Cannot find backups dir") + + def test_is_read_only(self): + self.assertTrue(SystemPart.is_read_only()) + + def test_is_system_model(self): + self.assertTrue(SystemPart.is_system_model()) + + def test_get_all(self): + parts = SystemPart.get(self.database) + self.assertEqual(len(list(parts)), 2) + + def test_get_active(self): + parts = list(SystemPart.get_active(self.database)) + self.assertEqual(len(parts), 2) + parts[0].detach() + parts = list(SystemPart.get_active(self.database)) + self.assertEqual(len(parts), 1) + + def test_get_conditions(self): + parts = list(SystemPart.get(self.database, conditions="table='testtable'")) + self.assertEqual(len(parts), 1) + parts = list(SystemPart.get(self.database, conditions=u"table='custompartitionedtable'")) + self.assertEqual(len(parts), 1) + parts = list(SystemPart.get(self.database, conditions=u"table='invalidtable'")) + self.assertEqual(len(parts), 0) + + def test_attach_detach(self): + parts = list(SystemPart.get_active(self.database)) + self.assertEqual(len(parts), 2) + for p in parts: + p.detach() + self.assertEqual(len(list(SystemPart.get_active(self.database))), 0) + for p in parts: + p.attach() + self.assertEqual(len(list(SystemPart.get_active(self.database))), 2) + + def test_drop(self): + parts = list(SystemPart.get_active(self.database)) + for p in parts: + p.drop() + self.assertEqual(len(list(SystemPart.get_active(self.database))), 0) + + def test_freeze(self): + parts = list(SystemPart.get(self.database)) + # There can be other backups in the folder + prev_backups = set(self._get_backups()) + for p in parts: + p.freeze() + backups = set(self._get_backups()) + self.assertEqual(len(backups), len(prev_backups) + 2) + + def test_fetch(self): + # TODO Not tested, as I have no replication set + pass + + def test_query(self): + SystemPart.objects_in(self.database).count() + list(SystemPart.objects_in(self.database).filter(table="testtable")) + + +class TestTable(Model): + date_field = DateField() + + engine = MergeTree("date_field", ("date_field",)) + + +class CustomPartitionedTable(Model): + date_field = DateField() + group_field = UInt32Field() + + engine = MergeTree(order_by=("date_field", "group_field"), partition_key=("toYYYYMM(date_field)", "group_field")) + + +class SystemTestModel(Model): + _system = True diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py index da5f4f6..af17738 100644 --- a/tests/test_uuid_fields.py +++ b/tests/test_uuid_fields.py @@ -1,9 +1,10 @@ import unittest from uuid import UUID + from clickhouse_orm.database import Database +from clickhouse_orm.engines import Memory from clickhouse_orm.fields import Int16Field, UUIDField from clickhouse_orm.models import Model -from clickhouse_orm.engines import Memory class UUIDFieldsTest(unittest.TestCase): From a3b822899f06f76e60a53bce00914e73cdc70fad Mon Sep 17 00:00:00 2001 From: olliemath Date: Tue, 27 Jul 2021 23:48:50 +0100 Subject: [PATCH 16/51] Chore: fix linting on tests --- tests/test_alias_fields.py | 2 +- tests/test_buffer.py | 2 -- tests/test_compressed_fields.py | 6 +++--- tests/test_constraints.py | 4 ++-- tests/test_database.py | 4 ++-- tests/test_decimal_fields.py | 6 +++--- tests/test_funcs.py | 12 ++++++------ tests/test_inheritance.py | 3 --- tests/test_ip_fields.py | 2 ++ tests/test_migrations.py | 7 ++----- tests/test_models.py | 2 +- tests/test_querysets.py | 1 - tests/test_readonly.py | 6 +++--- tests/test_uuid_fields.py | 1 + 14 files changed, 26 insertions(+), 32 deletions(-) diff --git a/tests/test_alias_fields.py b/tests/test_alias_fields.py index 710cddb..5b77513 100644 --- a/tests/test_alias_fields.py +++ b/tests/test_alias_fields.py @@ -59,7 +59,7 @@ def test_default_value(self): # Check that NO_VALUE can be assigned to a field instance.str_field = NO_VALUE # Check that NO_VALUE can be assigned when creating a new instance - instance2 = ModelWithAliasFields(**instance.to_dict()) + ModelWithAliasFields(**instance.to_dict()) class ModelWithAliasFields(Model): diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 98d5785..526e07d 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import unittest - from clickhouse_orm.engines import * from clickhouse_orm.models import BufferModel diff --git a/tests/test_compressed_fields.py b/tests/test_compressed_fields.py index 8b26227..8387f28 100644 --- a/tests/test_compressed_fields.py +++ b/tests/test_compressed_fields.py @@ -44,7 +44,7 @@ def test_assignment(self): ) instance = CompressedModel(**kwargs) self.database.insert([instance]) - for name, value in kwargs.items(): + for name in kwargs: self.assertEqual(kwargs[name], getattr(instance, name)) def test_string_conversion(self): @@ -106,8 +106,8 @@ def test_confirm_compression_codec(self): ) ) lines = r.splitlines() - field_names = parse_tsv(lines[0]) - field_types = parse_tsv(lines[1]) + parse_tsv(lines[0]) + parse_tsv(lines[1]) data = [tuple(parse_tsv(line)) for line in lines[2:]] self.assertListEqual( data, diff --git a/tests/test_constraints.py b/tests/test_constraints.py index c2dfa0d..399a154 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -26,14 +26,14 @@ def test_insert_invalid_values(self): [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2100-01-01", height=1.66)] ) self.assertEqual(e.code, 469) - self.assertTrue("Constraint `birthday_in_the_past`" in e.message) + self.assertTrue("Constraint `birthday_in_the_past`" in str(e)) with self.assertRaises(ServerError) as e: self.database.insert( [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="1970-01-01", height=3)] ) self.assertEqual(e.code, 469) - self.assertTrue("Constraint `max_height`" in e.message) + self.assertTrue("Constraint `max_height`" in str(e)) class PersonWithConstraints(Person): diff --git a/tests/test_database.py b/tests/test_database.py index bb73327..9566a3c 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -199,7 +199,7 @@ def test_nonexisting_db(self): self.assertEqual(exc.code, 81) self.assertTrue(exc.message.startswith("Database db_not_here doesn't exist")) # Create and delete the db twice, to ensure db_exists gets updated - for i in range(2): + for _ in range(2): # Now create the database - should succeed db.create_database() self.assertTrue(db.db_exists) @@ -284,7 +284,7 @@ def test_get_model_for_table__system(self): try: list(model.objects_in(self.database)[:10]) except ServerError as e: - if "Not enough privileges" in e.message: + if "Not enough privileges" in str(e): pass else: raise diff --git a/tests/test_decimal_fields.py b/tests/test_decimal_fields.py index 245360f..0480609 100644 --- a/tests/test_decimal_fields.py +++ b/tests/test_decimal_fields.py @@ -15,7 +15,7 @@ def setUp(self): self.database.create_table(DecimalModel) except ServerError as e: # This ClickHouse version does not support decimals yet - raise unittest.SkipTest(e.message) + raise unittest.SkipTest(str(e)) def tearDown(self): self.database.drop_database() @@ -78,11 +78,11 @@ def test_precision_and_scale(self): # Go over all valid combinations for precision in range(1, 39): for scale in range(0, precision + 1): - f = DecimalField(precision, scale) + DecimalField(precision, scale) # Some invalid combinations for precision, scale in [(0, 0), (-1, 7), (7, -1), (39, 5), (20, 21)]: with self.assertRaises(AssertionError): - f = DecimalField(precision, scale) + DecimalField(precision, scale) def test_min_max(self): # In range diff --git a/tests/test_funcs.py b/tests/test_funcs.py index 2d46550..3a1e07e 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -1,6 +1,6 @@ import logging import unittest -from datetime import date, datetime, timedelta, tzinfo +from datetime import date, datetime, timedelta from decimal import Decimal from ipaddress import IPv4Address, IPv6Address @@ -39,8 +39,8 @@ def _test_func(self, func, expected_value=NO_VALUE): self.assertEqual(result[0].value, expected_value) return result[0].value if result else None except ServerError as e: - if "Unknown function" in e.message: - logging.warning(e.message) + if "Unknown function" in str(e): + logging.warning(str(e)) return # ignore functions that don't exist in the used ClickHouse version raise @@ -54,8 +54,8 @@ def _test_aggr(self, func, expected_value=NO_VALUE): self.assertEqual(result[0].value, expected_value) return result[0].value if result else None except ServerError as e: - if "Unknown function" in e.message: - logging.warning(e.message) + if "Unknown function" in str(e): + logging.warning(str(e)) return # ignore functions that don't exist in the used ClickHouse version raise @@ -449,7 +449,7 @@ def test_base64_functions(self): self._test_func(F.tryBase64Decode(":-)")) except ServerError as e: # ClickHouse version that doesn't support these functions - raise unittest.SkipTest(e.message) + raise unittest.SkipTest(str(e)) def test_replace_functions(self): haystack = "hello" diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index 622167f..a326164 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -1,8 +1,5 @@ -import datetime import unittest -import pytz - from clickhouse_orm.database import Database from clickhouse_orm.engines import * from clickhouse_orm.fields import * diff --git a/tests/test_ip_fields.py b/tests/test_ip_fields.py index f5db64e..9b1c9cc 100644 --- a/tests/test_ip_fields.py +++ b/tests/test_ip_fields.py @@ -17,6 +17,7 @@ def tearDown(self): def test_ipv4_field(self): if self.database.server_version < (19, 17): raise unittest.SkipTest("ClickHouse version too old") + # Create a model class TestModel(Model): i = Int16Field() @@ -39,6 +40,7 @@ class TestModel(Model): def test_ipv6_field(self): if self.database.server_version < (19, 17): raise unittest.SkipTest("ClickHouse version too old") + # Create a model class TestModel(Model): i = Int16Field() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index fd226a1..547a803 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,6 +1,6 @@ -import os - # Add tests to path so that migrations will be importable +import logging +import os import sys import unittest from enum import Enum @@ -13,9 +13,6 @@ sys.path.append(os.path.dirname(__file__)) - -import logging - logging.basicConfig(level=logging.DEBUG, format="%(message)s") logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/test_models.py b/tests/test_models.py index a6a4917..ef78b96 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -30,7 +30,7 @@ def test_assignment(self): float_field=3.14, ) instance = SimpleModel(**kwargs) - for name, value in kwargs.items(): + for name in kwargs: self.assertEqual(kwargs[name], getattr(instance, name)) def test_assignment_error(self): diff --git a/tests/test_querysets.py b/tests/test_querysets.py index d021029..9619b34 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unittest from datetime import date, datetime -from decimal import Decimal from enum import Enum from logging import getLogger diff --git a/tests/test_readonly.py b/tests/test_readonly.py index 9d3174a..98d9c9e 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -25,9 +25,9 @@ def _test_readonly_db(self, username): self.database.drop_database() self._check_db_readonly_err(cm.exception, drop_table=True) except ServerError as e: - if e.code == 192 and e.message.startswith("Unknown user"): # ClickHouse version < 20.3 + if e.code == 192 and str(e).startswith("Unknown user"): # ClickHouse version < 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) - elif e.code == 516 and e.message.startswith( + elif e.code == 516 and str(e).startswith( "readonly: Authentication failed" ): # ClickHouse version >= 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) @@ -66,7 +66,7 @@ def test_drop_readonly_table(self): def test_nonexisting_readonly_database(self): with self.assertRaises(DatabaseException) as cm: - db = Database("dummy", readonly=True) + Database("dummy", readonly=True) self.assertEqual(str(cm.exception), "Database does not exist, and cannot be created under readonly connection") diff --git a/tests/test_uuid_fields.py b/tests/test_uuid_fields.py index af17738..9c5c11b 100644 --- a/tests/test_uuid_fields.py +++ b/tests/test_uuid_fields.py @@ -17,6 +17,7 @@ def tearDown(self): def test_uuid_field(self): if self.database.server_version < (18, 1): raise unittest.SkipTest("ClickHouse version too old") + # Create a model class TestModel(Model): i = Int16Field() From 44845176161198adfe5fb77374fbf6004a375d3b Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 00:28:17 +0100 Subject: [PATCH 17/51] Hack: add setup.cfg to cover for 'import *' --- tests/setup.cfg | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/setup.cfg diff --git a/tests/setup.cfg b/tests/setup.cfg new file mode 100644 index 0000000..a806d48 --- /dev/null +++ b/tests/setup.cfg @@ -0,0 +1,19 @@ +[flake8] +max-line-length = 120 +select = + # pycodestyle + E, W + # pyflakes + F + # flake8-bugbear + B, B9 + # pydocstyle + D + # isort + I +ignore = + E203 # Whitespace after ':' + W503 # Operator after new line + B950 # We use E501 + B008 # Using callable in function defintion, required for FastAPI + F405, F403 # Since * imports cause havok From c5d16a3ca956ab717ff1b25ecb51580163389ed6 Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 09:24:24 +0100 Subject: [PATCH 18/51] Bugfix: allow Q to be subclassed --- clickhouse_orm/query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse_orm/query.py b/clickhouse_orm/query.py index 885feb1..7287350 100644 --- a/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -221,7 +221,7 @@ def _construct_from(cls, l_child, r_child, mode): q._children.append(deepcopy(l_child)) else: # Different modes - q = Q() + q = cls() q._children = [l_child, r_child] q._mode = mode # AND/OR @@ -259,10 +259,10 @@ def to_sql(self, model_cls): return sql def __or__(self, other): - return Q._construct_from(self, other, self.OR_MODE) + return self.__class__._construct_from(self, other, self.OR_MODE) def __and__(self, other): - return Q._construct_from(self, other, self.AND_MODE) + return self.__class__._construct_from(self, other, self.AND_MODE) def __invert__(self): q = copy(self) @@ -273,7 +273,7 @@ def __bool__(self): return not self.is_empty def __deepcopy__(self, memo): - q = Q() + q = self.__class__() q._conds = [deepcopy(cond) for cond in self._conds] q._negate = self._negate q._mode = self._mode From 4a90eede16515c6764b9c786a10962c174458c68 Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 13:52:42 +0100 Subject: [PATCH 19/51] Chore: fix versioning --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 397130e..84509d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ line_length = 120 [tool.poetry] name = "clickhouse_orm" -version = "0.2.1" +version = "2.1.0" description = "A simple ORM for working with the Clickhouse database" authors = ["olliemath "] license = "BSD" From bd62a3c1de5177ff8674f2fdd76ee44a21f6780e Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 21:39:49 +0100 Subject: [PATCH 20/51] Enhancement: remove \N warning for utils Also improves code quality and implementation details. --- clickhouse_orm/engines.py | 8 +++--- clickhouse_orm/fields.py | 8 +++--- clickhouse_orm/funcs.py | 2 +- clickhouse_orm/utils.py | 48 +++++++++++++++--------------------- tests/__init__.py | 1 - tests/base_test_with_data.py | 4 +-- tests/test_utils.py | 29 ++++++++++++++++++++++ 7 files changed, 60 insertions(+), 40 deletions(-) create mode 100644 tests/test_utils.py diff --git a/clickhouse_orm/engines.py b/clickhouse_orm/engines.py index 8dfda54..af63af7 100644 --- a/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -86,12 +86,12 @@ def create_table_sql(self, db): # Let's check version and use new syntax if available if db.server_version >= (1, 1, 54310): partition_sql = "PARTITION BY (%s) ORDER BY (%s)" % ( - comma_join(self.partition_key, stringify=True), - comma_join(self.order_by, stringify=True), + comma_join(map(str, self.partition_key)), + comma_join(map(str, self.order_by)), ) if self.primary_key: - partition_sql += " PRIMARY KEY (%s)" % comma_join(self.primary_key, stringify=True) + partition_sql += " PRIMARY KEY (%s)" % comma_join(map(str, self.primary_key)) if self.sampling_expr: partition_sql += " SAMPLE BY %s" % self.sampling_expr @@ -126,7 +126,7 @@ def _build_sql_params(self, db): params.append(self.date_col) if self.sampling_expr: params.append(self.sampling_expr) - params.append("(%s)" % comma_join(self.order_by, stringify=True)) + params.append("(%s)" % comma_join(map(str(self.order_by)))) params.append(str(self.index_granularity)) return params diff --git a/clickhouse_orm/fields.py b/clickhouse_orm/fields.py index 31c36eb..8ead5a2 100644 --- a/clickhouse_orm/fields.py +++ b/clickhouse_orm/fields.py @@ -147,7 +147,7 @@ def to_python(self, value, timezone_in_use): if isinstance(value, str): return value if isinstance(value, bytes): - return value.decode("UTF-8") + return value.decode("utf-8") raise ValueError("Invalid value for %s: %r" % (self.__class__.__name__, value)) @@ -163,7 +163,7 @@ def to_python(self, value, timezone_in_use): def validate(self, value): if isinstance(value, str): - value = value.encode("UTF-8") + value = value.encode("utf-8") if len(value) > self._length: raise ValueError("Value of %d bytes is too long for FixedStringField(%d)" % (len(value), self._length)) @@ -475,7 +475,7 @@ def to_python(self, value, timezone_in_use): except Exception: return self.enum_cls(value) if isinstance(value, bytes): - decoded = value.decode("UTF-8") + decoded = value.decode("utf-8") try: return self.enum_cls[decoded] except Exception: @@ -533,7 +533,7 @@ def to_python(self, value, timezone_in_use): if isinstance(value, str): value = parse_array(value) elif isinstance(value, bytes): - value = parse_array(value.decode("UTF-8")) + value = parse_array(value.decode("utf-8")) elif not isinstance(value, (list, tuple)): raise ValueError("ArrayField expects list or tuple, not %s" % type(value)) return [self.inner_field.to_python(v, timezone_in_use) for v in value] diff --git a/clickhouse_orm/funcs.py b/clickhouse_orm/funcs.py index 52be002..4293c9b 100644 --- a/clickhouse_orm/funcs.py +++ b/clickhouse_orm/funcs.py @@ -76,7 +76,7 @@ def wrapper(*parameters): def inner(*args, **kwargs): f = func(*args, **kwargs) # Append the parameter to the function name - parameters_str = comma_join(parameters, stringify=True) + parameters_str = comma_join(map(str, parameters)) f.name = "%s(%s)" % (f.name, parameters_str) return f diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index 8be51f3..c4526f4 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -3,30 +3,27 @@ import pkgutil import re from datetime import date, datetime, timedelta, tzinfo +from inspect import isclass +from types import ModuleType +from typing import Any, Dict, Iterable, List, Optional, Type, Union -SPECIAL_CHARS = {"\b": "\\b", "\f": "\\f", "\r": "\\r", "\n": "\\n", "\t": "\\t", "\0": "\\0", "\\": "\\\\", "'": "\\'"} -SPECIAL_CHARS_REGEX = re.compile("[" + "".join(SPECIAL_CHARS.values()) + "]") - - -def escape(value, quote=True): +def escape(value: str, quote: bool = True) -> str: """ If the value is a string, escapes any special characters and optionally surrounds it with single quotes. If the value is not a string (e.g. a number), converts it to one. """ + value = codecs.escape_encode(value.encode("utf-8"))[0].decode("utf-8") + if quote: + value = "'" + value + "'" - def escape_one(match): - return SPECIAL_CHARS[match.group(0)] + return value - if isinstance(value, str): - value = SPECIAL_CHARS_REGEX.sub(escape_one, value) - if quote: - value = "'" + value + "'" - return str(value) - -def unescape(value): +def unescape(value: str) -> Optional[str]: + if value == "\\N": + return None return codecs.escape_decode(value)[0].decode("utf-8") @@ -34,7 +31,7 @@ def string_or_func(obj): return obj.to_sql() if hasattr(obj, "to_sql") else obj -def arg_to_sql(arg): +def arg_to_sql(arg: Any) -> str: """ Converts a function argument to SQL string according to its type. Supports functions, model fields, strings, dates, datetimes, timedeltas, booleans, @@ -69,15 +66,15 @@ def arg_to_sql(arg): return str(arg) -def parse_tsv(line): +def parse_tsv(line: Union[bytes, str]) -> List[str]: if isinstance(line, bytes): line = line.decode() if line and line[-1] == "\n": line = line[:-1] - return [unescape(value) for value in line.split(str("\t"))] + return [unescape(value) for value in line.split("\t")] -def parse_array(array_string): +def parse_array(array_string: str) -> List[Any]: """ Parse an array or tuple string as returned by clickhouse. For example: "['hello', 'world']" ==> ["hello", "world"] @@ -111,7 +108,7 @@ def parse_array(array_string): array_string = array_string[match.end() - 1 :] -def import_submodules(package_name): +def import_submodules(package_name: str) -> Dict[str, ModuleType]: """ Import all submodules of a module. """ @@ -122,17 +119,14 @@ def import_submodules(package_name): } -def comma_join(items, stringify=False): +def comma_join(items: Iterable[str]) -> str: """ Joins an iterable of strings with commas. """ - if stringify: - return ", ".join(str(item) for item in items) - else: - return ", ".join(items) + return ", ".join(items) -def is_iterable(obj): +def is_iterable(obj: Any) -> bool: """ Checks if the given object is iterable. """ @@ -143,9 +137,7 @@ def is_iterable(obj): return False -def get_subclass_names(locals, base_class): - from inspect import isclass - +def get_subclass_names(locals: Dict[str, Any], base_class: Type): return [c.__name__ for c in locals.values() if isclass(c) and issubclass(c, base_class)] diff --git a/tests/__init__.py b/tests/__init__.py index 5284146..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -__import__("pkg_resources").declare_namespace(__name__) diff --git a/tests/base_test_with_data.py b/tests/base_test_with_data.py index 17da512..a026f5a 100644 --- a/tests/base_test_with_data.py +++ b/tests/base_test_with_data.py @@ -3,8 +3,8 @@ import unittest from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, Float32Field, LowCardinalityField, NullableField, StringField, UInt32Field from clickhouse_orm.models import Model logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..74722f8 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from clickhouse_orm.utils import escape, unescape + +SPECIAL_CHARS = {"\b": "\\x08", "\f": "\\x0c", "\r": "\\r", "\n": "\\n", "\t": "\\t", "\0": "\\x00", "\\": "\\\\", "'": "\\'"} + + +def test_unescape(): + + for input_, expected in ( + ("π\\t", "π\t"), + ("\\new", "\new"), + ("cheeky 🐵", "cheeky 🐵"), + ("\\N", None), + ): + assert unescape(input_) == expected + + +def test_escape_special_chars(): + + initial = "".join(SPECIAL_CHARS.keys()) + expected = "".join(SPECIAL_CHARS.values()) + assert escape(initial, quote=False) == expected + assert escape(initial) == "'" + expected + "'" + + +def test_escape_unescape_parity(): + + for initial in ("π\t", "\new", "cheeky 🐵", "back \\ slash", "\\\\n"): + assert unescape(escape(initial, quote=False)) == initial From c4e2175db3d1aeca13f6b19ec86ac86530994bae Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 21:41:58 +0100 Subject: [PATCH 21/51] Chore: improve code quality for unit tests --- pyproject.toml | 1 + tests/test_funcs.py | 25 ++++++++++++++++--------- tests/test_querysets.py | 5 ++++- tests/test_readonly.py | 4 +--- tests/test_system_models.py | 14 +++++++------- tests/test_utils.py | 11 ++++++++++- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 84509d0..9ca6199 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ pytest = "^6.2.4" flake8-isort = "^4.0.0" black = "^21.7b0" isort = "^5.9.2" +freezegun = "^1.1.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/test_funcs.py b/tests/test_funcs.py index 3a1e07e..81e9318 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -7,10 +7,11 @@ import pytz from clickhouse_orm.database import ServerError +from clickhouse_orm.fields import DateTimeField from clickhouse_orm.funcs import F from clickhouse_orm.utils import NO_VALUE -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData from .test_querysets import SampleModel @@ -28,15 +29,12 @@ def _test_qs(self, qs, expected_count): self.assertEqual(count, expected_count) self.assertEqual(qs.count(), expected_count) - def _test_func(self, func, expected_value=NO_VALUE): + def _call_func(self, func): sql = "SELECT %s AS value" % func.to_sql() logging.info(sql) try: result = list(self.database.select(sql)) logging.info("\t==> %s", result[0].value if result else "") - if expected_value != NO_VALUE: - print("Comparing %s to %s" % (result[0].value, expected_value)) - self.assertEqual(result[0].value, expected_value) return result[0].value if result else None except ServerError as e: if "Unknown function" in str(e): @@ -44,6 +42,14 @@ def _test_func(self, func, expected_value=NO_VALUE): return # ignore functions that don't exist in the used ClickHouse version raise + def _test_func(self, func, expected_value=NO_VALUE): + result = self._call_func(func) + if expected_value != NO_VALUE: + print("Comparing %s to %s" % (result, expected_value)) + self.assertEqual(result, expected_value) + + return result if result else None + def _test_aggr(self, func, expected_value=NO_VALUE): qs = Person.objects_in(self.database).aggregate(value=func) logging.info(qs.as_sql()) @@ -313,7 +319,7 @@ def test_date_functions(self): F.now() + F.toIntervalSecond(3000) - F.toIntervalDay(3000) == F.now() + timedelta(seconds=3000, days=-3000) ) - def test_date_functions__utc_only(self): + def test_date_functions_utc_only(self): if self.database.server_timezone != pytz.utc: raise unittest.SkipTest("This test must run with UTC as the server timezone") d = date(2018, 12, 31) @@ -325,9 +331,6 @@ def test_date_functions__utc_only(self): self._test_func(F.toTime(dt, "Europe/Athens"), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) self._test_func(F.toTime(dt, athens_tz), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) self._test_func(F.toTimeZone(dt, "Europe/Athens"), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33))) - self._test_func( - F.now(), datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0) - ) # FIXME this may fail if the timing is just right self._test_func(F.today(), datetime.utcnow().date()) self._test_func(F.yesterday(), datetime.utcnow().date() - timedelta(days=1)) self._test_func(F.toYYYYMMDDhhmmss(dt), 20181231112233) @@ -335,6 +338,10 @@ def test_date_functions__utc_only(self): self._test_func(F.addHours(d, 7), datetime(2018, 12, 31, 7, 0, 0, tzinfo=pytz.utc)) self._test_func(F.addMinutes(d, 7), datetime(2018, 12, 31, 0, 7, 0, tzinfo=pytz.utc)) + actual = self._call_func(F.now()) + expected = datetime.utcnow().replace(tzinfo=pytz.utc, microsecond=0) + self.assertLess((actual - expected).total_seconds(), 1e-3) + def test_type_conversion_functions(self): for f in ( F.toUInt8, diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 9619b34..575435e 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -5,10 +5,13 @@ from logging import getLogger from clickhouse_orm.database import Database +from clickhouse_orm.engines import CollapsingMergeTree, Memory, MergeTree +from clickhouse_orm.fields import DateField, DateTimeField, Enum8Field, Int8Field, Int32Field, UInt64Field from clickhouse_orm.funcs import F +from clickhouse_orm.models import Model from clickhouse_orm.query import Q -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData, data logger = getLogger("tests") diff --git a/tests/test_readonly.py b/tests/test_readonly.py index 98d9c9e..94083a1 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -27,9 +27,7 @@ def _test_readonly_db(self, username): except ServerError as e: if e.code == 192 and str(e).startswith("Unknown user"): # ClickHouse version < 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) - elif e.code == 516 and str(e).startswith( - "readonly: Authentication failed" - ): # ClickHouse version >= 20.3 + elif e.code == 516 and str(e).startswith("readonly: Authentication failed"): # ClickHouse version >= 20.3 raise unittest.SkipTest('Database user "%s" is not defined' % username) else: raise diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 7f7d82f..15fdad2 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -3,8 +3,8 @@ from datetime import date from clickhouse_orm.database import Database, DatabaseException -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, UInt32Field from clickhouse_orm.models import Model from clickhouse_orm.system_models import SystemPart @@ -36,9 +36,9 @@ class SystemPartTest(unittest.TestCase): def setUp(self): self.database = Database("test-db", log_statements=True) - self.database.create_table(TestTable) + self.database.create_table(SomeTestTable) self.database.create_table(CustomPartitionedTable) - self.database.insert([TestTable(date_field=date.today())]) + self.database.insert([SomeTestTable(date_field=date.today())]) self.database.insert([CustomPartitionedTable(date_field=date.today(), group_field=13)]) def tearDown(self): @@ -69,7 +69,7 @@ def test_get_active(self): self.assertEqual(len(parts), 1) def test_get_conditions(self): - parts = list(SystemPart.get(self.database, conditions="table='testtable'")) + parts = list(SystemPart.get(self.database, conditions="table='sometesttable'")) self.assertEqual(len(parts), 1) parts = list(SystemPart.get(self.database, conditions=u"table='custompartitionedtable'")) self.assertEqual(len(parts), 1) @@ -107,10 +107,10 @@ def test_fetch(self): def test_query(self): SystemPart.objects_in(self.database).count() - list(SystemPart.objects_in(self.database).filter(table="testtable")) + list(SystemPart.objects_in(self.database).filter(table="sometesttable")) -class TestTable(Model): +class SomeTestTable(Model): date_field = DateField() engine = MergeTree("date_field", ("date_field",)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 74722f8..1d92487 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,16 @@ # -*- coding: utf-8 -*- from clickhouse_orm.utils import escape, unescape -SPECIAL_CHARS = {"\b": "\\x08", "\f": "\\x0c", "\r": "\\r", "\n": "\\n", "\t": "\\t", "\0": "\\x00", "\\": "\\\\", "'": "\\'"} +SPECIAL_CHARS = { + "\b": "\\x08", + "\f": "\\x0c", + "\r": "\\r", + "\n": "\\n", + "\t": "\\t", + "\0": "\\x00", + "\\": "\\\\", + "'": "\\'", +} def test_unescape(): From bdec4843cec023d0756a8d3d9a8b15132093763a Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 22:11:27 +0100 Subject: [PATCH 22/51] Chore: update supported python versions --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ca6199..bb0cb22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,16 +20,16 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Database" ] [tool.poetry.dependencies] -python = "^3.7" +python = ">=3.6.2,<4" requests = "^2.26.0" pytz = "^2021.1" iso8601 = "^0.1.16" From a36e69822bb686bbb6ae68b758a095e0b1c9f936 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Wed, 28 Jul 2021 21:43:21 +0000 Subject: [PATCH 23/51] Pipeline: add github workflows (#1) Tooling: add github actions * Add github actions to run pytest / flake8 / black * Use clickhouse:20.6 (tests fail with 21.x, 20.8) * Keep black away from pypy --- .github/workflows/python-package.yml | 63 ++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..a89172c --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,63 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Lint on smallest and largest active versions + python-version: [3.6, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install poetry + poetry install + - name: Lint with flake8 + run: | + poetry run flake8 clickhouse_orm/ + cd tests && poetry run flake8 + - name: Check formatting with black + run: | + poetry run black --line-length 120 --check clickhouse_orm/ tests/ + + test: + runs-on: ubuntu-latest + services: + clickhouse: + image: yandex/clickhouse-server:20.6 + ports: + - 8123:8123 + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9, pypy3] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install poetry + poetry install + - name: Test with pytest + run: | + poetry run pytest diff --git a/pyproject.toml b/pyproject.toml index bb0cb22..d500cd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ flake8-bugbear = "^21.4.3" pep8-naming = "^0.12.0" pytest = "^6.2.4" flake8-isort = "^4.0.0" -black = "^21.7b0" +black = {version = "^21.7b0", markers = "platform_python_implementation == 'CPython'"} isort = "^5.9.2" freezegun = "^1.1.0" From 73b74e9cfbc8df0ffe9bedd54eb990afbfe0d57c Mon Sep 17 00:00:00 2001 From: olliemath Date: Wed, 28 Jul 2021 22:49:52 +0100 Subject: [PATCH 24/51] Chore: license tweak --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index ff7bfb4..df334ae 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ +Copyright (c) 2021 Suade Labs Copyright (c) 2017 INFINIDAT Redistribution and use in source and binary forms, with or without From ff580d73d8e65e86f6f5529f4028424273bc096b Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 30 Jul 2021 09:33:28 +0100 Subject: [PATCH 25/51] Chore: organize tests --- tests/fields/__init__.py | 0 tests/{ => fields}/test_alias_fields.py | 6 +++--- tests/{ => fields}/test_array_fields.py | 0 tests/{ => fields}/test_compressed_fields.py | 0 tests/{ => fields}/test_custom_fields.py | 0 tests/{ => fields}/test_datetime_fields.py | 0 tests/{ => fields}/test_decimal_fields.py | 0 tests/{ => fields}/test_enum_fields.py | 0 tests/{ => fields}/test_fixed_string_fields.py | 0 tests/{ => fields}/test_ip_fields.py | 0 tests/{ => fields}/test_materialized_fields.py | 0 tests/{ => fields}/test_nullable_fields.py | 0 tests/{ => fields}/test_simple_fields.py | 0 tests/{ => fields}/test_uuid_fields.py | 0 14 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 tests/fields/__init__.py rename tests/{ => fields}/test_alias_fields.py (94%) rename tests/{ => fields}/test_array_fields.py (100%) rename tests/{ => fields}/test_compressed_fields.py (100%) rename tests/{ => fields}/test_custom_fields.py (100%) rename tests/{ => fields}/test_datetime_fields.py (100%) rename tests/{ => fields}/test_decimal_fields.py (100%) rename tests/{ => fields}/test_enum_fields.py (100%) rename tests/{ => fields}/test_fixed_string_fields.py (100%) rename tests/{ => fields}/test_ip_fields.py (100%) rename tests/{ => fields}/test_materialized_fields.py (100%) rename tests/{ => fields}/test_nullable_fields.py (100%) rename tests/{ => fields}/test_simple_fields.py (100%) rename tests/{ => fields}/test_uuid_fields.py (100%) diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_alias_fields.py b/tests/fields/test_alias_fields.py similarity index 94% rename from tests/test_alias_fields.py rename to tests/fields/test_alias_fields.py index 5b77513..4f800db 100644 --- a/tests/test_alias_fields.py +++ b/tests/fields/test_alias_fields.py @@ -2,8 +2,8 @@ from datetime import date from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, Int32Field, StringField from clickhouse_orm.funcs import F from clickhouse_orm.models import NO_VALUE, Model @@ -67,7 +67,7 @@ class ModelWithAliasFields(Model): date_field = DateField() str_field = StringField() - alias_str = StringField(alias=u"str_field") + alias_str = StringField(alias="str_field") alias_int = Int32Field(alias="int_field") alias_date = DateField(alias="date_field") alias_func = Int32Field(alias=F.toYYYYMM(date_field)) diff --git a/tests/test_array_fields.py b/tests/fields/test_array_fields.py similarity index 100% rename from tests/test_array_fields.py rename to tests/fields/test_array_fields.py diff --git a/tests/test_compressed_fields.py b/tests/fields/test_compressed_fields.py similarity index 100% rename from tests/test_compressed_fields.py rename to tests/fields/test_compressed_fields.py diff --git a/tests/test_custom_fields.py b/tests/fields/test_custom_fields.py similarity index 100% rename from tests/test_custom_fields.py rename to tests/fields/test_custom_fields.py diff --git a/tests/test_datetime_fields.py b/tests/fields/test_datetime_fields.py similarity index 100% rename from tests/test_datetime_fields.py rename to tests/fields/test_datetime_fields.py diff --git a/tests/test_decimal_fields.py b/tests/fields/test_decimal_fields.py similarity index 100% rename from tests/test_decimal_fields.py rename to tests/fields/test_decimal_fields.py diff --git a/tests/test_enum_fields.py b/tests/fields/test_enum_fields.py similarity index 100% rename from tests/test_enum_fields.py rename to tests/fields/test_enum_fields.py diff --git a/tests/test_fixed_string_fields.py b/tests/fields/test_fixed_string_fields.py similarity index 100% rename from tests/test_fixed_string_fields.py rename to tests/fields/test_fixed_string_fields.py diff --git a/tests/test_ip_fields.py b/tests/fields/test_ip_fields.py similarity index 100% rename from tests/test_ip_fields.py rename to tests/fields/test_ip_fields.py diff --git a/tests/test_materialized_fields.py b/tests/fields/test_materialized_fields.py similarity index 100% rename from tests/test_materialized_fields.py rename to tests/fields/test_materialized_fields.py diff --git a/tests/test_nullable_fields.py b/tests/fields/test_nullable_fields.py similarity index 100% rename from tests/test_nullable_fields.py rename to tests/fields/test_nullable_fields.py diff --git a/tests/test_simple_fields.py b/tests/fields/test_simple_fields.py similarity index 100% rename from tests/test_simple_fields.py rename to tests/fields/test_simple_fields.py diff --git a/tests/test_uuid_fields.py b/tests/fields/test_uuid_fields.py similarity index 100% rename from tests/test_uuid_fields.py rename to tests/fields/test_uuid_fields.py From 9b671d66d8b01cc08021219c5b344ac58615a7f3 Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 30 Jul 2021 09:36:57 +0100 Subject: [PATCH 26/51] Bugfix: correct boolean logic for Q objects Closes #4 --- clickhouse_orm/query.py | 11 ++++------- tests/test_querysets.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/clickhouse_orm/query.py b/clickhouse_orm/query.py index 7287350..e0ab42e 100644 --- a/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -209,21 +209,18 @@ def is_empty(self): Checks if there are any conditions in Q object Returns: Boolean """ - return not bool(self._conds or self._children) + return not (self._conds or self._children) @classmethod def _construct_from(cls, l_child, r_child, mode): - if mode == l_child._mode: + if mode == l_child._mode and not l_child._negate: q = deepcopy(l_child) q._children.append(deepcopy(r_child)) - elif mode == r_child._mode: - q = deepcopy(r_child) - q._children.append(deepcopy(l_child)) + else: - # Different modes q = cls() q._children = [l_child, r_child] - q._mode = mode # AND/OR + q._mode = mode return q diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 575435e..10ec863 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -6,7 +6,7 @@ from clickhouse_orm.database import Database from clickhouse_orm.engines import CollapsingMergeTree, Memory, MergeTree -from clickhouse_orm.fields import DateField, DateTimeField, Enum8Field, Int8Field, Int32Field, UInt64Field +from clickhouse_orm.fields import DateField, DateTimeField, Enum8Field, Int8Field, Int32Field, StringField, UInt64Field from clickhouse_orm.funcs import F from clickhouse_orm.models import Model from clickhouse_orm.query import Q @@ -568,10 +568,22 @@ def test_limit_by(self): limited_qs = qs.limit_by((6, 3), "height") self.assertEqual([p.first_name for p in limited_qs[:3]], ["Norman", "Octavius", "Oliver"]) + def test_boolean_logic(self): + p = ~Q(x="eggs") + q = Q(y="spam") + r = p & q + + self.assertEqual(r.to_sql(StringyModel), "(NOT (x = 'eggs')) AND (y = 'spam')") + Color = Enum("Color", u"red blue green yellow brown white black") +class StringyModel(Model): + x = StringField() + y = StringField() + + class SampleModel(Model): timestamp = DateTimeField() From b6d3012a4a0dacd06c03885b012311d5b094693e Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 6 Aug 2021 17:09:25 +0100 Subject: [PATCH 27/51] Bugfix: remove compression from alias fields This is not allowed in Clickhouse 20.8+ and would have no effect earlier versions. --- clickhouse_orm/fields.py | 2 + tests/fields/test_compressed_fields.py | 55 +++++++++++++++++++++----- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/clickhouse_orm/fields.py b/clickhouse_orm/fields.py index 8ead5a2..b5ea5b6 100644 --- a/clickhouse_orm/fields.py +++ b/clickhouse_orm/fields.py @@ -38,6 +38,8 @@ def __init__(self, default=None, alias=None, materialized=None, readonly=None, c ), "Materialized parameter must be a string or function object, if given" assert readonly is None or type(readonly) is bool, "readonly parameter must be bool if given" assert codec is None or isinstance(codec, str) and codec != "", "Codec field must be string, if given" + if alias: + assert codec is None, "Codec cannot be used for alias fields" self.creation_counter = Field.creation_counter Field.creation_counter += 1 diff --git a/tests/fields/test_compressed_fields.py b/tests/fields/test_compressed_fields.py index 8387f28..e44e15a 100644 --- a/tests/fields/test_compressed_fields.py +++ b/tests/fields/test_compressed_fields.py @@ -1,11 +1,22 @@ import datetime import unittest +import pytest import pytz from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import ( + ArrayField, + DateField, + DateTimeField, + FixedStringField, + Float32Field, + Int64Field, + NullableField, + StringField, + UInt64Field, +) from clickhouse_orm.models import NO_VALUE, Model from clickhouse_orm.utils import parse_tsv @@ -50,7 +61,11 @@ def test_assignment(self): def test_string_conversion(self): # Check field conversion from string during construction instance = CompressedModel( - date_field="1973-12-06", int64_field="100", float_field="7", nullable_field=None, array_field="[a,b,c]" + date_field="1973-12-06", + int64_field="100", + float_field="7", + nullable_field=None, + array_field="[a,b,c]", ) self.assertEqual(instance.date_field, datetime.date(1973, 12, 6)) self.assertEqual(instance.int64_field, 100) @@ -62,7 +77,12 @@ def test_string_conversion(self): self.assertEqual(instance.int64_field, 99) def test_to_dict(self): - instance = CompressedModel(date_field="1973-12-06", int64_field="100", float_field="7", array_field="[a,b,c]") + instance = CompressedModel( + date_field="1973-12-06", + int64_field="100", + float_field="7", + array_field="[a,b,c]", + ) self.assertDictEqual( instance.to_dict(), { @@ -70,7 +90,7 @@ def test_to_dict(self): "int64_field": 100, "float_field": 7.0, "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), - "alias_field": NO_VALUE, + "mat_field": NO_VALUE, "string_field": "dozo", "nullable_field": None, "uint64_field": 0, @@ -91,14 +111,25 @@ def test_to_dict(self): }, ) self.assertDictEqual( - instance.to_dict(include_readonly=False, field_names=("int64_field", "alias_field", "datetime_field")), - {"int64_field": 100, "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)}, + instance.to_dict( + include_readonly=False, + field_names=("int64_field", "mat_field", "datetime_field"), + ), + { + "int64_field": 100, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + }, ) def test_confirm_compression_codec(self): if self.database.server_version < (19, 17): raise unittest.SkipTest("ClickHouse version too old") - instance = CompressedModel(date_field="1973-12-06", int64_field="100", float_field="7", array_field="[a,b,c]") + instance = CompressedModel( + date_field="1973-12-06", + int64_field="100", + float_field="7", + array_field="[a,b,c]", + ) self.database.insert([instance]) r = self.database.raw( "select name, compression_codec from system.columns where table = '{}' and database='{}' FORMAT TabSeparatedWithNamesAndTypes".format( @@ -120,10 +151,14 @@ def test_confirm_compression_codec(self): ("nullable_field", "CODEC(ZSTD(1))"), ("array_field", "CODEC(Delta(2), LZ4HC(0))"), ("float_field", "CODEC(NONE)"), - ("alias_field", "CODEC(ZSTD(4))"), + ("mat_field", "CODEC(ZSTD(4))"), ], ) + def test_alias_field(self): + with pytest.raises(AssertionError): + Float32Field(alias="something", codec="ZSTD(4)") + class CompressedModel(Model): uint64_field = UInt64Field(codec="ZSTD(10)") @@ -134,6 +169,6 @@ class CompressedModel(Model): nullable_field = NullableField(Float32Field(), codec="ZSTD") array_field = ArrayField(FixedStringField(length=15), codec="Delta(2),LZ4HC") float_field = Float32Field(codec="NONE") - alias_field = Float32Field(alias="float_field", codec="ZSTD(4)") + mat_field = Float32Field(materialized="float_field", codec="ZSTD(4)") engine = MergeTree("datetime_field", ("uint64_field", "datetime_field")) From 078ffbfc9038378c851a93974147b1bb55a7cef6 Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 6 Aug 2021 17:28:22 +0100 Subject: [PATCH 28/51] Chore: remove some circular imports --- clickhouse_orm/database.py | 6 +----- clickhouse_orm/query.py | 25 +++++++++++++------------ clickhouse_orm/utils.py | 5 +++++ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/clickhouse_orm/database.py b/clickhouse_orm/database.py index ec9407c..3e1cf39 100644 --- a/clickhouse_orm/database.py +++ b/clickhouse_orm/database.py @@ -1,7 +1,6 @@ import datetime import logging import re -from collections import namedtuple from math import ceil from string import Template @@ -9,14 +8,11 @@ import requests from .models import ModelBase -from .utils import import_submodules, parse_tsv +from .utils import Page, import_submodules, parse_tsv logger = logging.getLogger("clickhouse_orm") -Page = namedtuple("Page", "objects number_of_objects pages_total number page_size") - - class DatabaseException(Exception): """ Raised when a database operation fails. diff --git a/clickhouse_orm/query.py b/clickhouse_orm/query.py index e0ab42e..c31e6be 100644 --- a/clickhouse_orm/query.py +++ b/clickhouse_orm/query.py @@ -3,7 +3,8 @@ import pytz -from .utils import arg_to_sql, comma_join, string_or_func +from .engines import CollapsingMergeTree, ReplacingMergeTree +from .utils import Page, arg_to_sql, comma_join, string_or_func # TODO # - check that field names are valid @@ -22,10 +23,10 @@ def to_sql(self, model_cls, field_name, value): raise NotImplementedError # pragma: no cover def _value_to_sql(self, field, value, quote=True): - from clickhouse_orm.funcs import F - - if isinstance(value, F): + if isinstance(value, Cond): + # This is an 'in-database' value, rather than a python one return value.to_sql() + return field.to_db_string(field.to_python(value, pytz.utc), quote) @@ -256,9 +257,15 @@ def to_sql(self, model_cls): return sql def __or__(self, other): + if not isinstance(other, Q): + return NotImplemented + return self.__class__._construct_from(self, other, self.OR_MODE) def __and__(self, other): + if not isinstance(other, Q): + return NotImplemented + return self.__class__._construct_from(self, other, self.AND_MODE) def __invert__(self): @@ -456,8 +463,6 @@ def only(self, *field_names): return qs def _filter_or_exclude(self, *q, **kwargs): - from .funcs import F - inverse = kwargs.pop("_inverse", False) prewhere = kwargs.pop("prewhere", False) @@ -467,10 +472,10 @@ def _filter_or_exclude(self, *q, **kwargs): for arg in q: if isinstance(arg, Q): condition &= arg - elif isinstance(arg, F): + elif isinstance(arg, Cond): condition &= Q(arg) else: - raise TypeError('Invalid argument "%r" to queryset filter' % arg) + raise TypeError(f"Invalid argument '{arg}' of type '{type(arg)}' to filter") if kwargs: condition &= Q(**kwargs) @@ -512,8 +517,6 @@ def paginate(self, page_num=1, page_size=100): The result is a namedtuple containing `objects` (list), `number_of_objects`, `pages_total`, `number` (of the current page), and `page_size`. """ - from .database import Page - count = self.count() pages_total = int(ceil(count / float(page_size))) if page_num == -1: @@ -543,8 +546,6 @@ def final(self): Adds a FINAL modifier to table, meaning data will be collapsed to final version. Can be used with the `CollapsingMergeTree` and `ReplacingMergeTree` engines only. """ - from .engines import CollapsingMergeTree, ReplacingMergeTree - if not isinstance(self._model_cls.engine, (CollapsingMergeTree, ReplacingMergeTree)): raise TypeError( "final() method can be used only with the CollapsingMergeTree and ReplacingMergeTree engines" diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index c4526f4..92c7c99 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -2,12 +2,17 @@ import importlib import pkgutil import re +from collections import namedtuple from datetime import date, datetime, timedelta, tzinfo from inspect import isclass from types import ModuleType from typing import Any, Dict, Iterable, List, Optional, Type, Union +Page = namedtuple("Page", "objects number_of_objects pages_total number page_size") +Page.__doc__ += "\nA simple data structure for paginated results." + + def escape(value: str, quote: bool = True) -> str: """ If the value is a string, escapes any special characters and optionally From 8f497ea3572b4f3b45e199842677dcb16da5d94d Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 6 Aug 2021 16:32:30 +0000 Subject: [PATCH 29/51] Chore: bump pipeline clickhouse version to 20.8 --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a89172c..e9311fa 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Python package +name: Test python package on: push: @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest services: clickhouse: - image: yandex/clickhouse-server:20.6 + image: yandex/clickhouse-server:20.8 ports: - 8123:8123 strategy: From f6c184ed547b8f0e46a5b234ec2651e643f6ea98 Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 6 Aug 2021 17:33:12 +0100 Subject: [PATCH 30/51] Chore: isort --- clickhouse_orm/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index 92c7c99..ef79c27 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -8,7 +8,6 @@ from types import ModuleType from typing import Any, Dict, Iterable, List, Optional, Type, Union - Page = namedtuple("Page", "objects number_of_objects pages_total number page_size") Page.__doc__ += "\nA simple data structure for paginated results." From caffb937173d7337d7ef820f41389b6f8f402876 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 6 Aug 2021 18:41:17 +0000 Subject: [PATCH 31/51] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 247d928..972859e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) aimed at more frequent maintenance and bugfixes. + +[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-package.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-package.yml) + Introduction ============ From a00e0c2002b84b10acde49418ab9189627136a28 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 6 Aug 2021 18:41:52 +0000 Subject: [PATCH 32/51] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e9311fa..45fef0b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Test python package +name: Tests on: push: From 3cc6eafb94219646c04876a4785721d99462082e Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 6 Aug 2021 18:57:26 +0000 Subject: [PATCH 33/51] Prepare 2.2.0 release --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3286b1..2c703d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ Change Log ========== +v2.2.0 +------ +- Support up to clickhouse 20.8 +- Fixed boolean logic for Q objects (https://github.com/Infinidat/infi.clickhouse_orm/issues/158) +- Remove implicit use of '\N' character (which was causing deprecation warnings in queries) +- Tooling updates: use poetry, pytest, isort, black + +**Backwards incompatible changes** + +You can no longer supply a codec for an `alias` field. Previously this had no effect in clickhouse, but now it explicitly returns an error. + v2.1.0 ------ - Support for model constraints From 84302aa1720d0f4eaacc1071f4e6d0cc50251d41 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Fri, 6 Aug 2021 18:57:52 +0000 Subject: [PATCH 34/51] Prepare 2.2.0 release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d500cd5..ea165f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ line_length = 120 [tool.poetry] name = "clickhouse_orm" -version = "2.1.0" +version = "2.2.0" description = "A simple ORM for working with the Clickhouse database" authors = ["olliemath "] license = "BSD" From 1054502d5d9e07c9d1cfaa48ab1b7ec8ae1c225e Mon Sep 17 00:00:00 2001 From: olliemath Date: Sun, 8 Aug 2021 10:04:18 +0100 Subject: [PATCH 35/51] Bugfix: fix NO_VALUE copy semantics Previously copying the NO_VALUE singleton would yield a different instance. This caused some problems. Now it really is a singleton. --- clickhouse_orm/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clickhouse_orm/utils.py b/clickhouse_orm/utils.py index ef79c27..d734914 100644 --- a/clickhouse_orm/utils.py +++ b/clickhouse_orm/utils.py @@ -154,5 +154,11 @@ class NoValue: def __repr__(self): return "NO_VALUE" + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + NO_VALUE = NoValue() From 45e16fe2f84b41b34abd4421f26679a00e7deeaf Mon Sep 17 00:00:00 2001 From: olliemath Date: Sun, 8 Aug 2021 10:05:35 +0100 Subject: [PATCH 36/51] Enhancement: add timezones to all date functions --- clickhouse_orm/funcs.py | 124 ++++++++++++++++++++-------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/clickhouse_orm/funcs.py b/clickhouse_orm/funcs.py index 4293c9b..ce1492f 100644 --- a/clickhouse_orm/funcs.py +++ b/clickhouse_orm/funcs.py @@ -400,103 +400,103 @@ def _notIn(a, b): # Functions for working with dates and times @staticmethod - def toYear(d): - return F("toYear", d) + def toYear(d, timezone=NO_VALUE): + return F("toYear", d, timezone) @staticmethod - def toISOYear(d, timezone=""): + def toISOYear(d, timezone=NO_VALUE): return F("toISOYear", d, timezone) @staticmethod - def toQuarter(d, timezone=""): - return F("toQuarter", d, timezone) if timezone else F("toQuarter", d) + def toQuarter(d, timezone=NO_VALUE): + return F("toQuarter", d, timezone) @staticmethod - def toMonth(d): - return F("toMonth", d) + def toMonth(d, timezone=NO_VALUE): + return F("toMonth", d, timezone) @staticmethod - def toWeek(d, mode=0, timezone=""): + def toWeek(d, mode=0, timezone=NO_VALUE): return F("toWeek", d, mode, timezone) @staticmethod - def toISOWeek(d, timezone=""): - return F("toISOWeek", d, timezone) if timezone else F("toISOWeek", d) + def toISOWeek(d, timezone=NO_VALUE): + return F("toISOWeek", d, timezone) @staticmethod - def toDayOfYear(d): - return F("toDayOfYear", d) + def toDayOfYear(d, timezone=NO_VALUE): + return F("toDayOfYear", d, timezone) @staticmethod - def toDayOfMonth(d): - return F("toDayOfMonth", d) + def toDayOfMonth(d, timezone=NO_VALUE): + return F("toDayOfMonth", d, timezone) @staticmethod - def toDayOfWeek(d): - return F("toDayOfWeek", d) + def toDayOfWeek(d, timezone=NO_VALUE): + return F("toDayOfWeek", d, timezone) @staticmethod - def toHour(d): - return F("toHour", d) + def toHour(d, timezone=NO_VALUE): + return F("toHour", d, timezone) @staticmethod - def toMinute(d): - return F("toMinute", d) + def toMinute(d, timezone=NO_VALUE): + return F("toMinute", d, timezone) @staticmethod - def toSecond(d): - return F("toSecond", d) + def toSecond(d, timezone=NO_VALUE): + return F("toSecond", d, timezone) @staticmethod - def toMonday(d): - return F("toMonday", d) + def toMonday(d, timezone=NO_VALUE): + return F("toMonday", d, timezone) @staticmethod - def toStartOfMonth(d): - return F("toStartOfMonth", d) + def toStartOfMonth(d, timezone=NO_VALUE): + return F("toStartOfMonth", d, timezone) @staticmethod - def toStartOfQuarter(d): - return F("toStartOfQuarter", d) + def toStartOfQuarter(d, timezone=NO_VALUE): + return F("toStartOfQuarter", d, timezone) @staticmethod - def toStartOfYear(d): - return F("toStartOfYear", d) + def toStartOfYear(d, timezone=NO_VALUE): + return F("toStartOfYear", d, timezone) @staticmethod - def toStartOfISOYear(d): - return F("toStartOfISOYear", d) + def toStartOfISOYear(d, timezone=NO_VALUE): + return F("toStartOfISOYear", d, timezone) @staticmethod - def toStartOfTenMinutes(d): - return F("toStartOfTenMinutes", d) + def toStartOfTenMinutes(d, timezone=NO_VALUE): + return F("toStartOfTenMinutes", d, timezone) @staticmethod - def toStartOfWeek(d, mode=0): - return F("toStartOfWeek", d) + def toStartOfWeek(d, timezone=NO_VALUE): + return F("toStartOfWeek", d, timezone) @staticmethod - def toStartOfMinute(d): - return F("toStartOfMinute", d) + def toStartOfMinute(d, timezone=NO_VALUE): + return F("toStartOfMinute", d, timezone) @staticmethod - def toStartOfFiveMinute(d): - return F("toStartOfFiveMinute", d) + def toStartOfFiveMinute(d, timezone=NO_VALUE): + return F("toStartOfFiveMinute", d, timezone) @staticmethod - def toStartOfFifteenMinutes(d): - return F("toStartOfFifteenMinutes", d) + def toStartOfFifteenMinutes(d, timezone=NO_VALUE): + return F("toStartOfFifteenMinutes", d, timezone) @staticmethod - def toStartOfHour(d): - return F("toStartOfHour", d) + def toStartOfHour(d, timezone=NO_VALUE): + return F("toStartOfHour", d, timezone) @staticmethod - def toStartOfDay(d): - return F("toStartOfDay", d) + def toStartOfDay(d, timezone=NO_VALUE): + return F("toStartOfDay", d, timezone) @staticmethod - def toTime(d, timezone=""): + def toTime(d, timezone=NO_VALUE): return F("toTime", d, timezone) @staticmethod @@ -504,47 +504,47 @@ def toTimeZone(dt, timezone): return F("toTimeZone", dt, timezone) @staticmethod - def toUnixTimestamp(dt, timezone=""): + def toUnixTimestamp(dt, timezone=NO_VALUE): return F("toUnixTimestamp", dt, timezone) @staticmethod - def toYYYYMM(dt, timezone=""): - return F("toYYYYMM", dt, timezone) if timezone else F("toYYYYMM", dt) + def toYYYYMM(dt, timezone=NO_VALUE): + return F("toYYYYMM", dt, timezone) @staticmethod - def toYYYYMMDD(dt, timezone=""): - return F("toYYYYMMDD", dt, timezone) if timezone else F("toYYYYMMDD", dt) + def toYYYYMMDD(dt, timezone=NO_VALUE): + return F("toYYYYMMDD", dt, timezone) @staticmethod - def toYYYYMMDDhhmmss(dt, timezone=""): - return F("toYYYYMMDDhhmmss", dt, timezone) if timezone else F("toYYYYMMDDhhmmss", dt) + def toYYYYMMDDhhmmss(dt, timezone=NO_VALUE): + return F("toYYYYMMDDhhmmss", dt, timezone) @staticmethod - def toRelativeYearNum(d, timezone=""): + def toRelativeYearNum(d, timezone=NO_VALUE): return F("toRelativeYearNum", d, timezone) @staticmethod - def toRelativeMonthNum(d, timezone=""): + def toRelativeMonthNum(d, timezone=NO_VALUE): return F("toRelativeMonthNum", d, timezone) @staticmethod - def toRelativeWeekNum(d, timezone=""): + def toRelativeWeekNum(d, timezone=NO_VALUE): return F("toRelativeWeekNum", d, timezone) @staticmethod - def toRelativeDayNum(d, timezone=""): + def toRelativeDayNum(d, timezone=NO_VALUE): return F("toRelativeDayNum", d, timezone) @staticmethod - def toRelativeHourNum(d, timezone=""): + def toRelativeHourNum(d, timezone=NO_VALUE): return F("toRelativeHourNum", d, timezone) @staticmethod - def toRelativeMinuteNum(d, timezone=""): + def toRelativeMinuteNum(d, timezone=NO_VALUE): return F("toRelativeMinuteNum", d, timezone) @staticmethod - def toRelativeSecondNum(d, timezone=""): + def toRelativeSecondNum(d, timezone=NO_VALUE): return F("toRelativeSecondNum", d, timezone) @staticmethod @@ -568,7 +568,7 @@ def timeSlots(start_time, duration): return F("timeSlots", start_time, F.toUInt32(duration)) @staticmethod - def formatDateTime(d, format, timezone=""): + def formatDateTime(d, format, timezone=NO_VALUE): return F("formatDateTime", d, format, timezone) @staticmethod From c5d8d356fe8ab4c2b22afa9bacb96da97ed8ebe1 Mon Sep 17 00:00:00 2001 From: olliemath Date: Sun, 8 Aug 2021 10:11:55 +0100 Subject: [PATCH 37/51] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c703d6..7f0e0a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Change Log v2.2.0 ------ -- Support up to clickhouse 20.8 +- Support up to clickhouse 20.12, including LTS 20.8 release - Fixed boolean logic for Q objects (https://github.com/Infinidat/infi.clickhouse_orm/issues/158) - Remove implicit use of '\N' character (which was causing deprecation warnings in queries) - Tooling updates: use poetry, pytest, isort, black From 817825e878c39f844d396d67d7dfd1a9dcb2f6b6 Mon Sep 17 00:00:00 2001 From: olliemath Date: Sat, 14 Aug 2021 11:27:12 +0100 Subject: [PATCH 38/51] Chore: blacken / pep8ify tests --- pyproject.toml | 9 ++ setup.cfg | 5 +- tests/fields/test_array_fields.py | 4 +- tests/fields/test_datetime_fields.py | 112 ++++++++++++++++++----- tests/fields/test_decimal_fields.py | 4 +- tests/fields/test_enum_fields.py | 4 +- tests/fields/test_fixed_string_fields.py | 4 +- tests/fields/test_materialized_fields.py | 4 +- tests/fields/test_nullable_fields.py | 45 ++++++++- tests/fields/test_simple_fields.py | 19 +++- tests/setup.cfg | 19 ---- tests/test_buffer.py | 4 +- tests/test_constraints.py | 29 +++++- tests/test_database.py | 30 ++++-- tests/test_dictionaries.py | 12 ++- tests/test_engines.py | 91 +++++++++++++++--- tests/test_funcs.py | 110 +++++++++++++++++----- tests/test_indexes.py | 3 +- tests/test_inheritance.py | 4 +- tests/test_migrations.py | 79 +++++++++++++--- tests/test_models.py | 20 ++-- tests/test_mutations.py | 2 +- tests/test_querysets.py | 18 +++- tests/test_readonly.py | 13 ++- tests/test_server_errors.py | 5 +- tests/test_system_models.py | 5 +- 26 files changed, 507 insertions(+), 147 deletions(-) delete mode 100644 tests/setup.cfg diff --git a/pyproject.toml b/pyproject.toml index ea165f6..98859a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,12 @@ +[tool.black] +line-length = 120 +extend-exclude = ''' +/( + | examples + | scripts +)/ +''' + [tool.isort] multi_line_output = 3 include_trailing_comma = true diff --git a/setup.cfg b/setup.cfg index 0317882..48f3a58 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,4 +15,7 @@ ignore = E203 # Whitespace after ':' W503 # Operator after new line B950 # We use E501 - B008 # Using callable in function defintion, required for FastAPI +exclude = + tests/sample_migrations + examples + scripts diff --git a/tests/fields/test_array_fields.py b/tests/fields/test_array_fields.py index f9fe301..268b5ee 100644 --- a/tests/fields/test_array_fields.py +++ b/tests/fields/test_array_fields.py @@ -2,8 +2,8 @@ from datetime import date from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import ArrayField, DateField, Int32Field, StringField from clickhouse_orm.models import Model diff --git a/tests/fields/test_datetime_fields.py b/tests/fields/test_datetime_fields.py index 7b8e231..e178466 100644 --- a/tests/fields/test_datetime_fields.py +++ b/tests/fields/test_datetime_fields.py @@ -4,8 +4,8 @@ import pytz from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, DateTime64Field, DateTimeField from clickhouse_orm.models import Model @@ -42,19 +42,39 @@ def test_ad_hoc_model(self): results = list(self.database.select(query)) self.assertEqual(len(results), 2) self.assertEqual(results[0].date_field, datetime.date(2016, 8, 30)) - self.assertEqual(results[0].datetime_field, datetime.datetime(2016, 8, 30, 3, 50, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[0].hour_start, datetime.datetime(2016, 8, 30, 3, 0, 0, tzinfo=pytz.UTC)) + self.assertEqual( + results[0].datetime_field, + datetime.datetime(2016, 8, 30, 3, 50, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[0].hour_start, + datetime.datetime(2016, 8, 30, 3, 0, 0, tzinfo=pytz.UTC), + ) self.assertEqual(results[1].date_field, datetime.date(2016, 8, 31)) - self.assertEqual(results[1].datetime_field, datetime.datetime(2016, 8, 31, 1, 30, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[1].hour_start, datetime.datetime(2016, 8, 31, 1, 0, 0, tzinfo=pytz.UTC)) + self.assertEqual( + results[1].datetime_field, + datetime.datetime(2016, 8, 31, 1, 30, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].hour_start, + datetime.datetime(2016, 8, 31, 1, 0, 0, tzinfo=pytz.UTC), + ) - self.assertEqual(results[0].datetime64_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 123456, tzinfo=pytz.UTC)) self.assertEqual( - results[0].datetime64_3_field, datetime.datetime(2016, 8, 30, 3, 50, 0, 123000, tzinfo=pytz.UTC) + results[0].datetime64_field, + datetime.datetime(2016, 8, 30, 3, 50, 0, 123456, tzinfo=pytz.UTC), ) - self.assertEqual(results[1].datetime64_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 123456, tzinfo=pytz.UTC)) self.assertEqual( - results[1].datetime64_3_field, datetime.datetime(2016, 8, 31, 1, 30, 0, 123000, tzinfo=pytz.UTC) + results[0].datetime64_3_field, + datetime.datetime(2016, 8, 30, 3, 50, 0, 123000, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime64_field, + datetime.datetime(2016, 8, 31, 1, 30, 0, 123456, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime64_3_field, + datetime.datetime(2016, 8, 31, 1, 30, 0, 123000, tzinfo=pytz.UTC), ) @@ -106,20 +126,62 @@ def test_ad_hoc_model(self): query = "SELECT * from $db.modelwithtz ORDER BY datetime_no_tz_field" results = list(self.database.select(query)) - self.assertEqual(results[0].datetime_no_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[0].datetime_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[0].datetime64_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[0].datetime_utc_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[1].datetime_no_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[1].datetime_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[1].datetime64_tz_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - self.assertEqual(results[1].datetime_utc_field, datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC)) - - self.assertEqual(results[0].datetime_no_tz_field.tzinfo.zone, self.database.server_timezone.zone) - self.assertEqual(results[0].datetime_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) - self.assertEqual(results[0].datetime64_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual( + results[0].datetime_no_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[0].datetime_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[0].datetime64_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[0].datetime_utc_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime_no_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime64_tz_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + self.assertEqual( + results[1].datetime_utc_field, + datetime.datetime(2020, 6, 11, 4, 0, 0, tzinfo=pytz.UTC), + ) + + self.assertEqual( + results[0].datetime_no_tz_field.tzinfo.zone, + self.database.server_timezone.zone, + ) + self.assertEqual( + results[0].datetime_tz_field.tzinfo.zone, + pytz.timezone("Europe/Madrid").zone, + ) + self.assertEqual( + results[0].datetime64_tz_field.tzinfo.zone, + pytz.timezone("Europe/Madrid").zone, + ) self.assertEqual(results[0].datetime_utc_field.tzinfo.zone, pytz.timezone("UTC").zone) - self.assertEqual(results[1].datetime_no_tz_field.tzinfo.zone, self.database.server_timezone.zone) - self.assertEqual(results[1].datetime_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) - self.assertEqual(results[1].datetime64_tz_field.tzinfo.zone, pytz.timezone("Europe/Madrid").zone) + self.assertEqual( + results[1].datetime_no_tz_field.tzinfo.zone, + self.database.server_timezone.zone, + ) + self.assertEqual( + results[1].datetime_tz_field.tzinfo.zone, + pytz.timezone("Europe/Madrid").zone, + ) + self.assertEqual( + results[1].datetime64_tz_field.tzinfo.zone, + pytz.timezone("Europe/Madrid").zone, + ) self.assertEqual(results[1].datetime_utc_field.tzinfo.zone, pytz.timezone("UTC").zone) diff --git a/tests/fields/test_decimal_fields.py b/tests/fields/test_decimal_fields.py index 0480609..8fa2503 100644 --- a/tests/fields/test_decimal_fields.py +++ b/tests/fields/test_decimal_fields.py @@ -3,8 +3,8 @@ from decimal import Decimal from clickhouse_orm.database import Database, ServerError -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import Memory +from clickhouse_orm.fields import DateField, Decimal32Field, Decimal64Field, Decimal128Field, DecimalField from clickhouse_orm.models import Model diff --git a/tests/fields/test_enum_fields.py b/tests/fields/test_enum_fields.py index 05ad366..b4d1cf0 100644 --- a/tests/fields/test_enum_fields.py +++ b/tests/fields/test_enum_fields.py @@ -2,8 +2,8 @@ from enum import Enum from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import ArrayField, DateField, Enum8Field, Enum16Field from clickhouse_orm.models import Model diff --git a/tests/fields/test_fixed_string_fields.py b/tests/fields/test_fixed_string_fields.py index d1bce32..aa686ae 100644 --- a/tests/fields/test_fixed_string_fields.py +++ b/tests/fields/test_fixed_string_fields.py @@ -2,8 +2,8 @@ import unittest from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, FixedStringField from clickhouse_orm.models import Model diff --git a/tests/fields/test_materialized_fields.py b/tests/fields/test_materialized_fields.py index ab68a09..ba3127d 100644 --- a/tests/fields/test_materialized_fields.py +++ b/tests/fields/test_materialized_fields.py @@ -2,8 +2,8 @@ from datetime import date from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, DateTimeField, Int32Field, StringField from clickhouse_orm.funcs import F from clickhouse_orm.models import NO_VALUE, Model diff --git a/tests/fields/test_nullable_fields.py b/tests/fields/test_nullable_fields.py index 8b81ef2..1bed711 100644 --- a/tests/fields/test_nullable_fields.py +++ b/tests/fields/test_nullable_fields.py @@ -4,8 +4,25 @@ import pytz from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import ( + BaseFloatField, + BaseIntField, + DateField, + DateTimeField, + Float32Field, + Float64Field, + Int8Field, + Int16Field, + Int32Field, + Int64Field, + NullableField, + StringField, + UInt8Field, + UInt16Field, + UInt32Field, + UInt64Field, +) from clickhouse_orm.models import Model from clickhouse_orm.utils import comma_join @@ -79,7 +96,16 @@ def test_isinstance(self): f = NullableField(field()) self.assertTrue(f.isinstance(field)) self.assertTrue(f.isinstance(NullableField)) - for field in (Int8Field, Int16Field, Int32Field, Int64Field, UInt8Field, UInt16Field, UInt32Field, UInt64Field): + for field in ( + Int8Field, + Int16Field, + Int32Field, + Int64Field, + UInt8Field, + UInt16Field, + UInt32Field, + UInt64Field, + ): f = NullableField(field()) self.assertTrue(f.isinstance(BaseIntField)) for field in (Float32Field, Float64Field): @@ -95,10 +121,19 @@ def _insert_sample_data(self): self.database.insert( [ ModelWithNullable(date_field="2016-08-30", null_str="", null_int=42, null_date=dt), - ModelWithNullable(date_field="2016-08-30", null_str="nothing", null_int=None, null_date=None), + ModelWithNullable( + date_field="2016-08-30", + null_str="nothing", + null_int=None, + null_date=None, + ), ModelWithNullable(date_field="2016-08-31", null_str=None, null_int=42, null_date=dt), ModelWithNullable( - date_field="2016-08-31", null_str=None, null_int=None, null_date=None, null_default=None + date_field="2016-08-31", + null_str=None, + null_int=None, + null_date=None, + null_default=None, ), ] ) diff --git a/tests/fields/test_simple_fields.py b/tests/fields/test_simple_fields.py index 3611bd9..c495471 100644 --- a/tests/fields/test_simple_fields.py +++ b/tests/fields/test_simple_fields.py @@ -3,7 +3,7 @@ import pytz -from clickhouse_orm.fields import * +from clickhouse_orm.fields import DateField, DateTime64Field, DateTimeField, UInt8Field class SimpleFieldsTest(unittest.TestCase): @@ -36,7 +36,14 @@ def test_datetime_field(self): dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) self.assertEqual(dt, dt2) # Invalid values - for value in ("nope", "21/7/1999", 0.5, "2017-01 15:06:00", "2017-01-01X15:06:00", "2017-13-01T15:06:00"): + for value in ( + "nope", + "21/7/1999", + 0.5, + "2017-01 15:06:00", + "2017-01-01X15:06:00", + "2017-13-01T15:06:00", + ): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) @@ -62,7 +69,13 @@ def test_datetime64_field(self): dt2 = f.to_python(f.to_db_string(dt, quote=False), pytz.utc) self.assertEqual(dt, dt2) # Invalid values - for value in ("nope", "21/7/1999", "2017-01 15:06:00", "2017-01-01X15:06:00", "2017-13-01T15:06:00"): + for value in ( + "nope", + "21/7/1999", + "2017-01 15:06:00", + "2017-01-01X15:06:00", + "2017-13-01T15:06:00", + ): with self.assertRaises(ValueError): f.to_python(value, pytz.utc) diff --git a/tests/setup.cfg b/tests/setup.cfg deleted file mode 100644 index a806d48..0000000 --- a/tests/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[flake8] -max-line-length = 120 -select = - # pycodestyle - E, W - # pyflakes - F - # flake8-bugbear - B, B9 - # pydocstyle - D - # isort - I -ignore = - E203 # Whitespace after ':' - W503 # Operator after new line - B950 # We use E501 - B008 # Using callable in function defintion, required for FastAPI - F405, F403 # Since * imports cause havok diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 526e07d..61e6743 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from clickhouse_orm.engines import * +from clickhouse_orm.engines import Buffer from clickhouse_orm.models import BufferModel -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData, data class BufferTestCase(TestCaseWithData): diff --git a/tests/test_constraints.py b/tests/test_constraints.py index 399a154..73baf9b 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -1,6 +1,6 @@ import unittest -from clickhouse_orm import * +from clickhouse_orm import Constraint, Database, F, ServerError from .base_test_with_data import Person @@ -17,20 +17,41 @@ def tearDown(self): def test_insert_valid_values(self): self.database.insert( - [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2000-01-01", height=1.66)] + [ + PersonWithConstraints( + first_name="Mike", + last_name="Caruzo", + birthday="2000-01-01", + height=1.66, + ) + ] ) def test_insert_invalid_values(self): with self.assertRaises(ServerError) as e: self.database.insert( - [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="2100-01-01", height=1.66)] + [ + PersonWithConstraints( + first_name="Mike", + last_name="Caruzo", + birthday="2100-01-01", + height=1.66, + ) + ] ) self.assertEqual(e.code, 469) self.assertTrue("Constraint `birthday_in_the_past`" in str(e)) with self.assertRaises(ServerError) as e: self.database.insert( - [PersonWithConstraints(first_name="Mike", last_name="Caruzo", birthday="1970-01-01", height=3)] + [ + PersonWithConstraints( + first_name="Mike", + last_name="Caruzo", + birthday="1970-01-01", + height=3, + ) + ] ) self.assertEqual(e.code, 469) self.assertTrue("Constraint `max_height`" in str(e)) diff --git a/tests/test_database.py b/tests/test_database.py index 9566a3c..388f7c7 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -2,14 +2,14 @@ import datetime import unittest -from clickhouse_orm.database import DatabaseException, ServerError +from clickhouse_orm.database import Database, DatabaseException, ServerError from clickhouse_orm.engines import Memory -from clickhouse_orm.fields import * +from clickhouse_orm.fields import DateField, DateTimeField, Float32Field, Int32Field, StringField from clickhouse_orm.funcs import F from clickhouse_orm.models import Model from clickhouse_orm.query import Q -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData, data class DatabaseTestCase(TestCaseWithData): @@ -136,12 +136,19 @@ def test_pagination_last_page(self): page_a = self.database.paginate(Person, "first_name, last_name", -1, page_size) page_b = self.database.paginate(Person, "first_name, last_name", page_a.pages_total, page_size) self.assertEqual(page_a[1:], page_b[1:]) - self.assertEqual([obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects]) + self.assertEqual( + [obj.to_tsv() for obj in page_a.objects], + [obj.to_tsv() for obj in page_b.objects], + ) def test_pagination_empty_page(self): for page_num in (-1, 1, 2): page = self.database.paginate( - Person, "first_name, last_name", page_num, 10, conditions="first_name = 'Ziggy'" + Person, + "first_name, last_name", + page_num, + 10, + conditions="first_name = 'Ziggy'", ) self.assertEqual(page.number_of_objects, 0) self.assertEqual(page.objects, []) @@ -160,7 +167,13 @@ def test_pagination_with_conditions(self): page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions="first_name < 'Ava'") self.assertEqual(page.number_of_objects, 10) # Conditions as expression - page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions=Person.first_name < "Ava") + page = self.database.paginate( + Person, + "first_name, last_name", + 1, + 100, + conditions=Person.first_name < "Ava", + ) self.assertEqual(page.number_of_objects, 10) # Conditions as Q object page = self.database.paginate(Person, "first_name, last_name", 1, 100, conditions=Q(first_name__lt="Ava")) @@ -177,7 +190,10 @@ def test_raw(self): self._insert_and_check(self._sample_data(), len(data)) query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name" results = self.database.raw(query) - self.assertEqual(results, "Whitney\tDurham\t1977-09-15\t1.72\t\\N\nWhitney\tScott\t1971-07-04\t1.7\t\\N\n") + self.assertEqual( + results, + "Whitney\tDurham\t1977-09-15\t1.72\t\\N\nWhitney\tScott\t1971-07-04\t1.7\t\\N\n", + ) def test_invalid_user(self): with self.assertRaises(ServerError) as cm: diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 39e1f3b..682eb09 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -1,7 +1,7 @@ import logging import unittest -from clickhouse_orm import * +from clickhouse_orm import Database, F, Memory, Model, StringField, UInt64Field class DictionaryTestMixin: @@ -99,8 +99,14 @@ def test_dictget(self): self._test_func(F.dictGet(self.dict_name, "region_name", F.toUInt64(99)), "?") def test_dictgetordefault(self): - self._test_func(F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(3), "n/a"), "Center") - self._test_func(F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(99), "n/a"), "n/a") + self._test_func( + F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(3), "n/a"), + "Center", + ) + self._test_func( + F.dictGetOrDefault(self.dict_name, "region_name", F.toUInt64(99), "n/a"), + "n/a", + ) def test_dicthas(self): self._test_func(F.dictHas(self.dict_name, F.toUInt64(3)), 1) diff --git a/tests/test_engines.py b/tests/test_engines.py index f796a27..765ce6e 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -2,7 +2,21 @@ import logging import unittest -from clickhouse_orm import * +from clickhouse_orm.database import Database, DatabaseException, ServerError +from clickhouse_orm.engines import ( + CollapsingMergeTree, + Log, + Memory, + Merge, + MergeTree, + ReplacingMergeTree, + SummingMergeTree, + TinyLog, +) +from clickhouse_orm.fields import DateField, Int8Field, UInt8Field, UInt16Field, UInt32Field +from clickhouse_orm.funcs import F +from clickhouse_orm.models import Distributed, DistributedModel, MergeModel, Model +from clickhouse_orm.system_models import SystemPart logging.getLogger("requests").setLevel(logging.WARNING) @@ -19,7 +33,15 @@ class EnginesTestCase(_EnginesHelperTestCase): def _create_and_insert(self, model_class): self.database.create_table(model_class) self.database.insert( - [model_class(date="2017-01-01", event_id=23423, event_group=13, event_count=7, event_version=1)] + [ + model_class( + date="2017-01-01", + event_id=23423, + event_group=13, + event_count=7, + event_version=1, + ) + ] ) def test_merge_tree(self): @@ -31,7 +53,9 @@ class TestModel(SampleModel): def test_merge_tree_with_sampling(self): class TestModel(SampleModel): engine = MergeTree( - "date", ("date", "event_id", "event_group", "intHash32(event_id)"), sampling_expr="intHash32(event_id)" + "date", + ("date", "event_id", "event_group", "intHash32(event_id)"), + sampling_expr="intHash32(event_id)", ) self._create_and_insert(TestModel) @@ -129,15 +153,44 @@ class TestMergeModel(MergeModel, SampleModel): # Insert operations are restricted for this model type with self.assertRaises(DatabaseException): self.database.insert( - [TestMergeModel(date="2017-01-01", event_id=23423, event_group=13, event_count=7, event_version=1)] + [ + TestMergeModel( + date="2017-01-01", + event_id=23423, + event_group=13, + event_count=7, + event_version=1, + ) + ] ) # Testing select - self.database.insert([TestModel1(date="2017-01-01", event_id=1, event_group=1, event_count=1, event_version=1)]) - self.database.insert([TestModel2(date="2017-01-02", event_id=2, event_group=2, event_count=2, event_version=2)]) + self.database.insert( + [ + TestModel1( + date="2017-01-01", + event_id=1, + event_group=1, + event_count=1, + event_version=1, + ) + ] + ) + self.database.insert( + [ + TestModel2( + date="2017-01-02", + event_id=2, + event_group=2, + event_count=2, + event_version=2, + ) + ] + ) # event_uversion is materialized field. So * won't select it and it will be zero res = self.database.select( - "SELECT *, _table, event_uversion FROM $table ORDER BY event_id", model_class=TestMergeModel + "SELECT *, _table, event_uversion FROM $table ORDER BY event_id", + model_class=TestMergeModel, ) res = list(res) self.assertEqual(2, len(res)) @@ -169,7 +222,8 @@ class TestMergeModel(MergeModel, SampleModel): def test_custom_partitioning(self): class TestModel(SampleModel): engine = MergeTree( - order_by=("date", "event_id", "event_group"), partition_key=("toYYYYMM(date)", "event_group") + order_by=("date", "event_id", "event_group"), + partition_key=("toYYYYMM(date)", "event_group"), ) class TestCollapseModel(SampleModel): @@ -327,12 +381,27 @@ def _test_insert_select(self, local_to_distributed, test_model=TestModel, includ self.database.insert( [ - to_insert(date="2017-01-01", event_id=1, event_group=1, event_count=1, event_version=1), - to_insert(date="2017-01-02", event_id=2, event_group=2, event_count=2, event_version=2), + to_insert( + date="2017-01-01", + event_id=1, + event_group=1, + event_count=1, + event_version=1, + ), + to_insert( + date="2017-01-02", + event_id=2, + event_group=2, + event_count=2, + event_version=2, + ), ] ) # event_uversion is materialized field. So * won't select it and it will be zero - res = self.database.select("SELECT *, event_uversion FROM $table ORDER BY event_id", model_class=to_select) + res = self.database.select( + "SELECT *, event_uversion FROM $table ORDER BY event_id", + model_class=to_select, + ) res = [row for row in res] self.assertEqual(2, len(res)) self.assertDictEqual( diff --git a/tests/test_funcs.py b/tests/test_funcs.py index 81e9318..f24c05d 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -246,11 +246,20 @@ def test_date_functions(self): self._test_func(F.toStartOfYear(d), date(2018, 1, 1)) self._test_func(F.toStartOfYear(dt), date(2018, 1, 1)) self._test_func(F.toStartOfMinute(dt), datetime(2018, 12, 31, 11, 22, 0, tzinfo=pytz.utc)) - self._test_func(F.toStartOfFiveMinute(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc)) - self._test_func(F.toStartOfFifteenMinutes(dt), datetime(2018, 12, 31, 11, 15, 0, tzinfo=pytz.utc)) + self._test_func( + F.toStartOfFiveMinute(dt), + datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc), + ) + self._test_func( + F.toStartOfFifteenMinutes(dt), + datetime(2018, 12, 31, 11, 15, 0, tzinfo=pytz.utc), + ) self._test_func(F.toStartOfHour(dt), datetime(2018, 12, 31, 11, 0, 0, tzinfo=pytz.utc)) self._test_func(F.toStartOfISOYear(dt), date(2018, 12, 31)) - self._test_func(F.toStartOfTenMinutes(dt), datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc)) + self._test_func( + F.toStartOfTenMinutes(dt), + datetime(2018, 12, 31, 11, 20, 0, tzinfo=pytz.utc), + ) self._test_func(F.toStartOfWeek(dt), date(2018, 12, 30)) self._test_func(F.toTime(dt), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) self._test_func(F.toUnixTimestamp(dt, "UTC"), int(dt.replace(tzinfo=pytz.utc).timestamp())) @@ -328,9 +337,18 @@ def test_date_functions_utc_only(self): self._test_func(F.toHour(dt), 11) self._test_func(F.toStartOfDay(dt), datetime(2018, 12, 31, 0, 0, 0, tzinfo=pytz.utc)) self._test_func(F.toTime(dt, pytz.utc), datetime(1970, 1, 2, 11, 22, 33, tzinfo=pytz.utc)) - self._test_func(F.toTime(dt, "Europe/Athens"), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) - self._test_func(F.toTime(dt, athens_tz), athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33))) - self._test_func(F.toTimeZone(dt, "Europe/Athens"), athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33))) + self._test_func( + F.toTime(dt, "Europe/Athens"), + athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)), + ) + self._test_func( + F.toTime(dt, athens_tz), + athens_tz.localize(datetime(1970, 1, 2, 13, 22, 33)), + ) + self._test_func( + F.toTimeZone(dt, "Europe/Athens"), + athens_tz.localize(datetime(2018, 12, 31, 13, 22, 33)), + ) self._test_func(F.today(), datetime.utcnow().date()) self._test_func(F.yesterday(), datetime.utcnow().date() - timedelta(days=1)) self._test_func(F.toYYYYMMDDhhmmss(dt), 20181231112233) @@ -390,16 +408,25 @@ def test_type_conversion_functions(self): def test_type_conversion_functions__utc_only(self): if self.database.server_timezone != pytz.utc: raise unittest.SkipTest("This test must run with UTC as the server timezone") - self._test_func(F.toDateTime("2018-12-31 11:22:33"), datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc)) self._test_func( - F.toDateTime64("2018-12-31 11:22:33.001", 6), datetime(2018, 12, 31, 11, 22, 33, 1000, tzinfo=pytz.utc) + F.toDateTime("2018-12-31 11:22:33"), + datetime(2018, 12, 31, 11, 22, 33, tzinfo=pytz.utc), + ) + self._test_func( + F.toDateTime64("2018-12-31 11:22:33.001", 6), + datetime(2018, 12, 31, 11, 22, 33, 1000, tzinfo=pytz.utc), ) - self._test_func(F.parseDateTimeBestEffort("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc)) self._test_func( - F.parseDateTimeBestEffortOrNull("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc) + F.parseDateTimeBestEffort("31/12/2019 10:05AM"), + datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc), ) self._test_func( - F.parseDateTimeBestEffortOrZero("31/12/2019 10:05AM"), datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc) + F.parseDateTimeBestEffortOrNull("31/12/2019 10:05AM"), + datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc), + ) + self._test_func( + F.parseDateTimeBestEffortOrZero("31/12/2019 10:05AM"), + datetime(2019, 12, 31, 10, 5, tzinfo=pytz.utc), ) def test_string_functions(self): @@ -420,7 +447,10 @@ def test_string_functions(self): self._test_func(F.substringUTF8("123456", 3, 2), "34") self._test_func(F.appendTrailingCharIfAbsent("Hello", "!"), "Hello!") self._test_func(F.appendTrailingCharIfAbsent("Hello!", "!"), "Hello!") - self._test_func(F.convertCharset(F.convertCharset("Hello", "latin1", "utf16"), "utf16", "latin1"), "Hello") + self._test_func( + F.convertCharset(F.convertCharset("Hello", "latin1", "utf16"), "utf16", "latin1"), + "Hello", + ) self._test_func(F.startsWith("aaa", "aa"), True) self._test_func(F.startsWith("aaa", "bb"), False) self._test_func(F.endsWith("aaa", "aa"), True) @@ -592,17 +622,36 @@ def test_bitmap_functions(self): self._test_func(F.bitmapContains(F.bitmapBuild([1, 5, 7, 9]), F.toUInt32(9)), 1) self._test_func(F.bitmapHasAny(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 1) self._test_func(F.bitmapHasAll(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 0) - self._test_func(F.bitmapToArray(F.bitmapAnd(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [3]) self._test_func( - F.bitmapToArray(F.bitmapOr(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2, 3, 4, 5] + F.bitmapToArray(F.bitmapAnd(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), + [3], + ) + self._test_func( + F.bitmapToArray(F.bitmapOr(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), + [1, 2, 3, 4, 5], + ) + self._test_func( + F.bitmapToArray(F.bitmapXor(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), + [1, 2, 4, 5], + ) + self._test_func( + F.bitmapToArray(F.bitmapAndnot(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), + [1, 2], ) - self._test_func(F.bitmapToArray(F.bitmapXor(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2, 4, 5]) - self._test_func(F.bitmapToArray(F.bitmapAndnot(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5]))), [1, 2]) self._test_func(F.bitmapCardinality(F.bitmapBuild([1, 2, 3, 4, 5])), 5) - self._test_func(F.bitmapAndCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 1) + self._test_func( + F.bitmapAndCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), + 1, + ) self._test_func(F.bitmapOrCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 5) - self._test_func(F.bitmapXorCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 4) - self._test_func(F.bitmapAndnotCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), 2) + self._test_func( + F.bitmapXorCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), + 4, + ) + self._test_func( + F.bitmapAndnotCardinality(F.bitmapBuild([1, 2, 3]), F.bitmapBuild([3, 4, 5])), + 2, + ) def test_hash_functions(self): args = ["x", "y", "z"] @@ -662,16 +711,26 @@ def test_ip_funcs(self): self._test_func(F.IPv4NumToString(F.toUInt32(1)), "0.0.0.1") self._test_func(F.IPv4NumToStringClassC(F.toUInt32(1)), "0.0.0.xxx") self._test_func(F.IPv4StringToNum("0.0.0.17"), 17) - self._test_func(F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum("192.168.0.1"))), "::ffff:192.168.0.1") + self._test_func( + F.IPv6NumToString(F.IPv4ToIPv6(F.IPv4StringToNum("192.168.0.1"))), + "::ffff:192.168.0.1", + ) self._test_func(F.IPv6NumToString(F.IPv6StringToNum("2a02:6b8::11")), "2a02:6b8::11") self._test_func(F.toIPv4("10.20.30.40"), IPv4Address("10.20.30.40")) - self._test_func(F.toIPv6("2001:438:ffff::407d:1bc1"), IPv6Address("2001:438:ffff::407d:1bc1")) self._test_func( - F.IPv4CIDRToRange(F.toIPv4("192.168.5.2"), 16), [IPv4Address("192.168.0.0"), IPv4Address("192.168.255.255")] + F.toIPv6("2001:438:ffff::407d:1bc1"), + IPv6Address("2001:438:ffff::407d:1bc1"), + ) + self._test_func( + F.IPv4CIDRToRange(F.toIPv4("192.168.5.2"), 16), + [IPv4Address("192.168.0.0"), IPv4Address("192.168.255.255")], ) self._test_func( F.IPv6CIDRToRange(F.toIPv6("2001:0db8:0000:85a3:0000:0000:ac1f:8001"), 32), - [IPv6Address("2001:db8::"), IPv6Address("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff")], + [ + IPv6Address("2001:db8::"), + IPv6Address("2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"), + ], ) def test_aggregate_funcs(self): @@ -680,7 +739,10 @@ def test_aggregate_funcs(self): self._test_aggr(F.anyLast(Person.first_name)) self._test_aggr(F.argMin(Person.first_name, Person.height)) self._test_aggr(F.argMax(Person.first_name, Person.height)) - self._test_aggr(F.round(F.avg(Person.height), 4), sum(p.height for p in self._sample_data()) / 100) + self._test_aggr( + F.round(F.avg(Person.height), 4), + sum(p.height for p in self._sample_data()) / 100, + ) self._test_aggr(F.corr(Person.height, Person.height), 1) self._test_aggr(F.count(), 100) self._test_aggr(F.round(F.covarPop(Person.height, Person.height), 2), 0) diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 7bf9bba..cdcf004 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -1,6 +1,7 @@ import unittest -from clickhouse_orm import * +from clickhouse_orm import Database, F, Index, MergeTree, Model +from clickhouse_orm.fields import DateField, Int32Field, StringField class IndexesTest(unittest.TestCase): diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py index a326164..53e2910 100644 --- a/tests/test_inheritance.py +++ b/tests/test_inheritance.py @@ -1,8 +1,8 @@ import unittest from clickhouse_orm.database import Database -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, Float32Field, Int32Field, StringField from clickhouse_orm.models import Model diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 547a803..a600ce0 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -6,8 +6,23 @@ from enum import Enum from clickhouse_orm.database import Database, ServerError -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import Buffer, MergeTree +from clickhouse_orm.fields import ( + ArrayField, + DateField, + DateTimeField, + Enum8Field, + Enum16Field, + Float32Field, + Float64Field, + Int8Field, + Int32Field, + Int64Field, + LowCardinalityField, + NullableField, + StringField, + UInt64Field, +) from clickhouse_orm.migrations import MigrationHistory from clickhouse_orm.models import BufferModel, Constraint, Index, Model @@ -45,7 +60,10 @@ def test_migrations(self): self.database.migrate("tests.sample_migrations", 3) self.assertTrue(self.table_exists(Model1)) # Adding, removing and altering simple fields - self.assertEqual(self.get_table_fields(Model1), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) + self.assertEqual( + self.get_table_fields(Model1), + [("date", "Date"), ("f1", "Int32"), ("f2", "String")], + ) self.database.migrate("tests.sample_migrations", 4) self.assertEqual( self.get_table_fields(Model2), @@ -60,36 +78,59 @@ def test_migrations(self): ) self.database.migrate("tests.sample_migrations", 5) self.assertEqual( - self.get_table_fields(Model3), [("date", "Date"), ("f1", "Int64"), ("f3", "Float64"), ("f4", "String")] + self.get_table_fields(Model3), + [("date", "Date"), ("f1", "Int64"), ("f3", "Float64"), ("f4", "String")], ) # Altering enum fields self.database.migrate("tests.sample_migrations", 6) self.assertTrue(self.table_exists(EnumModel1)) self.assertEqual( - self.get_table_fields(EnumModel1), [("date", "Date"), ("f1", "Enum8('dog' = 1, 'cat' = 2, 'cow' = 3)")] + self.get_table_fields(EnumModel1), + [("date", "Date"), ("f1", "Enum8('dog' = 1, 'cat' = 2, 'cow' = 3)")], ) self.database.migrate("tests.sample_migrations", 7) self.assertTrue(self.table_exists(EnumModel1)) self.assertEqual( self.get_table_fields(EnumModel2), - [("date", "Date"), ("f1", "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)")], + [ + ("date", "Date"), + ("f1", "Enum16('dog' = 1, 'cat' = 2, 'horse' = 3, 'pig' = 4)"), + ], ) # Materialized fields and alias fields self.database.migrate("tests.sample_migrations", 8) self.assertTrue(self.table_exists(MaterializedModel)) - self.assertEqual(self.get_table_fields(MaterializedModel), [("date_time", "DateTime"), ("date", "Date")]) + self.assertEqual( + self.get_table_fields(MaterializedModel), + [("date_time", "DateTime"), ("date", "Date")], + ) self.database.migrate("tests.sample_migrations", 9) self.assertTrue(self.table_exists(AliasModel)) - self.assertEqual(self.get_table_fields(AliasModel), [("date", "Date"), ("date_alias", "Date")]) + self.assertEqual( + self.get_table_fields(AliasModel), + [("date", "Date"), ("date_alias", "Date")], + ) # Buffer models creation and alteration self.database.migrate("tests.sample_migrations", 10) self.assertTrue(self.table_exists(Model4)) self.assertTrue(self.table_exists(Model4Buffer)) - self.assertEqual(self.get_table_fields(Model4), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) - self.assertEqual(self.get_table_fields(Model4Buffer), [("date", "Date"), ("f1", "Int32"), ("f2", "String")]) + self.assertEqual( + self.get_table_fields(Model4), + [("date", "Date"), ("f1", "Int32"), ("f2", "String")], + ) + self.assertEqual( + self.get_table_fields(Model4Buffer), + [("date", "Date"), ("f1", "Int32"), ("f2", "String")], + ) self.database.migrate("tests.sample_migrations", 11) - self.assertEqual(self.get_table_fields(Model4), [("date", "Date"), ("f3", "DateTime"), ("f2", "String")]) - self.assertEqual(self.get_table_fields(Model4Buffer), [("date", "Date"), ("f3", "DateTime"), ("f2", "String")]) + self.assertEqual( + self.get_table_fields(Model4), + [("date", "Date"), ("f3", "DateTime"), ("f2", "String")], + ) + self.assertEqual( + self.get_table_fields(Model4Buffer), + [("date", "Date"), ("f3", "DateTime"), ("f2", "String")], + ) self.database.migrate("tests.sample_migrations", 12) self.assertEqual(self.database.count(Model3), 3) @@ -105,12 +146,22 @@ def test_migrations(self): self.assertTrue(self.table_exists(MaterializedModel1)) self.assertEqual( self.get_table_fields(MaterializedModel1), - [("date_time", "DateTime"), ("int_field", "Int8"), ("date", "Date"), ("int_field_plus_one", "Int8")], + [ + ("date_time", "DateTime"), + ("int_field", "Int8"), + ("date", "Date"), + ("int_field_plus_one", "Int8"), + ], ) self.assertTrue(self.table_exists(AliasModel1)) self.assertEqual( self.get_table_fields(AliasModel1), - [("date", "Date"), ("int_field", "Int8"), ("date_alias", "Date"), ("int_field_plus_one", "Int8")], + [ + ("date", "Date"), + ("int_field", "Int8"), + ("date_alias", "Date"), + ("int_field_plus_one", "Int8"), + ], ) # Codecs and low cardinality self.database.migrate("tests.sample_migrations", 15) diff --git a/tests/test_models.py b/tests/test_models.py index ef78b96..58ec9e6 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -3,8 +3,8 @@ import pytz -from clickhouse_orm.engines import * -from clickhouse_orm.fields import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, DateTimeField, Float32Field, Int32Field, StringField from clickhouse_orm.funcs import F from clickhouse_orm.models import NO_VALUE, Model @@ -83,8 +83,14 @@ def test_to_dict(self): }, ) self.assertDictEqual( - instance.to_dict(include_readonly=False, field_names=("int_field", "alias_field", "datetime_field")), - {"int_field": 100, "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc)}, + instance.to_dict( + include_readonly=False, + field_names=("int_field", "alias_field", "datetime_field"), + ), + { + "int_field": 100, + "datetime_field": datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytz.utc), + }, ) def test_field_name_in_error_message_for_invalid_value_in_constructor(self): @@ -93,7 +99,8 @@ def test_field_name_in_error_message_for_invalid_value_in_constructor(self): SimpleModel(str_field=bad_value) self.assertEqual( - "Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)), str(cm.exception) + "Invalid value for StringField: {} (field 'str_field')".format(repr(bad_value)), + str(cm.exception), ) def test_field_name_in_error_message_for_invalid_value_in_assignment(self): @@ -103,7 +110,8 @@ def test_field_name_in_error_message_for_invalid_value_in_assignment(self): instance.float_field = bad_value self.assertEqual( - "Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)), str(cm.exception) + "Invalid value for Float32Field - {} (field 'float_field')".format(repr(bad_value)), + str(cm.exception), ) diff --git a/tests/test_mutations.py b/tests/test_mutations.py index afbb1ed..9ede2a4 100644 --- a/tests/test_mutations.py +++ b/tests/test_mutations.py @@ -3,7 +3,7 @@ from clickhouse_orm import F -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData class MutationsTestCase(TestCaseWithData): diff --git a/tests/test_querysets.py b/tests/test_querysets.py index 10ec863..a476545 100644 --- a/tests/test_querysets.py +++ b/tests/test_querysets.py @@ -79,7 +79,10 @@ def test_filter_with_q_objects(self): qs = Person.objects_in(self.database) self._test_qs(qs.filter(Q(first_name="Ciaran")), 2) self._test_qs(qs.filter(Q(first_name="Ciaran") | Q(first_name="Chelsea")), 3) - self._test_qs(qs.filter(Q(first_name__in=["Warren", "Whilemina", "Whitney"]) & Q(height__gte=1.7)), 3) + self._test_qs( + qs.filter(Q(first_name__in=["Warren", "Whilemina", "Whitney"]) & Q(height__gte=1.7)), + 3, + ) self._test_qs( qs.filter( ( @@ -103,7 +106,10 @@ def test_filter_with_q_objects(self): ), 2, ) - self._test_qs(qs.filter(Q(first_name="Courtney") | Q(first_name="Cassady") & Q(last_name="Knapp")), 3) + self._test_qs( + qs.filter(Q(first_name="Courtney") | Q(first_name="Cassady") & Q(last_name="Knapp")), + 3, + ) def test_filter_unicode_string(self): self.database.insert([Person(first_name=u"דונלד", last_name=u"דאק")]) @@ -269,7 +275,10 @@ def test_pagination_last_page(self): page_a = qs.paginate(-1, page_size) page_b = qs.paginate(page_a.pages_total, page_size) self.assertEqual(page_a[1:], page_b[1:]) - self.assertEqual([obj.to_tsv() for obj in page_a.objects], [obj.to_tsv() for obj in page_b.objects]) + self.assertEqual( + [obj.to_tsv() for obj in page_a.objects], + [obj.to_tsv() for obj in page_b.objects], + ) def test_pagination_invalid_page(self): qs = Person.objects_in(self.database).order_by("first_name", "last_name") @@ -320,7 +329,8 @@ def test_mixed_filter(self): qs = Person.objects_in(self.database) qs = qs.filter(Q(first_name="a"), F("greater", Person.height, 1.7), last_name="b") self.assertEqual( - qs.conditions_as_sql(), "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')" + qs.conditions_as_sql(), + "(first_name = 'a') AND (greater(`height`, 1.7)) AND (last_name = 'b')", ) def test_invalid_filter(self): diff --git a/tests/test_readonly.py b/tests/test_readonly.py index 94083a1..03b1587 100644 --- a/tests/test_readonly.py +++ b/tests/test_readonly.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- +import unittest -from clickhouse_orm.database import DatabaseException, ServerError +from clickhouse_orm.database import Database, DatabaseException, ServerError +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import DateField, StringField +from clickhouse_orm.models import Model -from .base_test_with_data import * +from .base_test_with_data import Person, TestCaseWithData, data class ReadonlyTestCase(TestCaseWithData): @@ -65,7 +69,10 @@ def test_drop_readonly_table(self): def test_nonexisting_readonly_database(self): with self.assertRaises(DatabaseException) as cm: Database("dummy", readonly=True) - self.assertEqual(str(cm.exception), "Database does not exist, and cannot be created under readonly connection") + self.assertEqual( + str(cm.exception), + "Database does not exist, and cannot be created under readonly connection", + ) class ReadOnlyModel(Model): diff --git a/tests/test_server_errors.py b/tests/test_server_errors.py index d4cff3d..5328ad6 100644 --- a/tests/test_server_errors.py +++ b/tests/test_server_errors.py @@ -16,7 +16,10 @@ def test_old_format(self): "Code: 161, e.displayText() = DB::Exception: Limit for number of columns to read exceeded. Requested: 11, maximum: 1, e.what() = DB::Exception\n" ) self.assertEqual(code, 161) - self.assertEqual(msg, "Limit for number of columns to read exceeded. Requested: 11, maximum: 1") + self.assertEqual( + msg, + "Limit for number of columns to read exceeded. Requested: 11, maximum: 1", + ) def test_new_format(self): diff --git a/tests/test_system_models.py b/tests/test_system_models.py index 15fdad2..3e62ce9 100644 --- a/tests/test_system_models.py +++ b/tests/test_system_models.py @@ -120,7 +120,10 @@ class CustomPartitionedTable(Model): date_field = DateField() group_field = UInt32Field() - engine = MergeTree(order_by=("date_field", "group_field"), partition_key=("toYYYYMM(date_field)", "group_field")) + engine = MergeTree( + order_by=("date_field", "group_field"), + partition_key=("toYYYYMM(date_field)", "group_field"), + ) class SystemTestModel(Model): From 1a4785d61e877d839584a9eb7de06f934f5e10d0 Mon Sep 17 00:00:00 2001 From: olliemath Date: Sat, 14 Aug 2021 11:36:10 +0100 Subject: [PATCH 39/51] Compatibility: get tests working with CH 21.1 --- tests/test_engines.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_engines.py b/tests/test_engines.py index 765ce6e..354089d 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -227,7 +227,7 @@ class TestModel(SampleModel): ) class TestCollapseModel(SampleModel): - sign = Int8Field() + sign = Int8Field(default=-1) engine = CollapsingMergeTree( sign_col="sign", @@ -259,7 +259,7 @@ class TestModel(SampleModel): ) class TestCollapseModel(SampleModel): - sign = Int8Field() + sign = Int8Field(default=1) engine = CollapsingMergeTree( sign_col="sign", @@ -427,10 +427,6 @@ def _test_insert_select(self, local_to_distributed, test_model=TestModel, includ res[1].to_dict(include_readonly=include_readonly), ) - @unittest.skip( - "Bad support of materialized fields in Distributed tables " - "https://groups.google.com/forum/#!topic/clickhouse/XEYRRwZrsSc" - ) def test_insert_distributed_select_local(self): return self._test_insert_select(local_to_distributed=False) From a924f6c7e25b4edaac21967af43ea5a22f7bb094 Mon Sep 17 00:00:00 2001 From: olliemath Date: Sat, 14 Aug 2021 12:24:33 +0100 Subject: [PATCH 40/51] Compatibility: update tests for 21.3+ --- tests/test_database.py | 4 ++++ tests/test_dictionaries.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 388f7c7..de808ba 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -296,6 +296,10 @@ def test_get_model_for_table__system(self): self.assertTrue(model.is_system_model()) self.assertTrue(model.is_read_only()) self.assertEqual(model.table_name(), row.name) + + if row.name == "distributed_ddl_queue": + continue # Since zookeeper is not set up in our tests + # Read a few records try: list(model.objects_in(self.database)[:10]) diff --git a/tests/test_dictionaries.py b/tests/test_dictionaries.py index 682eb09..11c37d2 100644 --- a/tests/test_dictionaries.py +++ b/tests/test_dictionaries.py @@ -14,14 +14,17 @@ def setUp(self): def tearDown(self): self.database.drop_database() - def _test_func(self, func, expected_value): + def _call_func(self, func): sql = "SELECT %s AS value" % func.to_sql() logging.info(sql) result = list(self.database.select(sql)) logging.info("\t==> %s", result[0].value if result else "") + return result + + def _test_func(self, func, expected_value): + result = self._call_func(func) print("Comparing %s to %s" % (result[0].value, expected_value)) - self.assertEqual(result[0].value, expected_value) - return result[0].value if result else None + assert result[0].value == expected_value class SimpleDictionaryTest(DictionaryTestMixin, unittest.TestCase): @@ -114,7 +117,10 @@ def test_dicthas(self): def test_dictgethierarchy(self): self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(3)), [3, 2, 1]) - self._test_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99)), [99]) + # Default behaviour changed in CH, but we're not really testing that + default = self._call_func(F.dictGetHierarchy(self.dict_name, F.toUInt64(99))) + assert isinstance(default, list) + assert len(default) <= 1 # either [] or [99] def test_dictisin(self): self._test_func(F.dictIsIn(self.dict_name, F.toUInt64(3), F.toUInt64(1)), 1) From 19f89e0efe5d711e02413aef779014c601e37963 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Sat, 14 Aug 2021 11:29:53 +0000 Subject: [PATCH 41/51] Test with CH 21.3 and 20.8 --- .github/workflows/python-package.yml | 34 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 45fef0b..bd19f72 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,17 +30,16 @@ jobs: poetry install - name: Lint with flake8 run: | - poetry run flake8 clickhouse_orm/ - cd tests && poetry run flake8 + poetry run flake8 - name: Check formatting with black run: | - poetry run black --line-length 120 --check clickhouse_orm/ tests/ + poetry run black --check . test: runs-on: ubuntu-latest services: clickhouse: - image: yandex/clickhouse-server:20.8 + image: yandex/clickhouse-server:21.3 ports: - 8123:8123 strategy: @@ -61,3 +60,30 @@ jobs: - name: Test with pytest run: | poetry run pytest + + test_compat: + # Tests compatibility with an older LTS release of clickhouse + runs-on: ubuntu-latest + services: + clickhouse: + image: yandex/clickhouse-server:20.8 + ports: + - 8123:8123 + strategy: + fail-fast: false + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install poetry + poetry install + - name: Test with pytest + run: | + poetry run pytest From 0e9dea5bcb9a0ebda66d39826b0c046f402fef42 Mon Sep 17 00:00:00 2001 From: olliemath Date: Mon, 16 Aug 2021 09:35:59 +0100 Subject: [PATCH 42/51] Chore: update scripts/docs --- docs/class_reference.md | 391 ++++++++++++++++++++++++++++---- docs/toc.md | 14 +- pyproject.toml | 1 - scripts/docs2html.sh | 2 +- scripts/generate_all.sh | 3 +- scripts/generate_ref.py | 77 ++++--- scripts/generate_toc.sh | 4 +- scripts/html_to_markdown_toc.py | 17 +- scripts/test_python3.sh | 11 - setup.cfg | 1 - 10 files changed, 409 insertions(+), 112 deletions(-) delete mode 100755 scripts/test_python3.sh diff --git a/docs/class_reference.md b/docs/class_reference.md index 3d91cdc..490db16 100644 --- a/docs/class_reference.md +++ b/docs/class_reference.md @@ -2,7 +2,7 @@ Class Reference =============== clickhouse_orm.database ----------------------------- +----------------------- ### Database @@ -153,7 +153,7 @@ Extends Exception Raised when a database operation fails. clickhouse_orm.models --------------------------- +--------------------- ### Model @@ -812,7 +812,7 @@ separated by non-alphanumeric characters. clickhouse_orm.fields --------------------------- +--------------------- ### ArrayField @@ -1047,7 +1047,7 @@ Extends Field clickhouse_orm.engines ---------------------------- +---------------------- ### Engine @@ -1141,7 +1141,7 @@ Extends MergeTree clickhouse_orm.query -------------------------- +-------------------- ### QuerySet @@ -1444,7 +1444,7 @@ https://clickhouse.tech/docs/en/query_language/select/#with-totals-modifier clickhouse_orm.funcs -------------------------- +-------------------- ### F @@ -2012,7 +2012,7 @@ Initializer. #### floor(n=None) -#### formatDateTime(format, timezone="") +#### formatDateTime(format, timezone=NO_VALUE) #### gcd(b) @@ -2804,13 +2804,13 @@ Initializer. #### toDateTimeOrZero() -#### toDayOfMonth() +#### toDayOfMonth(timezone=NO_VALUE) -#### toDayOfWeek() +#### toDayOfWeek(timezone=NO_VALUE) -#### toDayOfYear() +#### toDayOfYear(timezone=NO_VALUE) #### toDecimal128(**kwargs) @@ -2861,7 +2861,7 @@ Initializer. #### toFloat64OrZero() -#### toHour() +#### toHour(timezone=NO_VALUE) #### toIPv4() @@ -2870,10 +2870,10 @@ Initializer. #### toIPv6() -#### toISOWeek(timezone="") +#### toISOWeek(timezone=NO_VALUE) -#### toISOYear(timezone="") +#### toISOYear(timezone=NO_VALUE) #### toInt16(**kwargs) @@ -2936,73 +2936,73 @@ Initializer. #### toIntervalYear() -#### toMinute() +#### toMinute(timezone=NO_VALUE) -#### toMonday() +#### toMonday(timezone=NO_VALUE) -#### toMonth() +#### toMonth(timezone=NO_VALUE) -#### toQuarter(timezone="") +#### toQuarter(timezone=NO_VALUE) -#### toRelativeDayNum(timezone="") +#### toRelativeDayNum(timezone=NO_VALUE) -#### toRelativeHourNum(timezone="") +#### toRelativeHourNum(timezone=NO_VALUE) -#### toRelativeMinuteNum(timezone="") +#### toRelativeMinuteNum(timezone=NO_VALUE) -#### toRelativeMonthNum(timezone="") +#### toRelativeMonthNum(timezone=NO_VALUE) -#### toRelativeSecondNum(timezone="") +#### toRelativeSecondNum(timezone=NO_VALUE) -#### toRelativeWeekNum(timezone="") +#### toRelativeWeekNum(timezone=NO_VALUE) -#### toRelativeYearNum(timezone="") +#### toRelativeYearNum(timezone=NO_VALUE) -#### toSecond() +#### toSecond(timezone=NO_VALUE) -#### toStartOfDay() +#### toStartOfDay(timezone=NO_VALUE) -#### toStartOfFifteenMinutes() +#### toStartOfFifteenMinutes(timezone=NO_VALUE) -#### toStartOfFiveMinute() +#### toStartOfFiveMinute(timezone=NO_VALUE) -#### toStartOfHour() +#### toStartOfHour(timezone=NO_VALUE) -#### toStartOfISOYear() +#### toStartOfISOYear(timezone=NO_VALUE) -#### toStartOfMinute() +#### toStartOfMinute(timezone=NO_VALUE) -#### toStartOfMonth() +#### toStartOfMonth(timezone=NO_VALUE) -#### toStartOfQuarter() +#### toStartOfQuarter(timezone=NO_VALUE) -#### toStartOfTenMinutes() +#### toStartOfTenMinutes(timezone=NO_VALUE) -#### toStartOfWeek(mode=0) +#### toStartOfWeek(timezone=NO_VALUE) -#### toStartOfYear() +#### toStartOfYear(timezone=NO_VALUE) #### toString() @@ -3011,7 +3011,7 @@ Initializer. #### toStringCutToZero() -#### toTime(timezone="") +#### toTime(timezone=NO_VALUE) #### toTimeZone(timezone) @@ -3056,22 +3056,22 @@ Initializer. #### toUUID() -#### toUnixTimestamp(timezone="") +#### toUnixTimestamp(timezone=NO_VALUE) -#### toWeek(mode=0, timezone="") +#### toWeek(mode=0, timezone=NO_VALUE) -#### toYYYYMM(timezone="") +#### toYYYYMM(timezone=NO_VALUE) -#### toYYYYMMDD(timezone="") +#### toYYYYMMDD(timezone=NO_VALUE) -#### toYYYYMMDDhhmmss(timezone="") +#### toYYYYMMDDhhmmss(timezone=NO_VALUE) -#### toYear() +#### toYear(timezone=NO_VALUE) #### to_sql(*args) @@ -3144,3 +3144,308 @@ For other functions: #### uniqExact(**kwargs) +#### uniqExactIf(*args) + + +#### uniqExactOrDefault() + + +#### uniqExactOrDefaultIf(*args) + + +#### uniqExactOrNull() + + +#### uniqExactOrNullIf(*args) + + +#### uniqHLL12(**kwargs) + + +#### uniqHLL12If(*args) + + +#### uniqHLL12OrDefault() + + +#### uniqHLL12OrDefaultIf(*args) + + +#### uniqHLL12OrNull() + + +#### uniqHLL12OrNullIf(*args) + + +#### uniqIf(*args) + + +#### uniqOrDefault() + + +#### uniqOrDefaultIf(*args) + + +#### uniqOrNull() + + +#### uniqOrNullIf(*args) + + +#### upper(**kwargs) + + +#### upperUTF8() + + +#### varPop(**kwargs) + + +#### varPopIf(cond) + + +#### varPopOrDefault() + + +#### varPopOrDefaultIf(cond) + + +#### varPopOrNull() + + +#### varPopOrNullIf(cond) + + +#### varSamp(**kwargs) + + +#### varSampIf(cond) + + +#### varSampOrDefault() + + +#### varSampOrDefaultIf(cond) + + +#### varSampOrNull() + + +#### varSampOrNullIf(cond) + + +#### xxHash32() + + +#### xxHash64() + + +#### yesterday() + + +clickhouse_orm.system_models +---------------------------- + +### SystemPart + +Extends Model + + +Contains information about parts of a table in the MergeTree family. +This model operates only fields, described in the reference. Other fields are ignored. +https://clickhouse.tech/docs/en/system_tables/system.parts/ + +#### SystemPart(**kwargs) + + +Creates a model instance, using keyword arguments as field values. +Since values are immediately converted to their Pythonic type, +invalid values will cause a `ValueError` to be raised. +Unrecognized field names will cause an `AttributeError`. + + +#### attach(settings=None) + + + Add a new part or partition from the 'detached' directory to the table. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.create_table_sql(db) + + +Returns the SQL statement for creating a table for this model. + + +#### detach(settings=None) + + +Move a partition to the 'detached' directory and forget it. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### drop(settings=None) + + +Delete a partition + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.drop_table_sql(db) + + +Returns the SQL command for deleting this model's table. + + +#### fetch(zookeeper_path, settings=None) + + +Download a partition from another server. + +- `zookeeper_path`: Path in zookeeper to fetch from +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.fields(writable=False) + + +Returns an `OrderedDict` of the model's fields (from name to `Field` instance). +If `writable` is true, only writable fields are included. +Callers should not modify the dictionary. + + +#### freeze(settings=None) + + +Create a backup of a partition. + +- `settings`: Settings for executing request to ClickHouse over db.raw() method + +Returns: SQL Query + + +#### SystemPart.from_tsv(line, field_names, timezone_in_use=UTC, database=None) + + +Create a model instance from a tab-separated line. The line may or may not include a newline. +The `field_names` list must match the fields defined in the model, but does not have to include all of them. + +- `line`: the TSV-formatted data. +- `field_names`: names of the model fields in the data. +- `timezone_in_use`: the timezone to use when parsing dates and datetimes. Some fields use their own timezones. +- `database`: if given, sets the database that this instance belongs to. + + +#### SystemPart.get(database, conditions="") + + +Get all data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database condition is added automatically + +Returns: A list of SystemPart objects + + +#### SystemPart.get_active(database, conditions="") + + +Gets active data from system.parts table + +- `database`: A database object to fetch data from. +- `conditions`: WHERE clause conditions. Database and active conditions are added automatically + +Returns: A list of SystemPart objects + + +#### get_database() + + +Gets the `Database` that this model instance belongs to. +Returns `None` unless the instance was read from the database or written to it. + + +#### get_field(name) + + +Gets a `Field` instance given its name, or `None` if not found. + + +#### SystemPart.has_funcs_as_defaults() + + +Return True if some of the model's fields use a function expression +as a default value. This requires special handling when inserting instances. + + +#### SystemPart.is_read_only() + + +Returns true if the model is marked as read only. + + +#### SystemPart.is_system_model() + + +Returns true if the model represents a system table. + + +#### SystemPart.objects_in(database) + + +Returns a `QuerySet` for selecting instances of this model class. + + +#### set_database(db) + + +Sets the `Database` that this model instance belongs to. +This is done automatically when the instance is read from the database or written to it. + + +#### SystemPart.table_name() + + +#### to_db_string() + + +Returns the instance as a bytestring ready to be inserted into the database. + + +#### to_dict(include_readonly=True, field_names=None) + + +Returns the instance's column values as a dict. + +- `include_readonly`: if false, returns only fields that can be inserted into database. +- `field_names`: an iterable of field names to return (optional) + + +#### to_tskv(include_readonly=True) + + +Returns the instance's column keys and values as a tab-separated line. A newline is not included. +Fields that were not assigned a value are omitted. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + +#### to_tsv(include_readonly=True) + + +Returns the instance's column values as a tab-separated line. A newline is not included. + +- `include_readonly`: if false, returns only fields that can be inserted into database. + + diff --git a/docs/toc.md b/docs/toc.md index e973772..57b7f02 100644 --- a/docs/toc.md +++ b/docs/toc.md @@ -78,17 +78,17 @@ * [Tests](contributing.md#tests) * [Class Reference](class_reference.md#class-reference) - * [clickhouse_orm.database](class_reference.md#inficlickhouse_ormdatabase) + * [clickhouse_orm.database](class_reference.md#clickhouse_ormdatabase) * [Database](class_reference.md#database) * [DatabaseException](class_reference.md#databaseexception) - * [clickhouse_orm.models](class_reference.md#inficlickhouse_ormmodels) + * [clickhouse_orm.models](class_reference.md#clickhouse_ormmodels) * [Model](class_reference.md#model) * [BufferModel](class_reference.md#buffermodel) * [MergeModel](class_reference.md#mergemodel) * [DistributedModel](class_reference.md#distributedmodel) * [Constraint](class_reference.md#constraint) * [Index](class_reference.md#index) - * [clickhouse_orm.fields](class_reference.md#inficlickhouse_ormfields) + * [clickhouse_orm.fields](class_reference.md#clickhouse_ormfields) * [ArrayField](class_reference.md#arrayfield) * [BaseEnumField](class_reference.md#baseenumfield) * [BaseFloatField](class_reference.md#basefloatfield) @@ -120,7 +120,7 @@ * [UInt64Field](class_reference.md#uint64field) * [UInt8Field](class_reference.md#uint8field) * [UUIDField](class_reference.md#uuidfield) - * [clickhouse_orm.engines](class_reference.md#inficlickhouse_ormengines) + * [clickhouse_orm.engines](class_reference.md#clickhouse_ormengines) * [Engine](class_reference.md#engine) * [TinyLog](class_reference.md#tinylog) * [Log](class_reference.md#log) @@ -132,10 +132,12 @@ * [CollapsingMergeTree](class_reference.md#collapsingmergetree) * [SummingMergeTree](class_reference.md#summingmergetree) * [ReplacingMergeTree](class_reference.md#replacingmergetree) - * [clickhouse_orm.query](class_reference.md#inficlickhouse_ormquery) + * [clickhouse_orm.query](class_reference.md#clickhouse_ormquery) * [QuerySet](class_reference.md#queryset) * [AggregateQuerySet](class_reference.md#aggregatequeryset) * [Q](class_reference.md#q) - * [clickhouse_orm.funcs](class_reference.md#inficlickhouse_ormfuncs) + * [clickhouse_orm.funcs](class_reference.md#clickhouse_ormfuncs) * [F](class_reference.md#f) + * [clickhouse_orm.system_models](class_reference.md#clickhouse_ormsystem_models) + * [SystemPart](class_reference.md#systempart) diff --git a/pyproject.toml b/pyproject.toml index 98859a5..8eb10c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ line-length = 120 extend-exclude = ''' /( | examples - | scripts )/ ''' diff --git a/scripts/docs2html.sh b/scripts/docs2html.sh index 15dc07e..73d6613 100755 --- a/scripts/docs2html.sh +++ b/scripts/docs2html.sh @@ -1,4 +1,4 @@ - +#!/bin/bash mkdir -p ../htmldocs find ./ -iname "*.md" -type f -exec sh -c 'echo "Converting ${0}"; pandoc "${0}" -s -o "../htmldocs/${0%.md}.html"' {} \; diff --git a/scripts/generate_all.sh b/scripts/generate_all.sh index eabf65c..db4a328 100755 --- a/scripts/generate_all.sh +++ b/scripts/generate_all.sh @@ -1,5 +1,6 @@ +#!/bin/bash # Class reference -../bin/python ../scripts/generate_ref.py > class_reference.md +poetry run python ../scripts/generate_ref.py > class_reference.md # Table of contents ../scripts/generate_toc.sh diff --git a/scripts/generate_ref.py b/scripts/generate_ref.py index 16473b4..7d1fe5a 100644 --- a/scripts/generate_ref.py +++ b/scripts/generate_ref.py @@ -1,11 +1,11 @@ - import inspect from collections import namedtuple -DefaultArgSpec = namedtuple('DefaultArgSpec', 'has_default default_value') +DefaultArgSpec = namedtuple("DefaultArgSpec", "has_default default_value") + def _get_default_arg(args, defaults, arg_index): - """ Method that determines if an argument has default value or not, + """Method that determines if an argument has default value or not, and if yes what is the default value for the argument :param args: array of arguments, eg: ['first_arg', 'second_arg', 'third_arg'] @@ -25,12 +25,13 @@ def _get_default_arg(args, defaults, arg_index): return DefaultArgSpec(False, None) else: value = defaults[arg_index - args_with_no_defaults] - if (type(value) is str): + if type(value) is str: value = '"%s"' % value return DefaultArgSpec(True, value) + def get_method_sig(method): - """ Given a function, it returns a string that pretty much looks how the + """Given a function, it returns a string that pretty much looks how the function signature would be written in python. :param method: a python method @@ -42,31 +43,37 @@ def get_method_sig(method): # list of defaults are returned in separate array. # eg: ArgSpec(args=['first_arg', 'second_arg', 'third_arg'], # varargs=None, keywords=None, defaults=(42, 'something')) - argspec = inspect.getargspec(method) - arg_index=0 + argspec = inspect.getfullargspec(method) args = [] # Use the args and defaults array returned by argspec and find out # which arguments has default - for arg in argspec.args: - default_arg = _get_default_arg(argspec.args, argspec.defaults, arg_index) + for idx, arg in enumerate(argspec.args): + default_arg = _get_default_arg(argspec.args, argspec.defaults, idx) + if default_arg.has_default: + val = default_arg.default_value + args.append("%s=%s" % (arg, val)) + else: + args.append(arg) + + for idx, arg in enumerate(argspec.kwonlyargs): + default_arg = _get_default_arg(argspec.kwonlyargs, argspec.kwonlydefaults, idx) if default_arg.has_default: val = default_arg.default_value args.append("%s=%s" % (arg, val)) else: args.append(arg) - arg_index += 1 if argspec.varargs: - args.append('*' + argspec.varargs) - if argspec.keywords: - args.append('**' + argspec.keywords) + args.append("*" + argspec.varargs) + if argspec.varkw: + args.append("**" + argspec.varkw) return "%s(%s)" % (method.__name__, ", ".join(args[1:])) def docstring(obj): - doc = (obj.__doc__ or '').rstrip() + doc = (obj.__doc__ or "").rstrip() if doc: - lines = doc.split('\n') + lines = doc.split("\n") # Find the length of the whitespace prefix common to all non-empty lines indentation = min(len(line) - len(line.lstrip()) for line in lines if line.strip()) # Output the lines without the indentation @@ -76,30 +83,30 @@ def docstring(obj): def class_doc(cls, list_methods=True): - bases = ', '.join([b.__name__ for b in cls.__bases__]) - print('###', cls.__name__) + bases = ", ".join([b.__name__ for b in cls.__bases__]) + print("###", cls.__name__) print() - if bases != 'object': - print('Extends', bases) + if bases != "object": + print("Extends", bases) print() docstring(cls) for name, method in inspect.getmembers(cls, lambda m: inspect.ismethod(m) or inspect.isfunction(m)): - if name == '__init__': + if name == "__init__": # Initializer - print('####', get_method_sig(method).replace(name, cls.__name__)) - elif name[0] == '_': + print("####", get_method_sig(method).replace(name, cls.__name__)) + elif name[0] == "_": # Private method continue - elif hasattr(method, '__self__') and method.__self__ == cls: + elif hasattr(method, "__self__") and method.__self__ == cls: # Class method if not list_methods: continue - print('#### %s.%s' % (cls.__name__, get_method_sig(method))) + print("#### %s.%s" % (cls.__name__, get_method_sig(method))) else: # Regular method if not list_methods: continue - print('####', get_method_sig(method)) + print("####", get_method_sig(method)) print() docstring(method) print() @@ -108,7 +115,7 @@ def class_doc(cls, list_methods=True): def module_doc(classes, list_methods=True): mdl = classes[0].__module__ print(mdl) - print('-' * len(mdl)) + print("-" * len(mdl)) print() for cls in classes: class_doc(cls, list_methods) @@ -118,21 +125,17 @@ def all_subclasses(cls): return cls.__subclasses__() + [g for s in cls.__subclasses__() for g in all_subclasses(s)] -if __name__ == '__main__': +if __name__ == "__main__": - from clickhouse_orm import database - from clickhouse_orm import fields - from clickhouse_orm import engines - from clickhouse_orm import models - from clickhouse_orm import query - from clickhouse_orm import funcs - from clickhouse_orm import system_models + from clickhouse_orm import database, engines, fields, funcs, models, query, system_models - print('Class Reference') - print('===============') + print("Class Reference") + print("===============") print() module_doc([database.Database, database.DatabaseException]) - module_doc([models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index]) + module_doc( + [models.Model, models.BufferModel, models.MergeModel, models.DistributedModel, models.Constraint, models.Index] + ) module_doc(sorted([fields.Field] + all_subclasses(fields.Field), key=lambda x: x.__name__), False) module_doc([engines.Engine] + all_subclasses(engines.Engine), False) module_doc([query.QuerySet, query.AggregateQuerySet, query.Q]) diff --git a/scripts/generate_toc.sh b/scripts/generate_toc.sh index a77aaaa..bc0db33 100755 --- a/scripts/generate_toc.sh +++ b/scripts/generate_toc.sh @@ -1,7 +1,7 @@ - +#!/bin/bash generate_one() { # Converts Markdown to HTML using Pandoc, and then extracts the header tags - pandoc "$1" | python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md + pandoc "$1" | poetry run python "../scripts/html_to_markdown_toc.py" "$1" >> toc.md } printf "# Table of Contents\n\n" > toc.md diff --git a/scripts/html_to_markdown_toc.py b/scripts/html_to_markdown_toc.py index 552137f..956732c 100644 --- a/scripts/html_to_markdown_toc.py +++ b/scripts/html_to_markdown_toc.py @@ -1,14 +1,13 @@ -from html.parser import HTMLParser import sys +from html.parser import HTMLParser - -HEADER_TAGS = ('h1', 'h2', 'h3') +HEADER_TAGS = ("h1", "h2", "h3") class HeadersToMarkdownParser(HTMLParser): inside = None - text = '' + text = "" def handle_starttag(self, tag, attrs): if tag.lower() in HEADER_TAGS: @@ -16,11 +15,11 @@ def handle_starttag(self, tag, attrs): def handle_endtag(self, tag): if tag.lower() in HEADER_TAGS: - indent = ' ' * int(self.inside[1]) - fragment = self.text.lower().replace(' ', '-').replace('.', '') - print('%s* [%s](%s#%s)' % (indent, self.text, sys.argv[1], fragment)) + indent = " " * int(self.inside[1]) + fragment = self.text.lower().replace(" ", "-").replace(".", "") + print("%s* [%s](%s#%s)" % (indent, self.text, sys.argv[1], fragment)) self.inside = None - self.text = '' + self.text = "" def handle_data(self, data): if self.inside: @@ -28,4 +27,4 @@ def handle_data(self, data): HeadersToMarkdownParser().feed(sys.stdin.read()) -print('') +print("") diff --git a/scripts/test_python3.sh b/scripts/test_python3.sh deleted file mode 100755 index 455d5b7..0000000 --- a/scripts/test_python3.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -cd /tmp -rm -rf /tmp/orm_env* -virtualenv -p python3 /tmp/orm_env -cd /tmp/orm_env -source bin/activate -pip install infi.projector -git clone https://github.com/Infinidat/clickhouse_orm.git -cd clickhouse_orm -projector devenv build -bin/nosetests diff --git a/setup.cfg b/setup.cfg index 48f3a58..ab5c925 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,4 +18,3 @@ ignore = exclude = tests/sample_migrations examples - scripts From ce79b394072bba66c84d2bc680e179c2d69c5479 Mon Sep 17 00:00:00 2001 From: olliemath Date: Mon, 16 Aug 2021 09:44:48 +0100 Subject: [PATCH 43/51] Chore: fix linting for examples --- examples/cpu_usage/collect.py | 21 ++++-- examples/cpu_usage/models.py | 3 +- examples/cpu_usage/results.py | 8 +- examples/db_explorer/charts.py | 45 ++++++----- examples/db_explorer/server.py | 78 +++++++++++--------- examples/full_text_search/download_ebooks.py | 17 +++-- examples/full_text_search/load.py | 35 +++++---- examples/full_text_search/models.py | 16 ++-- examples/full_text_search/search.py | 41 +++++----- pyproject.toml | 5 -- setup.cfg | 1 - 11 files changed, 146 insertions(+), 124 deletions(-) diff --git a/examples/cpu_usage/collect.py b/examples/cpu_usage/collect.py index 62102ac..0aab1c4 100644 --- a/examples/cpu_usage/collect.py +++ b/examples/cpu_usage/collect.py @@ -1,20 +1,25 @@ -import psutil, time, datetime -from clickhouse_orm import Database +import datetime +import time + +import psutil from models import CPUStats +from clickhouse_orm import Database -db = Database('demo') +db = Database("demo") db.create_table(CPUStats) -psutil.cpu_percent(percpu=True) # first sample should be discarded +psutil.cpu_percent(percpu=True) # first sample should be discarded while True: time.sleep(1) stats = psutil.cpu_percent(percpu=True) timestamp = datetime.datetime.now() print(timestamp) - db.insert([ - CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent) - for cpu_id, cpu_percent in enumerate(stats) - ]) + db.insert( + [ + CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent) + for cpu_id, cpu_percent in enumerate(stats) + ] + ) diff --git a/examples/cpu_usage/models.py b/examples/cpu_usage/models.py index fe9afc6..6bba423 100644 --- a/examples/cpu_usage/models.py +++ b/examples/cpu_usage/models.py @@ -1,4 +1,4 @@ -from clickhouse_orm import Model, DateTimeField, UInt16Field, Float32Field, Memory +from clickhouse_orm import DateTimeField, Float32Field, Memory, Model, UInt16Field class CPUStats(Model): @@ -8,4 +8,3 @@ class CPUStats(Model): cpu_percent = Float32Field() engine = Memory() - diff --git a/examples/cpu_usage/results.py b/examples/cpu_usage/results.py index 06ee1f0..f7ef9b1 100644 --- a/examples/cpu_usage/results.py +++ b/examples/cpu_usage/results.py @@ -1,13 +1,13 @@ -from clickhouse_orm import Database, F from models import CPUStats +from clickhouse_orm import Database, F -db = Database('demo') +db = Database("demo") queryset = CPUStats.objects_in(db) total = queryset.filter(CPUStats.cpu_id == 1).count() busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count() -print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total)) +print("CPU 1 was busy {:.2f}% of the time".format(busy * 100.0 / total)) # Calculate the average usage per CPU for row in queryset.aggregate(CPUStats.cpu_id, average=F.avg(CPUStats.cpu_percent)): - print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row)) + print("CPU {row.cpu_id}: {row.average:.2f}%".format(row=row)) diff --git a/examples/db_explorer/charts.py b/examples/db_explorer/charts.py index 1a690e6..1b01d56 100644 --- a/examples/db_explorer/charts.py +++ b/examples/db_explorer/charts.py @@ -1,62 +1,73 @@ import pygal -from pygal.style import RotateStyle from jinja2.filters import do_filesizeformat +from pygal.style import RotateStyle # Formatting functions -number_formatter = lambda v: '{:,}'.format(v) -bytes_formatter = lambda v: do_filesizeformat(v, True) +def number_formatter(v): + return "{:,}".format(v) + + +def bytes_formatter(v): + do_filesizeformat(v, True) def tables_piechart(db, by_field, value_formatter): - ''' + """ Generate a pie chart of the top n tables in the database. `db` - the database instance `by_field` - the field name to sort by `value_formatter` - a function to use for formatting the numeric values - ''' - Tables = db.get_model_for_table('tables', system_table=True) - qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer') + """ + Tables = db.get_model_for_table("tables", system_table=True) + qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine="Buffer") tuples = [(getattr(table, by_field), table.name) for table in qs] return _generate_piechart(tuples, value_formatter) def columns_piechart(db, tbl_name, by_field, value_formatter): - ''' + """ Generate a pie chart of the top n columns in the table. `db` - the database instance `tbl_name` - the table name `by_field` - the field name to sort by `value_formatter` - a function to use for formatting the numeric values - ''' - ColumnsTable = db.get_model_for_table('columns', system_table=True) + """ + ColumnsTable = db.get_model_for_table("columns", system_table=True) qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name) tuples = [(getattr(col, by_field), col.name) for col in qs] return _generate_piechart(tuples, value_formatter) def _get_top_tuples(tuples, n=15): - ''' + """ Given a list of tuples (value, name), this function sorts the list and returns only the top n results. All other tuples are aggregated to a single "others" tuple. - ''' + """ non_zero_tuples = [t for t in tuples if t[0]] sorted_tuples = sorted(non_zero_tuples, reverse=True) if len(sorted_tuples) > n: - others = (sum(t[0] for t in sorted_tuples[n:]), 'others') + others = (sum(t[0] for t in sorted_tuples[n:]), "others") sorted_tuples = sorted_tuples[:n] + [others] return sorted_tuples def _generate_piechart(tuples, value_formatter): - ''' + """ Generates a pie chart. `tuples` - a list of (value, name) tuples to include in the chart `value_formatter` - a function to use for formatting the values - ''' - style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24) - chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1) + """ + style = RotateStyle( + "#9e6ffe", + background="white", + legend_font_family="Roboto", + legend_font_size=18, + tooltip_font_family="Roboto", + tooltip_font_size=24, + ) + chart = pygal.Pie(style=style, margin=0, title=" ", value_formatter=value_formatter, truncate_legend=-1) for t in _get_top_tuples(tuples): chart.add(t[1], t[0]) return chart.render(is_unicode=True, disable_xml_declaration=True) diff --git a/examples/db_explorer/server.py b/examples/db_explorer/server.py index 9fbcc03..d1406e1 100644 --- a/examples/db_explorer/server.py +++ b/examples/db_explorer/server.py @@ -1,87 +1,93 @@ -from clickhouse_orm import Database, F -from charts import tables_piechart, columns_piechart, number_formatter, bytes_formatter -from flask import Flask -from flask import render_template import sys +from charts import bytes_formatter, columns_piechart, number_formatter, tables_piechart +from flask import Flask, render_template + +from clickhouse_orm import Database, F app = Flask(__name__) -@app.route('/') +@app.route("/") def homepage_view(): - ''' + """ Root view that lists all databases. - ''' - db = _get_db('system') + """ + db = _get_db("system") # Get all databases in the system.databases table - DatabasesTable = db.get_model_for_table('databases', system_table=True) - databases = DatabasesTable.objects_in(db).exclude(name='system') + DatabasesTable = db.get_model_for_table("databases", system_table=True) + databases = DatabasesTable.objects_in(db).exclude(name="system") databases = databases.order_by(F.lower(DatabasesTable.name)) # Generate the page - return render_template('homepage.html', db=db, databases=databases) + return render_template("homepage.html", db=db, databases=databases) -@app.route('//') +@app.route("//") def database_view(db_name): - ''' + """ A view that displays information about a single database. - ''' + """ db = _get_db(db_name) # Get all the tables in the database, by aggregating information from system.columns - ColumnsTable = db.get_model_for_table('columns', system_table=True) - tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate( - ColumnsTable.table, - compressed_size=F.sum(ColumnsTable.data_compressed_bytes), - uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes), - ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes) + ColumnsTable = db.get_model_for_table("columns", system_table=True) + tables = ( + ColumnsTable.objects_in(db) + .filter(database=db_name) + .aggregate( + ColumnsTable.table, + compressed_size=F.sum(ColumnsTable.data_compressed_bytes), + uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes), + ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes), + ) ) tables = tables.order_by(F.lower(ColumnsTable.table)) # Generate the page - return render_template('database.html', + return render_template( + "database.html", db=db, tables=tables, - tables_piechart_by_rows=tables_piechart(db, 'total_rows', value_formatter=number_formatter), - tables_piechart_by_size=tables_piechart(db, 'total_bytes', value_formatter=bytes_formatter), + tables_piechart_by_rows=tables_piechart(db, "total_rows", value_formatter=number_formatter), + tables_piechart_by_size=tables_piechart(db, "total_bytes", value_formatter=bytes_formatter), ) -@app.route('///') +@app.route("///") def table_view(db_name, tbl_name): - ''' + """ A view that displays information about a single table. - ''' + """ db = _get_db(db_name) # Get table information from system.tables - TablesTable = db.get_model_for_table('tables', system_table=True) + TablesTable = db.get_model_for_table("tables", system_table=True) tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0] # Get the SQL used for creating the table - create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name) + create_table_sql = db.raw("SHOW CREATE TABLE %s FORMAT TabSeparatedRaw" % tbl_name) # Get all columns in the table from system.columns - ColumnsTable = db.get_model_for_table('columns', system_table=True) + ColumnsTable = db.get_model_for_table("columns", system_table=True) columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name) # Generate the page - return render_template('table.html', + return render_template( + "table.html", db=db, tbl_name=tbl_name, tbl_info=tbl_info, create_table_sql=create_table_sql, columns=columns, - piechart=columns_piechart(db, tbl_name, 'data_compressed_bytes', value_formatter=bytes_formatter), + piechart=columns_piechart(db, tbl_name, "data_compressed_bytes", value_formatter=bytes_formatter), ) def _get_db(db_name): - ''' + """ Returns a Database instance using connection information from the command line arguments (optional). - ''' - db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/' + """ + db_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8123/" username = sys.argv[2] if len(sys.argv) > 2 else None password = sys.argv[3] if len(sys.argv) > 3 else None return Database(db_name, db_url, username, password, readonly=True) -if __name__ == '__main__': - _get_db('system') # fail early on db connection problems +if __name__ == "__main__": + _get_db("system") # fail early on db connection problems app.run(debug=True) diff --git a/examples/full_text_search/download_ebooks.py b/examples/full_text_search/download_ebooks.py index 170d5e1..865e59c 100644 --- a/examples/full_text_search/download_ebooks.py +++ b/examples/full_text_search/download_ebooks.py @@ -1,27 +1,28 @@ -import requests import os +import requests + def download_ebook(id): - print(id, end=' ') + print(id, end=" ") # Download the ebook's text - r = requests.get('https://www.gutenberg.org/files/{id}/{id}-0.txt'.format(id=id)) + r = requests.get("https://www.gutenberg.org/files/{id}/{id}-0.txt".format(id=id)) if r.status_code == 404: - print('NOT FOUND, SKIPPING') + print("NOT FOUND, SKIPPING") return r.raise_for_status() # Find the ebook's title - text = r.content.decode('utf-8') + text = r.content.decode("utf-8") for line in text.splitlines(): - if line.startswith('Title:'): + if line.startswith("Title:"): title = line[6:].strip() print(title) # Save the ebook - with open('ebooks/{}.txt'.format(title), 'wb') as f: + with open("ebooks/{}.txt".format(title), "wb") as f: f.write(r.content) if __name__ == "__main__": - os.makedirs('ebooks', exist_ok=True) + os.makedirs("ebooks", exist_ok=True) for i in [1342, 11, 84, 2701, 25525, 1661, 98, 74, 43, 215, 1400, 76]: download_ebook(i) diff --git a/examples/full_text_search/load.py b/examples/full_text_search/load.py index 51564f4..225b155 100644 --- a/examples/full_text_search/load.py +++ b/examples/full_text_search/load.py @@ -1,61 +1,64 @@ import sys +from glob import glob + import nltk +from models import Fragment from nltk.stem.porter import PorterStemmer -from glob import glob + from clickhouse_orm import Database -from models import Fragment def trim_punctuation(word): - ''' + """ Trim punctuation characters from the beginning and end of the word - ''' + """ start = end = len(word) for i in range(len(word)): if word[i].isalnum(): start = min(start, i) end = i + 1 - return word[start : end] + return word[start:end] def parse_file(filename): - ''' + """ Parses a text file at the give path. Returns a generator of tuples (original_word, stemmed_word) The original_word may include punctuation characters. - ''' + """ stemmer = PorterStemmer() - with open(filename, 'r', encoding='utf-8') as f: + with open(filename, "r", encoding="utf-8") as f: for line in f: for word in line.split(): yield (word, stemmer.stem(trim_punctuation(word))) def get_fragments(filename): - ''' + """ Converts a text file at the given path to a generator of Fragment instances. - ''' + """ from os import path + document = path.splitext(path.basename(filename))[0] idx = 0 for word, stem in parse_file(filename): idx += 1 yield Fragment(document=document, idx=idx, word=word, stem=stem) - print('{} - {} words'.format(filename, idx)) + print("{} - {} words".format(filename, idx)) -if __name__ == '__main__': +if __name__ == "__main__": # Load NLTK data if necessary - nltk.download('punkt') - nltk.download('wordnet') + nltk.download("punkt") + nltk.download("wordnet") # Initialize database - db = Database('default') + db = Database("default") db.create_table(Fragment) # Load files from the command line or everything under ebooks/ - filenames = sys.argv[1:] or glob('ebooks/*.txt') + filenames = sys.argv[1:] or glob("ebooks/*.txt") for filename in filenames: db.insert(get_fragments(filename), batch_size=100000) diff --git a/examples/full_text_search/models.py b/examples/full_text_search/models.py index 80de2b7..ae4c666 100644 --- a/examples/full_text_search/models.py +++ b/examples/full_text_search/models.py @@ -1,16 +1,18 @@ -from clickhouse_orm import * +from clickhouse_orm.engines import MergeTree +from clickhouse_orm.fields import LowCardinalityField, StringField, UInt64Field +from clickhouse_orm.models import Index, Model class Fragment(Model): - language = LowCardinalityField(StringField(), default='EN') + language = LowCardinalityField(StringField(), default="EN") document = LowCardinalityField(StringField()) - idx = UInt64Field() - word = StringField() - stem = StringField() + idx = UInt64Field() + word = StringField() + stem = StringField() # An index for faster search by document and fragment idx - index = Index((document, idx), type=Index.minmax(), granularity=1) + index = Index((document, idx), type=Index.minmax(), granularity=1) # The primary key allows efficient lookup of stems - engine = MergeTree(order_by=(stem, document, idx), partition_key=('language',)) + engine = MergeTree(order_by=(stem, document, idx), partition_key=("language",)) diff --git a/examples/full_text_search/search.py b/examples/full_text_search/search.py index c4d0918..4175929 100644 --- a/examples/full_text_search/search.py +++ b/examples/full_text_search/search.py @@ -1,19 +1,20 @@ import sys -from colorama import init, Fore, Back, Style -from nltk.stem.porter import PorterStemmer -from clickhouse_orm import Database, F -from models import Fragment + +from colorama import Fore, Style, init from load import trim_punctuation +from models import Fragment +from nltk.stem.porter import PorterStemmer +from clickhouse_orm import Database, F # The wildcard character -WILDCARD = '*' +WILDCARD = "*" def prepare_search_terms(text): - ''' + """ Convert the text to search into a list of stemmed words. - ''' + """ stemmer = PorterStemmer() stems = [] for word in text.split(): @@ -25,10 +26,10 @@ def prepare_search_terms(text): def build_query(db, stems): - ''' + """ Returns a queryset instance for finding sequences of Fragment instances that matche the list of stemmed words. - ''' + """ # Start by searching for the first stemmed word all_fragments = Fragment.objects_in(db) query = all_fragments.filter(stem=stems[0]).only(Fragment.document, Fragment.idx) @@ -47,44 +48,44 @@ def build_query(db, stems): def get_matching_text(db, document, from_idx, to_idx, extra=5): - ''' + """ Reconstructs the document text between the given indexes (inclusive), plus `extra` words before and after the match. The words that are included in the given range are highlighted in green. - ''' + """ text = [] conds = (Fragment.document == document) & (Fragment.idx >= from_idx - extra) & (Fragment.idx <= to_idx + extra) - for fragment in Fragment.objects_in(db).filter(conds).order_by('document', 'idx'): + for fragment in Fragment.objects_in(db).filter(conds).order_by("document", "idx"): word = fragment.word if fragment.idx == from_idx: word = Fore.GREEN + word if fragment.idx == to_idx: word = word + Style.RESET_ALL text.append(word) - return ' '.join(text) + return " ".join(text) def find(db, text): - ''' + """ Performs the search for the given text, and prints out the matches. - ''' + """ stems = prepare_search_terms(text) query = build_query(db, stems) - print('\n' + Fore.MAGENTA + str(query) + Style.RESET_ALL + '\n') + print("\n" + Fore.MAGENTA + str(query) + Style.RESET_ALL + "\n") for match in query: text = get_matching_text(db, match.document, match.idx, match.idx + len(stems) - 1) - print(Fore.CYAN + match.document + ':' + Style.RESET_ALL, text) + print(Fore.CYAN + match.document + ":" + Style.RESET_ALL, text) -if __name__ == '__main__': +if __name__ == "__main__": # Initialize colored output init() # Initialize database - db = Database('default') + db = Database("default") # Search - text = ' '.join(sys.argv[1:]) + text = " ".join(sys.argv[1:]) if text: find(db, text) diff --git a/pyproject.toml b/pyproject.toml index 8eb10c4..70e5f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,5 @@ [tool.black] line-length = 120 -extend-exclude = ''' -/( - | examples -)/ -''' [tool.isort] multi_line_output = 3 diff --git a/setup.cfg b/setup.cfg index ab5c925..d43c3f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,4 +17,3 @@ ignore = B950 # We use E501 exclude = tests/sample_migrations - examples From c357aad8e73aca538e076c746290d6ea53b44c14 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Mon, 16 Aug 2021 18:26:07 +0000 Subject: [PATCH 44/51] Create python-publish.yml --- .github/workflows/python-publish.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..95f8360 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,40 @@ +name: Upload to PIP + +# Controls when the action will run. +on: + # Triggers the workflow when a release is created + release: + types: [created] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "upload" + upload: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Sets up python + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + # Install dependencies + - name: Install dependencies + run: | + python -m pip install poetry + poetry install --no-dev + + # Build and upload to PyPI + - name: Builds and upload to PyPI + run: | + poetry config pypi-token.pypi "$TWINE_TOKEN" + poetry publish + env: + TWINE_TOKEN: ${{ secrets.TWINE_TOKEN }} From 223febdc8200d731c8eb77ae0d30d4515d82770a Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Mon, 16 Aug 2021 19:07:01 +0000 Subject: [PATCH 45/51] Update python-publish.yml --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 95f8360..5b62b83 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -35,6 +35,6 @@ jobs: - name: Builds and upload to PyPI run: | poetry config pypi-token.pypi "$TWINE_TOKEN" - poetry publish + poetry publish --build env: TWINE_TOKEN: ${{ secrets.TWINE_TOKEN }} From 441d2d1f08b32fd037a32569584ef42bfeb593fc Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Mon, 16 Aug 2021 19:13:05 +0000 Subject: [PATCH 46/51] Rename python-package.yml to python-test.yml --- .github/workflows/{python-package.yml => python-test.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{python-package.yml => python-test.yml} (100%) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-test.yml similarity index 100% rename from .github/workflows/python-package.yml rename to .github/workflows/python-test.yml From 22e124e3aa2fac3509f7a3d8add49b970d4d2af2 Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Mon, 16 Aug 2021 19:17:40 +0000 Subject: [PATCH 47/51] Moar badges --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 972859e..a45a2eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) aimed at more frequent maintenance and bugfixes. -[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-package.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-package.yml) +[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml) +![PyPI](https://img.shields.io/pypi/v/clickhouse_orm) Introduction ============ From 35b5a11e65f9eb6d1f5b795a0e084d84b2b80f7f Mon Sep 17 00:00:00 2001 From: Oliver Margetts Date: Mon, 16 Aug 2021 19:19:34 +0000 Subject: [PATCH 48/51] Chore: improve description for PyPI --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 70e5f06..1ce6ff3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ line_length = 120 [tool.poetry] name = "clickhouse_orm" version = "2.2.0" -description = "A simple ORM for working with the Clickhouse database" +description = "A simple ORM for working with the Clickhouse database. Maintainance fork of infi.clickhouse_orm." authors = ["olliemath "] license = "BSD" homepage = "https://github.com/SuadeLabs/clickhouse_orm" From 23cef67da056b0ff3d2d5eddabf16ee5fad0378b Mon Sep 17 00:00:00 2001 From: olliemath Date: Mon, 16 Aug 2021 20:21:46 +0100 Subject: [PATCH 49/51] Prep 2.2.1 release --- CHANGELOG.md | 5 +++++ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0e0a7..6eb8958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Change Log ========== +v2.2.1 +------ +- Minor tooling changes for PyPI +- Better project description for PyPI + v2.2.0 ------ - Support up to clickhouse 20.12, including LTS 20.8 release diff --git a/pyproject.toml b/pyproject.toml index 1ce6ff3..8b74e1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ line_length = 120 [tool.poetry] name = "clickhouse_orm" -version = "2.2.0" +version = "2.2.1" description = "A simple ORM for working with the Clickhouse database. Maintainance fork of infi.clickhouse_orm." authors = ["olliemath "] license = "BSD" From cd724c213224583bff5e78cc63221ecec769c317 Mon Sep 17 00:00:00 2001 From: olliemath Date: Fri, 20 Aug 2021 14:08:17 +0100 Subject: [PATCH 50/51] Tooling: unpin requirements --- CHANGELOG.md | 4 ++++ pyproject.toml | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eb8958..d4110e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ Change Log ========== +v2.2.2 +------ +- Unpined requirements to enhance compatability + v2.2.1 ------ - Minor tooling changes for PyPI diff --git a/pyproject.toml b/pyproject.toml index 8b74e1a..093a67b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ line_length = 120 [tool.poetry] name = "clickhouse_orm" -version = "2.2.1" +version = "2.2.2" description = "A simple ORM for working with the Clickhouse database. Maintainance fork of infi.clickhouse_orm." authors = ["olliemath "] license = "BSD" @@ -33,9 +33,9 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.6.2,<4" -requests = "^2.26.0" -pytz = "^2021.1" -iso8601 = "^0.1.16" +requests = "*" +pytz = "*" +iso8601 = "*" [tool.poetry.dev-dependencies] flake8 = "^3.9.2" From d7d30e8e65326108851b5844306f429731b5a163 Mon Sep 17 00:00:00 2001 From: Beda Kosata Date: Sun, 3 Oct 2021 16:19:00 +0200 Subject: [PATCH 51/51] add support for arbitrary SETTINGS in CREATE TABLE --- clickhouse_orm/engines.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clickhouse_orm/engines.py b/clickhouse_orm/engines.py index af63af7..b8b5043 100644 --- a/clickhouse_orm/engines.py +++ b/clickhouse_orm/engines.py @@ -36,6 +36,7 @@ def __init__( replica_name=None, partition_key=None, primary_key=None, + settings=None, ): assert type(order_by) in (list, tuple), "order_by must be a list or tuple" assert date_col is None or isinstance(date_col, str), "date_col must be string if present" @@ -47,6 +48,7 @@ def __init__( assert (replica_table_path is None) == ( replica_name is None ), "both replica_table_path and replica_name must be specified" + assert settings is None or type(settings) is dict, 'settings must be dict' # These values conflict with each other (old and new syntax of table engines. # So let's control only one of them is given. @@ -60,6 +62,7 @@ def __init__( self.index_granularity = index_granularity self.replica_table_path = replica_table_path self.replica_name = replica_name + self.settings = settings # I changed field name for new reality and syntax @property @@ -97,6 +100,9 @@ def create_table_sql(self, db): partition_sql += " SAMPLE BY %s" % self.sampling_expr partition_sql += " SETTINGS index_granularity=%d" % self.index_granularity + if self.settings: + settings_sql = ", ".join('%s=%s' % (key, value) for key, value in self.settings.items()) + partition_sql += ", " + settings_sql elif not self.date_col: # Can't import it globally due to circular import @@ -144,6 +150,7 @@ def __init__( replica_name=None, partition_key=None, primary_key=None, + settings=None, ): super(CollapsingMergeTree, self).__init__( date_col, @@ -154,6 +161,7 @@ def __init__( replica_name, partition_key, primary_key, + settings=settings, ) self.sign_col = sign_col