From d0add4849a34ec38241b23f5f33c81878b447eba Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 15:50:05 +0900 Subject: [PATCH 01/12] Add pre-commit Signed-off-by: Yu Ishikawa --- .pre-commit-config.yaml | 55 +++++++++++++++++ bigfun/cli.py | 127 ++++++++++++++++++++++++---------------- setup.py | 5 ++ 3 files changed, 138 insertions(+), 49 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..52de8a10 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-json + - id: check-toml + - id: check-yaml + - id: detect-private-key + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + additional_dependencies: + - types-PyYAML + files: "^logingress/.*" + - repo: https://github.com/rhysd/actionlint + rev: v1.7.0 + hooks: + - id: actionlint-docker + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + - repo: https://github.com/PyCQA/bandit + rev: 1.7.8 + hooks: + - id: bandit + name: bandit (python) + args: [--configfile, bandit.yaml, --severity-level, all] + additional_dependencies: [bandit] + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + args: [--strict, -c=.yamllint] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker diff --git a/bigfun/cli.py b/bigfun/cli.py index 60c33170..e0a5b0bb 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -12,20 +12,19 @@ from . import bigfunctions as bf from . import utils - TABLES_FOLDER = 'data' THIS_FOLDER = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') WEBSITE_CONFIG_FOLDER = THIS_FOLDER + '/website' CATEGORIES_DOC_TEMPLATE_FILENAME = f'{THIS_FOLDER}/templates/categories.md' -CONFIG_FILENAME = 'config.yaml' -CONFIG = {} -if os.path.exists(CONFIG_FILENAME): - CONFIG = yaml.safe_load(open(CONFIG_FILENAME, encoding='utf-8').read()) or {} +def load_config(config_filename): + if os.path.exists(config_filename): + return yaml.safe_load(open(config_filename, encoding='utf-8').read()) or {} + return {} -def get_config_value(name): - if name in CONFIG: - return CONFIG[name] +def get_config_value(name, config, config_filename): + if name in config: + return config[name] text, default = { 'project': ("Default GCP project where to deploy bigfunctions", None), @@ -33,11 +32,10 @@ def get_config_value(name): 'project_for_tests': ("Default GCP project where to deploy bigfunctions for TESTING purposes", None), 'dataset_for_tests': ("Default dataset where to deploy bigfunctions for TESTING purposes", None), # eu,us,asia_east1,asia_east2,asia_northeast1,asia_northeast2,asia_northeast3,asia_south1,asia_southeast1,australia_southeast1,europe_north1,europe_west1,europe_west2,europe_west3,europe_west4,europe_west6,northamerica_northeast1,southamerica_east1,us_central1,us_east1,us_east4,us_west1,us_west2 }[name] - CONFIG[name] = click.prompt(text, default=default) - with open(CONFIG_FILENAME, 'w', encoding='utf-8') as outfile: - yaml.dump(CONFIG, outfile, default_flow_style=False) - return CONFIG[name] - + config[name] = click.prompt(text, default=default) + with open(config_filename, 'w', encoding='utf-8') as outfile: + yaml.dump(config, outfile, default_flow_style=False) + return config[name] def generate_doc(project, dataset): @@ -101,32 +99,41 @@ def generate_bigfunctions_list_markdown(bigfunctions): help_headers_color='yellow', help_options_color='cyan' ) -def cli(): - pass +@click.pass_context +def cli(ctx): + ctx.ensure_object(dict) @cli.command() @click.argument('bigfunction') -def get(bigfunction): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def get(ctx, bigfunction, config): + """ Download BIGFUNCTION yaml file from unytics/bigfunctions github repo - ''' + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) if not os.path.isdir('bigfunctions'): os.makedirs('bigfunctions') url = f'https://raw.githubusercontent.com/unytics/bigfunctions/main/bigfunctions/{bigfunction}.yaml' utils.download(url, f'bigfunctions/{bigfunction}.yaml') - - @cli.command() @click.argument('bigfunction') -def test(bigfunction): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def test(ctx, bigfunction, config): + """ Test BIGFUNCTION - ''' - project = get_config_value('project_for_tests') - dataset = get_config_value('dataset_for_tests') + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) + config = ctx.obj['CONFIG'] + config_filename = ctx.obj['CONFIG_FILENAME'] + project = get_config_value('project_for_tests', config, config_filename) + dataset = get_config_value('dataset_for_tests', config, config_filename) bigfunction = bf.BigFunction(bigfunction, project=project, dataset=dataset) bigfunction.test() @@ -135,14 +142,20 @@ def test(bigfunction): @click.argument('bigfunction') @click.option('--project', help='Google Cloud project where the function will be deployed') @click.option('--dataset', help='BigQuery dataset name where the function will be deployed') -def deploy(bigfunction, project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def deploy(ctx, bigfunction, project, dataset, config): + """ Deploy BIGFUNCTION Deploy the function defined in `bigfunctions/{BIGFUNCTION}.yaml` file. If BIGFUNCTION = 'ALL' then all bigfunctions contained in bigfunctions folder are deployed. - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) + config = ctx.obj['CONFIG'] + config_filename = ctx.obj['CONFIG_FILENAME'] + project = project or get_config_value('project', config, config_filename) + dataset = dataset or get_config_value('dataset', config, config_filename) datasets = [dataset.strip() for dataset in dataset.split(',')] bigfunctions = [bigfunction.strip() for bigfunction in bigfunction.split(',')] if bigfunction == 'ALL': @@ -162,19 +175,24 @@ def deploy(bigfunction, project, dataset): ) - @cli.command() @click.argument('table') @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def load_table(table, project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def load_table(ctx, table, project, dataset, config): + """ Create or replace bigquery table TABLE with data contained in `data/{TABLE}.csv`. If TABLE=ALL, then all tables defined in `data` folder are created. - ''' + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) from .load_table import load_table as upload_table - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + config = ctx.obj['CONFIG'] + config_filename = ctx.obj['CONFIG_FILENAME'] + project = project or get_config_value('project', config, config_filename) + dataset = dataset or get_config_value('dataset', config, config_filename) datasets = [dataset.strip() for dataset in dataset.split(',')] if table == 'ALL': tables = [f.replace('.yaml', '') for f in os.listdir(TABLES_FOLDER) if f.endswith('.yaml')] @@ -190,35 +208,46 @@ def load_table(table, project, dataset): @cli.group() def docs(): - ''' + """ Generate, serve and publish documentation - ''' + """ pass @docs.command() @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def generate(project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def generate(ctx, project, dataset, config): + """ Generate markdown files for documentation from yaml bigfunctions files - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) + config = ctx.obj['CONFIG'] + config_filename = ctx.obj['CONFIG_FILENAME'] + project = project or get_config_value('project', config, config_filename) + dataset = dataset or get_config_value('dataset', config, config_filename) generate_doc(project, dataset) os.system('mkdocs build') - @docs.command() @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def serve(project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +@click.pass_context +def serve(ctx, project, dataset, config): + """ Serve docs locally on http://localhost:8000 - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + ctx.obj['CONFIG_FILENAME'] = config + ctx.obj['CONFIG'] = load_config(config) + config = ctx.obj['CONFIG'] + config_filename = ctx.obj['CONFIG_FILENAME'] + project = project or get_config_value('project', config, config_filename) + dataset = dataset or get_config_value('dataset', config, config_filename) generate_doc(project, dataset) class EventHandler(RegexMatchingEventHandler): diff --git a/setup.py b/setup.py index d98bf082..fadd1168 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,11 @@ 'click', 'click-help-colors', ], + extras_require={ + 'dev': [ + 'pre-commit>=3,<4', + ], + }, entry_points={ 'console_scripts': [ 'bigfun = bigfun.cli:cli', From 704953c8d2c59f6fd014c6ee151c80520630d1aa Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 15:50:24 +0900 Subject: [PATCH 02/12] Format Signed-off-by: Yu Ishikawa --- .github/ISSUE_TEMPLATE/0_new_bigfunction.yaml | 6 +- .github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml | 8 +-- .../ISSUE_TEMPLATE/2_improve_bigfunction.yaml | 8 +-- .github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml | 6 +- .../ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml | 4 +- .gitignore | 2 +- .pre-commit-config.yaml | 6 +- README.md | 3 +- bigfun/bigfunctions.py | 14 +++-- bigfun/cli.py | 58 +++++++------------ bigfun/datastore/index.yaml | 1 - bigfun/load_table.py | 1 - bigfun/templates/bookmarklet.js | 4 -- bigfun/templates/categories.md | 1 - bigfun/templates/function_py.py | 7 +-- bigfun/templates/function_py.sql | 2 +- bigfun/templates/function_py_test.py | 2 +- bigfun/templates/procedure_test.sql | 2 +- bigfun/utils.py | 14 ++--- bigfun/website/theme_overrides/main.html | 4 -- bigfunctions/README.md | 5 +- bigfunctions/ask_ai.yaml | 1 - bigfunctions/deidentify.yaml | 2 +- bigfunctions/detect_sensitive_info.yaml | 2 +- bigfunctions/exchange_rate.yaml | 2 +- bigfunctions/explore_column.yaml | 1 - bigfunctions/explore_dataset.yaml | 2 - bigfunctions/explore_events.yaml | 1 - bigfunctions/explore_funnels.yaml | 3 - bigfunctions/export_to_datastore.yaml | 2 +- bigfunctions/find_greater_value.yaml | 2 +- bigfunctions/find_lower_value.yaml | 2 +- bigfunctions/find_value.yaml | 2 +- bigfunctions/generate_face_embedding.yaml | 2 +- bigfunctions/get_json.yaml | 2 +- bigfunctions/get_value.yaml | 2 +- bigfunctions/get_view_history.yaml | 2 +- bigfunctions/gregorian2hijri.yaml | 2 +- bigfunctions/html2pdf.yaml | 1 - bigfunctions/ip2asn.yaml | 1 - bigfunctions/ip2continent.yaml | 1 - bigfunctions/ip2continent_name.yaml | 1 - bigfunctions/ip2country.yaml | 1 - bigfunctions/ip2country_name.yaml | 1 - bigfunctions/ip_range2ip_networks.yaml | 2 +- bigfunctions/json2excel.yaml | 2 +- bigfunctions/json2xml.yaml | 2 +- bigfunctions/json_items.yaml | 2 +- bigfunctions/json_keys.yaml | 2 +- bigfunctions/json_query.yaml | 2 +- bigfunctions/json_schema.yaml | 2 +- bigfunctions/json_values.yaml | 2 +- bigfunctions/last_value.yaml | 2 +- bigfunctions/levenshtein.yaml | 2 +- ...bigquery_resources_in_current_project.yaml | 2 +- bigfunctions/max_value.yaml | 2 +- bigfunctions/min_value.yaml | 2 +- bigfunctions/parse_date.yaml | 1 - bigfunctions/parse_url.yaml | 2 +- bigfunctions/percentile_value.yaml | 2 +- bigfunctions/phone_number_info.yaml | 2 +- bigfunctions/post.yaml | 2 +- bigfunctions/precision_recall_auc.yaml | 2 +- bigfunctions/precision_recall_curve.yaml | 2 +- bigfunctions/prophet.yaml | 2 +- bigfunctions/remove_accents.yaml | 2 +- bigfunctions/remove_extra_whitespaces.yaml | 2 +- bigfunctions/remove_words.yaml | 2 +- bigfunctions/roc_auc.yaml | 2 +- bigfunctions/roc_curve.yaml | 2 +- bigfunctions/send_mail.yaml | 2 +- bigfunctions/sort_values_desc.yaml | 2 +- bigfunctions/translate.yaml | 2 +- bigfunctions/translated_month_name.yaml | 2 +- bigfunctions/upload_to_gsheet.yaml | 2 +- bigfunctions/xml2json.yaml | 2 +- bigfunctions/xml_extract.yaml | 2 +- data/README.md | 1 - data/holidays.yaml | 2 +- data/natality.sql | 2 +- data/natality.yaml | 2 +- data/sales.csv | 2 +- data/sales.yaml | 2 +- data/sample_graph.yaml | 2 +- setup.py | 1 - 85 files changed, 114 insertions(+), 161 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml b/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml index 25fa8d05..4eb92a97 100644 --- a/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to suggest a new BigFunction! - + ## πŸ™ Thanks for taking the time to suggest a new BigFunction! + Please fill the below fields to submit your idea. - type: checkboxes id: idea_does_not_exist attributes: label: Check the idea has not already been suggested - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find my idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) required: true diff --git a/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml b/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml index 20fc9ac4..feb1dba5 100644 --- a/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to report a bug in a BigFunction! - + ## πŸ™ Thanks for taking the time to report a bug in a BigFunction! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check the bug has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the bug in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) required: true @@ -21,7 +21,7 @@ body: id: title_is_ok attributes: label: Edit `function_name` and the short error description in title above - description: Edit the `function_name` and the short error description in the issue title above. + description: Edit the `function_name` and the short error description in the issue title above. options: - label: I wrote the correct function name and a short error description in the title above required: true diff --git a/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml b/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml index fc593a8b..f3e7026c 100644 --- a/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to help us improve! - + ## πŸ™ Thanks for taking the time to help us improve! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check your idea has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) required: true @@ -21,7 +21,7 @@ body: id: title_is_ok attributes: label: Edit `function_name` and the short idea description in title above - description: Edit the `function_name` and the short idea description in the issue title above. + description: Edit the `function_name` and the short idea description in the issue title above. options: - label: I wrote the correct function name and a short idea description in the title above required: true diff --git a/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml b/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml index 77b6bfe7..36cadab0 100644 --- a/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml +++ b/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to report a bug of `bigfun` Command-Line-Interface! - + ## πŸ™ Thanks for taking the time to report a bug of `bigfun` Command-Line-Interface! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check the bug has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the bug in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) required: true diff --git a/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml b/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml index 418fc22f..4f3f2cff 100644 --- a/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml +++ b/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml @@ -7,13 +7,13 @@ body: attributes: value: | ## πŸ™ Thanks for taking the time to help us improve! - + Please fill the below fields to submit your idea. - type: checkboxes id: idea_does_not_exist attributes: label: Check your idea has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) required: true diff --git a/.gitignore b/.gitignore index 02e005e5..d758e338 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,4 @@ tests/ site/ TEST -venv \ No newline at end of file +venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52de8a10..0005dc20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - types-PyYAML files: "^logingress/.*" - repo: https://github.com/rhysd/actionlint - rev: v1.7.0 + rev: v1.7.1 hooks: - id: actionlint-docker - repo: https://github.com/shellcheck-py/shellcheck-py @@ -24,7 +24,7 @@ repos: hooks: - id: shellcheck - repo: https://github.com/PyCQA/bandit - rev: 1.7.8 + rev: 1.7.9 hooks: - id: bandit name: bandit (python) @@ -50,6 +50,6 @@ repos: - id: isort name: isort (python) - repo: https://github.com/hadolint/hadolint - rev: v2.12.0 + rev: v2.13.0-beta hooks: - id: hadolint-docker diff --git a/README.md b/README.md index 1fe13f3f..d137482a 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Commands: ### 5.3 Create you first function πŸ‘· -Functions are defined as yaml files under `bigfunctions` folder. To create your first function locally, the easiest is to download an existing yaml file of unytics/bigfunctions Github repo. +Functions are defined as yaml files under `bigfunctions` folder. To create your first function locally, the easiest is to download an existing yaml file of unytics/bigfunctions Github repo. For instance to download `is_email_valid.yaml` into bigfunctions folder, do: @@ -229,4 +229,3 @@ BigFunctions is fully open-source. Any contribution is more than welcome πŸ€—! - diff --git a/bigfun/bigfunctions.py b/bigfun/bigfunctions.py index 0dd343e6..a5524be4 100644 --- a/bigfun/bigfunctions.py +++ b/bigfun/bigfunctions.py @@ -1,14 +1,16 @@ -import tempfile -import re +import functools +import json import os +import re import shutil -import json -import functools +import tempfile -import yaml import jinja2 +import yaml -from .utils import BigQuery, CloudRun, handle_error, print_success, print_info, print_command, build_and_upload_npm_package, merge_dict +from .utils import (BigQuery, CloudRun, build_and_upload_npm_package, + handle_error, merge_dict, print_command, print_info, + print_success) BIGFUNCTIONS_FOLDER = 'bigfunctions' DEFAULT_CONFIG_FILENAME = './config.yaml' diff --git a/bigfun/cli.py b/bigfun/cli.py index e0a5b0bb..9ec82c8f 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -1,13 +1,13 @@ +import multiprocessing import os import shutil -import multiprocessing +import click import jinja2 import yaml -import click from click_help_colors import HelpColorsGroup -from watchdog.observers import Observer from watchdog.events import RegexMatchingEventHandler +from watchdog.observers import Observer from . import bigfunctions as bf from . import utils @@ -19,7 +19,8 @@ def load_config(config_filename): if os.path.exists(config_filename): - return yaml.safe_load(open(config_filename, encoding='utf-8').read()) or {} + with open(config_filename, encoding='utf-8') as file: + return yaml.safe_load(file) or {} return {} def get_config_value(name, config, config_filename): @@ -106,14 +107,11 @@ def cli(ctx): @cli.command() @click.argument('bigfunction') -@click.option('--config', default='config.yaml', help='Path to the config file') @click.pass_context -def get(ctx, bigfunction, config): +def get(ctx, bigfunction): """ Download BIGFUNCTION yaml file from unytics/bigfunctions github repo """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) if not os.path.isdir('bigfunctions'): os.makedirs('bigfunctions') url = f'https://raw.githubusercontent.com/unytics/bigfunctions/main/bigfunctions/{bigfunction}.yaml' @@ -128,10 +126,7 @@ def test(ctx, bigfunction, config): """ Test BIGFUNCTION """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) - config = ctx.obj['CONFIG'] - config_filename = ctx.obj['CONFIG_FILENAME'] + config_filename = config project = get_config_value('project_for_tests', config, config_filename) dataset = get_config_value('dataset_for_tests', config, config_filename) bigfunction = bf.BigFunction(bigfunction, project=project, dataset=dataset) @@ -150,12 +145,9 @@ def deploy(ctx, bigfunction, project, dataset, config): Deploy the function defined in `bigfunctions/{BIGFUNCTION}.yaml` file. If BIGFUNCTION = 'ALL' then all bigfunctions contained in bigfunctions folder are deployed. """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) - config = ctx.obj['CONFIG'] - config_filename = ctx.obj['CONFIG_FILENAME'] - project = project or get_config_value('project', config, config_filename) - dataset = dataset or get_config_value('dataset', config, config_filename) + config_data = load_config(config) + project = project or get_config_value('project', config_data, config) + dataset = dataset or get_config_value('dataset', config_data, config) datasets = [dataset.strip() for dataset in dataset.split(',')] bigfunctions = [bigfunction.strip() for bigfunction in bigfunction.split(',')] if bigfunction == 'ALL': @@ -186,13 +178,10 @@ def load_table(ctx, table, project, dataset, config): Create or replace bigquery table TABLE with data contained in `data/{TABLE}.csv`. If TABLE=ALL, then all tables defined in `data` folder are created. """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) + config_data = load_config(config) from .load_table import load_table as upload_table - config = ctx.obj['CONFIG'] - config_filename = ctx.obj['CONFIG_FILENAME'] - project = project or get_config_value('project', config, config_filename) - dataset = dataset or get_config_value('dataset', config, config_filename) + project = project or get_config_value('project', config_data, config) + dataset = dataset or get_config_value('dataset', config_data, config) datasets = [dataset.strip() for dataset in dataset.split(',')] if table == 'ALL': tables = [f.replace('.yaml', '') for f in os.listdir(TABLES_FOLDER) if f.endswith('.yaml')] @@ -223,12 +212,9 @@ def generate(ctx, project, dataset, config): """ Generate markdown files for documentation from yaml bigfunctions files """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) - config = ctx.obj['CONFIG'] - config_filename = ctx.obj['CONFIG_FILENAME'] - project = project or get_config_value('project', config, config_filename) - dataset = dataset or get_config_value('dataset', config, config_filename) + config_data = load_config(config) + project = project or get_config_value('project', config_data, config) + dataset = dataset or get_config_value('dataset', config_data, config) generate_doc(project, dataset) os.system('mkdocs build') @@ -242,12 +228,9 @@ def serve(ctx, project, dataset, config): """ Serve docs locally on http://localhost:8000 """ - ctx.obj['CONFIG_FILENAME'] = config - ctx.obj['CONFIG'] = load_config(config) - config = ctx.obj['CONFIG'] - config_filename = ctx.obj['CONFIG_FILENAME'] - project = project or get_config_value('project', config, config_filename) - dataset = dataset or get_config_value('dataset', config, config_filename) + config_data = load_config(config) + project = project or get_config_value('project', config_data, config) + dataset = dataset or get_config_value('dataset', config_data, config) generate_doc(project, dataset) class EventHandler(RegexMatchingEventHandler): @@ -260,3 +243,6 @@ def on_any_event(self, event): # observer.start() # bf.generate_doc(project, dataset) os.system('mkdocs serve') + # observer.start() + # bf.generate_doc(project, dataset) + os.system('mkdocs serve') diff --git a/bigfun/datastore/index.yaml b/bigfun/datastore/index.yaml index bc37fb50..591b47f0 100644 --- a/bigfun/datastore/index.yaml +++ b/bigfun/datastore/index.yaml @@ -4,4 +4,3 @@ indexes: properties: - name: user_bigfunction_date - name: row_count - diff --git a/bigfun/load_table.py b/bigfun/load_table.py index 198776c8..5ed5f603 100644 --- a/bigfun/load_table.py +++ b/bigfun/load_table.py @@ -22,4 +22,3 @@ def load_table(table): job_config = google.cloud.bigquery.job.LoadJobConfig(**load_config) bigquery.load_table_from_file(file, table, job_config=job_config).result() print_success('successfully loaded ' + table) - diff --git a/bigfun/templates/bookmarklet.js b/bigfun/templates/bookmarklet.js index 224de348..dc74451b 100644 --- a/bigfun/templates/bookmarklet.js +++ b/bigfun/templates/bookmarklet.js @@ -121,7 +121,3 @@ loadFunnelGraphJs(); loadChartJs(); loadGoogleChart(); setInterval(run, 100); - - - - diff --git a/bigfun/templates/categories.md b/bigfun/templates/categories.md index c0fcd755..9ebc2eaf 100644 --- a/bigfun/templates/categories.md +++ b/bigfun/templates/categories.md @@ -45,4 +45,3 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** {% endfor %} {% endfor %} - diff --git a/bigfun/templates/function_py.py b/bigfun/templates/function_py.py index 9e7a59c0..86c51fc8 100644 --- a/bigfun/templates/function_py.py +++ b/bigfun/templates/function_py.py @@ -1,12 +1,11 @@ -import time -import json import datetime +import json +import time import traceback -from flask import Flask, request, jsonify import google.auth import google.cloud.error_reporting - +from flask import Flask, jsonify, request error_reporter = google.cloud.error_reporting.Client() app = Flask(__name__) diff --git a/bigfun/templates/function_py.sql b/bigfun/templates/function_py.sql index 651d4430..42621804 100644 --- a/bigfun/templates/function_py.sql +++ b/bigfun/templates/function_py.sql @@ -4,4 +4,4 @@ remote with connection `{{ remote_connection }}` options ( endpoint = '{{ remote_endpoint }}' {% if max_batching_rows %}, max_batching_rows = {{ max_batching_rows }}{% endif %} -); \ No newline at end of file +); diff --git a/bigfun/templates/function_py_test.py b/bigfun/templates/function_py_test.py index 38c0a0f3..b5b97cc2 100644 --- a/bigfun/templates/function_py_test.py +++ b/bigfun/templates/function_py_test.py @@ -10,4 +10,4 @@ def run(): {{ code | indent(4) }} result = run() -print(result) \ No newline at end of file +print(result) diff --git a/bigfun/templates/procedure_test.sql b/bigfun/templates/procedure_test.sql index 5709e768..f0c401db 100644 --- a/bigfun/templates/procedure_test.sql +++ b/bigfun/templates/procedure_test.sql @@ -1,3 +1,3 @@ {% if bigfunction.type == 'procedure' %}call{% elif bigfunction.type == 'table_function' %}select * from{% else %}select{% endif %} {{ dataset }}.{{ bigfunction.name }}({% for argument in example.arguments %}{{ argument | replace('{BIGFUNCTIONS_DATASET}', dataset) | replace('\n', '\n ') }}{% if not loop.last %}, {% endif %}{% endfor %}){% if bigfunction.type == 'procedure' %};{% elif 'output' in bigfunction and bigfunction.type != 'table_function' %} as {{ bigfunction.output.name }}{% endif %} {% if bigfunction.type == 'procedure' and bigfunction.template %}select html from bigfunction_result;{% endif %} -{%- if bigfunction.type == 'procedure' and example.output %}select * from bigfunction_result;{% endif %} \ No newline at end of file +{%- if bigfunction.type == 'procedure' and example.output %}select * from bigfunction_result;{% endif %} diff --git a/bigfun/utils.py b/bigfun/utils.py index 126b5740..152d8f73 100644 --- a/bigfun/utils.py +++ b/bigfun/utils.py @@ -1,20 +1,18 @@ -import tempfile -import shutil +import math import os -import sys +import shutil import subprocess -import math +import sys +import tempfile import urllib.request - +import click import google.api_core.exceptions import google.auth.exceptions import google.cloud.bigquery import google.cloud.bigquery_connection_v1 import google.cloud.storage import google.iam.v1.policy_pb2 -import click - def print_color(msg): @@ -364,4 +362,4 @@ def download(url, destination_filename): try: urllib.request.urlretrieve(url, destination_filename) except Exception as e: - handle_error(f'Could not download file at url `{url}`. Reason: {e}') \ No newline at end of file + handle_error(f'Could not download file at url `{url}`. Reason: {e}') diff --git a/bigfun/website/theme_overrides/main.html b/bigfun/website/theme_overrides/main.html index 354b4867..d37f0457 100644 --- a/bigfun/website/theme_overrides/main.html +++ b/bigfun/website/theme_overrides/main.html @@ -222,7 +222,3 @@

Supercharge BigQuery
with BigFunctions

{% endblock %} - - - - diff --git a/bigfunctions/README.md b/bigfunctions/README.md index ce9d43b9..57583989 100644 --- a/bigfunctions/README.md +++ b/bigfunctions/README.md @@ -66,7 +66,7 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** | `us-west2` | `bigfunctions.us_west2` | | `us-west3` | `bigfunctions.us_west3` | | `us-west4` | `bigfunctions.us_west4` | - + @@ -251,6 +251,3 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** - [timestamp_from_unix_date_time(unix_date_time, date_time_part)](timestamp_from_unix_date_time.md): Interprets `unix_date_time` as the number of `date_time_part` since `1970-01-01 00:00:00 UTC`. - [timestamp_to_unix_date_time(timestamp_expression, date_time_part)](timestamp_to_unix_date_time.md): Returns the number of `date_time_part` since `1970-01-01 00:00:00 UTC`. - [upsert(query_or_table_or_view, destination_table, insertion_mode, primary_keys, recency_field)](upsert.md): Merges `query_or_table_or_view` into the `destination_table`. - - - diff --git a/bigfunctions/ask_ai.yaml b/bigfunctions/ask_ai.yaml index adba0456..1c6531eb 100644 --- a/bigfunctions/ask_ai.yaml +++ b/bigfunctions/ask_ai.yaml @@ -102,4 +102,3 @@ requirements: | quotas: max_rows_per_user_per_day: 1000 max_rows_per_query: 10 - diff --git a/bigfunctions/deidentify.yaml b/bigfunctions/deidentify.yaml index 4bcd8e48..fd5b6b9b 100644 --- a/bigfunctions/deidentify.yaml +++ b/bigfunctions/deidentify.yaml @@ -88,4 +88,4 @@ code: | requirements: | google-cloud-dlp quotas: - max_rows_per_user_per_day: 10000 \ No newline at end of file + max_rows_per_user_per_day: 10000 diff --git a/bigfunctions/detect_sensitive_info.yaml b/bigfunctions/detect_sensitive_info.yaml index ab45eb46..5d587749 100644 --- a/bigfunctions/detect_sensitive_info.yaml +++ b/bigfunctions/detect_sensitive_info.yaml @@ -67,4 +67,4 @@ code: | ] return json.dumps(results, ensure_ascii=False) if results else None requirements: | - google-cloud-dlp \ No newline at end of file + google-cloud-dlp diff --git a/bigfunctions/exchange_rate.yaml b/bigfunctions/exchange_rate.yaml index 1df93de9..bdb1a895 100644 --- a/bigfunctions/exchange_rate.yaml +++ b/bigfunctions/exchange_rate.yaml @@ -54,4 +54,4 @@ code: | requirements: | yfinance cloud_run: - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/explore_column.yaml b/bigfunctions/explore_column.yaml index e78ed93a..1a1bc171 100644 --- a/bigfunctions/explore_column.yaml +++ b/bigfunctions/explore_column.yaml @@ -226,4 +226,3 @@ template: | - diff --git a/bigfunctions/explore_dataset.yaml b/bigfunctions/explore_dataset.yaml index 959ab4c0..8c30e6a9 100644 --- a/bigfunctions/explore_dataset.yaml +++ b/bigfunctions/explore_dataset.yaml @@ -102,5 +102,3 @@ template: |-
{% endfor %} - - diff --git a/bigfunctions/explore_events.yaml b/bigfunctions/explore_events.yaml index 9fb5ba95..975931d9 100644 --- a/bigfunctions/explore_events.yaml +++ b/bigfunctions/explore_events.yaml @@ -252,4 +252,3 @@ template: | } })(); - diff --git a/bigfunctions/explore_funnels.yaml b/bigfunctions/explore_funnels.yaml index 6d6dbe44..e55a507d 100644 --- a/bigfunctions/explore_funnels.yaml +++ b/bigfunctions/explore_funnels.yaml @@ -247,6 +247,3 @@ template: | })(); - - - diff --git a/bigfunctions/export_to_datastore.yaml b/bigfunctions/export_to_datastore.yaml index f84d3452..1b4c450e 100644 --- a/bigfunctions/export_to_datastore.yaml +++ b/bigfunctions/export_to_datastore.yaml @@ -86,4 +86,4 @@ cloud_run: concurrency: 16 max_instances: 10 quotas: - max_rows_per_user_per_day: 1000000 \ No newline at end of file + max_rows_per_user_per_day: 1000000 diff --git a/bigfunctions/find_greater_value.yaml b/bigfunctions/find_greater_value.yaml index 90905a6e..b5d6b1e0 100644 --- a/bigfunctions/find_greater_value.yaml +++ b/bigfunctions/find_greater_value.yaml @@ -39,4 +39,4 @@ code: | where value >= x order by offset limit 1 - ) \ No newline at end of file + ) diff --git a/bigfunctions/find_lower_value.yaml b/bigfunctions/find_lower_value.yaml index 2e5acce2..3281a818 100644 --- a/bigfunctions/find_lower_value.yaml +++ b/bigfunctions/find_lower_value.yaml @@ -39,4 +39,4 @@ code: | where value <= x order by offset limit 1 - ) \ No newline at end of file + ) diff --git a/bigfunctions/find_value.yaml b/bigfunctions/find_value.yaml index ae280129..1cf70831 100644 --- a/bigfunctions/find_value.yaml +++ b/bigfunctions/find_value.yaml @@ -26,4 +26,4 @@ examples: - "[3, 4]" - "7" output: "null" -code: (select offset from unnest(arr) as x with offset where x = value order by offset limit 1) \ No newline at end of file +code: (select offset from unnest(arr) as x with offset where x = value order by offset limit 1) diff --git a/bigfunctions/generate_face_embedding.yaml b/bigfunctions/generate_face_embedding.yaml index 9c7d1127..35e1f006 100644 --- a/bigfunctions/generate_face_embedding.yaml +++ b/bigfunctions/generate_face_embedding.yaml @@ -66,4 +66,4 @@ quotas: cloud_run: memory: 2048Mi concurrency: 1 - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/get_json.yaml b/bigfunctions/get_json.yaml index 167e6b96..a952dd21 100644 --- a/bigfunctions/get_json.yaml +++ b/bigfunctions/get_json.yaml @@ -25,4 +25,4 @@ code: | url, json_set(headers, '$.Content-Type', 'application/json') ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/get_value.yaml b/bigfunctions/get_value.yaml index 3c97989c..a8891a35 100644 --- a/bigfunctions/get_value.yaml +++ b/bigfunctions/get_value.yaml @@ -31,4 +31,4 @@ examples: - "[struct('a' as key, 8 as value), struct('a' as key, 9 as value)]" - "'a'" output: "8" -code: (select value from unnest(key_value_items) item where item.key = search_key limit 1) \ No newline at end of file +code: (select value from unnest(key_value_items) item where item.key = search_key limit 1) diff --git a/bigfunctions/get_view_history.yaml b/bigfunctions/get_view_history.yaml index dff4f819..e467bf56 100644 --- a/bigfunctions/get_view_history.yaml +++ b/bigfunctions/get_view_history.yaml @@ -45,4 +45,4 @@ code: | to_json(struct( fully_qualified_view as fully_qualified_view )) - ); \ No newline at end of file + ); diff --git a/bigfunctions/gregorian2hijri.yaml b/bigfunctions/gregorian2hijri.yaml index dcdfe7d3..ee503ac0 100644 --- a/bigfunctions/gregorian2hijri.yaml +++ b/bigfunctions/gregorian2hijri.yaml @@ -4,7 +4,7 @@ author: name: Fahed Sabellioglu url: https://stackoverflow.com/users/23326889/fahed-sabellioglu avatar_url: "https://lh3.googleusercontent.com/a/ACg8ocLdYt_9sKc80I-jaH_0VVz6zaSpYZ3-wV3XiNh8Z8QBiA=k-s256" -description: | +description: | Convert Gregorian Date to Hijri Date (taken from [here](https://stackoverflow.com/questions/78072960/convert-dates-gregorian-to-hijri-bigquery#answer-78079872)) arguments: - name: gregorian_date diff --git a/bigfunctions/html2pdf.yaml b/bigfunctions/html2pdf.yaml index 6c0cbba4..44b186a0 100644 --- a/bigfunctions/html2pdf.yaml +++ b/bigfunctions/html2pdf.yaml @@ -55,4 +55,3 @@ cloud_run: memory: 1024Mi concurrency: 1 max_instances: 10 - diff --git a/bigfunctions/ip2asn.yaml b/bigfunctions/ip2asn.yaml index 49b940f0..8e8b9f51 100644 --- a/bigfunctions/ip2asn.yaml +++ b/bigfunctions/ip2asn.yaml @@ -72,4 +72,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2continent.yaml b/bigfunctions/ip2continent.yaml index 65f414ce..ae4e4df0 100644 --- a/bigfunctions/ip2continent.yaml +++ b/bigfunctions/ip2continent.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2continent_name.yaml b/bigfunctions/ip2continent_name.yaml index e4dde6d7..10c71f51 100644 --- a/bigfunctions/ip2continent_name.yaml +++ b/bigfunctions/ip2continent_name.yaml @@ -50,4 +50,3 @@ secrets: cloud_run: memory: 512Mi max_instances: 10 - diff --git a/bigfunctions/ip2country.yaml b/bigfunctions/ip2country.yaml index 5bbb51e1..e4febbea 100644 --- a/bigfunctions/ip2country.yaml +++ b/bigfunctions/ip2country.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2country_name.yaml b/bigfunctions/ip2country_name.yaml index 7425ef83..3c0dd3c4 100644 --- a/bigfunctions/ip2country_name.yaml +++ b/bigfunctions/ip2country_name.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip_range2ip_networks.yaml b/bigfunctions/ip_range2ip_networks.yaml index b409a425..d642d87c 100644 --- a/bigfunctions/ip_range2ip_networks.yaml +++ b/bigfunctions/ip_range2ip_networks.yaml @@ -34,4 +34,4 @@ code: | requirements: | netaddr cloud_run: - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/json2excel.yaml b/bigfunctions/json2excel.yaml index a0ef037b..41b8c28f 100644 --- a/bigfunctions/json2excel.yaml +++ b/bigfunctions/json2excel.yaml @@ -26,4 +26,4 @@ code: | xlsx.utils.book_append_sheet(wb, ws, "Sheet1"); return xlsx.write(wb, {type: "base64", bookType: "xlsx"}); npm_packages: - - xlsx@0.18.5 \ No newline at end of file + - xlsx@0.18.5 diff --git a/bigfunctions/json2xml.yaml b/bigfunctions/json2xml.yaml index 3645a178..985b3c1a 100644 --- a/bigfunctions/json2xml.yaml +++ b/bigfunctions/json2xml.yaml @@ -34,4 +34,4 @@ code: | return null } npm_packages: - - fast-xml-parser@4.2.7 \ No newline at end of file + - fast-xml-parser@4.2.7 diff --git a/bigfunctions/json_items.yaml b/bigfunctions/json_items.yaml index 870032aa..2b5abf01 100644 --- a/bigfunctions/json_items.yaml +++ b/bigfunctions/json_items.yaml @@ -34,4 +34,4 @@ code: | for (const i in obj) { arr.push({key: i, value: obj[i] }); } - return arr \ No newline at end of file + return arr diff --git a/bigfunctions/json_keys.yaml b/bigfunctions/json_keys.yaml index f703f22a..1115c151 100644 --- a/bigfunctions/json_keys.yaml +++ b/bigfunctions/json_keys.yaml @@ -26,4 +26,4 @@ code: | catch { throw('could not parse ' + json_string) } - return Object.keys(obj) \ No newline at end of file + return Object.keys(obj) diff --git a/bigfunctions/json_query.yaml b/bigfunctions/json_query.yaml index 3602905b..6234f1fd 100644 --- a/bigfunctions/json_query.yaml +++ b/bigfunctions/json_query.yaml @@ -50,4 +50,4 @@ code: | } return JSON.stringify(jmespath.search(obj, query)); npm_packages: - - jmespath@0.16.0 \ No newline at end of file + - jmespath@0.16.0 diff --git a/bigfunctions/json_schema.yaml b/bigfunctions/json_schema.yaml index 9de5e06f..2d0e47d6 100644 --- a/bigfunctions/json_schema.yaml +++ b/bigfunctions/json_schema.yaml @@ -68,4 +68,4 @@ code: | } const schema = getSchema(data); - return schema \ No newline at end of file + return schema diff --git a/bigfunctions/json_values.yaml b/bigfunctions/json_values.yaml index a3a5ea19..204334f5 100644 --- a/bigfunctions/json_values.yaml +++ b/bigfunctions/json_values.yaml @@ -26,4 +26,4 @@ code: | catch { throw('could not parse ' + json_string) } - return Object.values(obj) \ No newline at end of file + return Object.values(obj) diff --git a/bigfunctions/last_value.yaml b/bigfunctions/last_value.yaml index abce9f70..285f4ea6 100644 --- a/bigfunctions/last_value.yaml +++ b/bigfunctions/last_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 2, 3]" output: "3" -code: arr[ordinal(array_length(arr))] \ No newline at end of file +code: arr[ordinal(array_length(arr))] diff --git a/bigfunctions/levenshtein.yaml b/bigfunctions/levenshtein.yaml index bf1100c3..3c0ba1f9 100644 --- a/bigfunctions/levenshtein.yaml +++ b/bigfunctions/levenshtein.yaml @@ -22,4 +22,4 @@ examples: code: | return js_levenshtein(string1 || '', string2 || ''); npm_packages: - - js-levenshtein@1.1.6 \ No newline at end of file + - js-levenshtein@1.1.6 diff --git a/bigfunctions/list_bigquery_resources_in_current_project.yaml b/bigfunctions/list_bigquery_resources_in_current_project.yaml index 735ae681..f17adf40 100644 --- a/bigfunctions/list_bigquery_resources_in_current_project.yaml +++ b/bigfunctions/list_bigquery_resources_in_current_project.yaml @@ -288,4 +288,4 @@ code: | ), regexp_replace(id, r'[^a-zA-Z]', ' ') as broken_id, from joined - ''', '{PROJECT}', @@project_id); \ No newline at end of file + ''', '{PROJECT}', @@project_id); diff --git a/bigfunctions/max_value.yaml b/bigfunctions/max_value.yaml index d79efb26..fd296f95 100644 --- a/bigfunctions/max_value.yaml +++ b/bigfunctions/max_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 4, 3]" output: "4" -code: (select max(value) from unnest(arr) value) \ No newline at end of file +code: (select max(value) from unnest(arr) value) diff --git a/bigfunctions/min_value.yaml b/bigfunctions/min_value.yaml index e5dc6e10..f26f369e 100644 --- a/bigfunctions/min_value.yaml +++ b/bigfunctions/min_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 4, 3]" output: "1" -code: (select min(value) from unnest(arr) value) \ No newline at end of file +code: (select min(value) from unnest(arr) value) diff --git a/bigfunctions/parse_date.yaml b/bigfunctions/parse_date.yaml index 09d35724..c7f3f47c 100644 --- a/bigfunctions/parse_date.yaml +++ b/bigfunctions/parse_date.yaml @@ -61,4 +61,3 @@ code: | ) else null end - diff --git a/bigfunctions/parse_url.yaml b/bigfunctions/parse_url.yaml index 4af28fa5..ed15efe6 100644 --- a/bigfunctions/parse_url.yaml +++ b/bigfunctions/parse_url.yaml @@ -25,4 +25,4 @@ code: | regexp_extract(url, r'\?(.*)') as query, regexp_extract(url, r'#(.*)') as ref, regexp_extract(url, r'^([a-za-z]+)://') as protocol - ) \ No newline at end of file + ) diff --git a/bigfunctions/percentile_value.yaml b/bigfunctions/percentile_value.yaml index 8377c1b9..af545fe2 100644 --- a/bigfunctions/percentile_value.yaml +++ b/bigfunctions/percentile_value.yaml @@ -81,4 +81,4 @@ code: | ) end end - ) \ No newline at end of file + ) diff --git a/bigfunctions/phone_number_info.yaml b/bigfunctions/phone_number_info.yaml index f61c80c1..1c6ab04b 100644 --- a/bigfunctions/phone_number_info.yaml +++ b/bigfunctions/phone_number_info.yaml @@ -173,4 +173,4 @@ code: | isNonGeographic: phoneNumber.isNonGeographic(), }; npm_packages: - - libphonenumber-js/max@1.10.51 \ No newline at end of file + - libphonenumber-js/max@1.10.51 diff --git a/bigfunctions/post.yaml b/bigfunctions/post.yaml index 472c2992..69212a91 100644 --- a/bigfunctions/post.yaml +++ b/bigfunctions/post.yaml @@ -42,4 +42,4 @@ code: | requirements: | requests quotas: - max_rows_per_query: 10 \ No newline at end of file + max_rows_per_query: 10 diff --git a/bigfunctions/precision_recall_auc.yaml b/bigfunctions/precision_recall_auc.yaml index 658936f7..4b25f983 100644 --- a/bigfunctions/precision_recall_auc.yaml +++ b/bigfunctions/precision_recall_auc.yaml @@ -29,4 +29,4 @@ code: | select (recall - lag(recall) over (order by recall)) * (precision + lag(precision) over (order by recall)) / 2 as auc_contrib from {BIGFUNCTIONS_DATASET}.precision_recall_curve(predictions) ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/precision_recall_curve.yaml b/bigfunctions/precision_recall_curve.yaml index 34545ee6..eb755d1a 100644 --- a/bigfunctions/precision_recall_curve.yaml +++ b/bigfunctions/precision_recall_curve.yaml @@ -56,4 +56,4 @@ code: | from unnested union all select 1, 0 - order by recall desc \ No newline at end of file + order by recall desc diff --git a/bigfunctions/prophet.yaml b/bigfunctions/prophet.yaml index ce1626ad..b66f5935 100644 --- a/bigfunctions/prophet.yaml +++ b/bigfunctions/prophet.yaml @@ -39,4 +39,4 @@ requirements: | prophet cloud_run: max_instances: 20 - concurrency: 1 \ No newline at end of file + concurrency: 1 diff --git a/bigfunctions/remove_accents.yaml b/bigfunctions/remove_accents.yaml index 44ea9f70..548237d1 100644 --- a/bigfunctions/remove_accents.yaml +++ b/bigfunctions/remove_accents.yaml @@ -16,4 +16,4 @@ examples: arguments: - "'VoilΓ  !'" output: Voila ! -code: regexp_replace(normalize(str, nfd), r"\pm", '') \ No newline at end of file +code: regexp_replace(normalize(str, nfd), r"\pm", '') diff --git a/bigfunctions/remove_extra_whitespaces.yaml b/bigfunctions/remove_extra_whitespaces.yaml index c5d33cfb..42eb1ab7 100644 --- a/bigfunctions/remove_extra_whitespaces.yaml +++ b/bigfunctions/remove_extra_whitespaces.yaml @@ -18,4 +18,4 @@ examples: arguments: - "'Hi Madison and Mateusz!\\n How are you doing?'" output: Hi Madison and Mateusz! How are you doing? -code: trim(regexp_replace(str, r'\s\s+', ' ')) \ No newline at end of file +code: trim(regexp_replace(str, r'\s\s+', ' ')) diff --git a/bigfunctions/remove_words.yaml b/bigfunctions/remove_words.yaml index 1599725f..805af9f9 100644 --- a/bigfunctions/remove_words.yaml +++ b/bigfunctions/remove_words.yaml @@ -20,4 +20,4 @@ examples: - "['can', 'eat']" output: "I candies" code: | - (select regexp_replace(string, r'\b(' || array_to_string(words_to_remove, '|') || r')\b', '')) \ No newline at end of file + (select regexp_replace(string, r'\b(' || array_to_string(words_to_remove, '|') || r')\b', '')) diff --git a/bigfunctions/roc_auc.yaml b/bigfunctions/roc_auc.yaml index dffc2e1e..8a9b03c9 100644 --- a/bigfunctions/roc_auc.yaml +++ b/bigfunctions/roc_auc.yaml @@ -33,4 +33,4 @@ code: | select (false_positive_rate - lag(false_positive_rate) over (order by false_positive_rate)) * (true_positive_rate + lag(true_positive_rate) over (order by false_positive_rate)) / 2 as auc_contrib from {BIGFUNCTIONS_DATASET}.roc_curve(predictions) ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/roc_curve.yaml b/bigfunctions/roc_curve.yaml index c4746653..4567d6f0 100644 --- a/bigfunctions/roc_curve.yaml +++ b/bigfunctions/roc_curve.yaml @@ -59,4 +59,4 @@ code: | from unnested union all select 0, 0 - order by false_positive_rate, true_positive_rate \ No newline at end of file + order by false_positive_rate, true_positive_rate diff --git a/bigfunctions/send_mail.yaml b/bigfunctions/send_mail.yaml index 0d441bc7..31f0c69f 100644 --- a/bigfunctions/send_mail.yaml +++ b/bigfunctions/send_mail.yaml @@ -507,4 +507,4 @@ secrets: description: "Sendgrid API Key" documentation_link: https://docs.sendgrid.com/for-developers/sending-email/api-getting-started quotas: - max_rows_per_query: 10 \ No newline at end of file + max_rows_per_query: 10 diff --git a/bigfunctions/sort_values_desc.yaml b/bigfunctions/sort_values_desc.yaml index 36250d03..c73f7ec3 100644 --- a/bigfunctions/sort_values_desc.yaml +++ b/bigfunctions/sort_values_desc.yaml @@ -16,4 +16,4 @@ examples: arguments: - "[1, 4, 3]" output: "[4, 3, 1]" -code: array(select value from unnest(arr) as value order by value desc) \ No newline at end of file +code: array(select value from unnest(arr) as value order by value desc) diff --git a/bigfunctions/translate.yaml b/bigfunctions/translate.yaml index 27c6a742..8490b31b 100644 --- a/bigfunctions/translate.yaml +++ b/bigfunctions/translate.yaml @@ -19,7 +19,7 @@ examples: - '"Salut πŸ‘‹ Florian. Merci d''avoir contribuΓ© !"' - "'en'" output: "Hello πŸ‘‹ Florian. Thanks for contributing!" -init_code: | +init_code: | from google.cloud import translate_v2 as translate translator = translate.Client() code: | diff --git a/bigfunctions/translated_month_name.yaml b/bigfunctions/translated_month_name.yaml index cc5bfb7f..ff65c652 100644 --- a/bigfunctions/translated_month_name.yaml +++ b/bigfunctions/translated_month_name.yaml @@ -30,4 +30,4 @@ code: | where lower(locale) = lower(language) and month_nb = extract(month from date) - ) \ No newline at end of file + ) diff --git a/bigfunctions/upload_to_gsheet.yaml b/bigfunctions/upload_to_gsheet.yaml index c05278d4..8feefdf2 100644 --- a/bigfunctions/upload_to_gsheet.yaml +++ b/bigfunctions/upload_to_gsheet.yaml @@ -113,4 +113,4 @@ quotas: max_rows_per_query: 1 cloud_run: service_account: 749389685934-compute@developer.gserviceaccount.com - memory: 512Mi \ No newline at end of file + memory: 512Mi diff --git a/bigfunctions/xml2json.yaml b/bigfunctions/xml2json.yaml index 2dfe4b30..1a2cd8fe 100644 --- a/bigfunctions/xml2json.yaml +++ b/bigfunctions/xml2json.yaml @@ -33,4 +33,4 @@ code: | } return null; npm_packages: - - fast-xml-parser@4.1.3 \ No newline at end of file + - fast-xml-parser@4.1.3 diff --git a/bigfunctions/xml_extract.yaml b/bigfunctions/xml_extract.yaml index 1b1d83c6..c6003f6b 100644 --- a/bigfunctions/xml_extract.yaml +++ b/bigfunctions/xml_extract.yaml @@ -41,4 +41,4 @@ code: | } npm_packages: - xmldom@0.6.0 - - xpath@0.0.32 \ No newline at end of file + - xpath@0.0.32 diff --git a/data/README.md b/data/README.md index ac0e5212..f2b2dd43 100644 --- a/data/README.md +++ b/data/README.md @@ -321,4 +321,3 @@ with open('translated_months.csv', 'w', encoding='utf-8') as out: --- - diff --git a/data/holidays.yaml b/data/holidays.yaml index 96d7a883..3b42929e 100644 --- a/data/holidays.yaml +++ b/data/holidays.yaml @@ -19,4 +19,4 @@ schema: type: date description: "public holiday local date" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/data/natality.sql b/data/natality.sql index bbd1030a..0cd6d5dc 100644 --- a/data/natality.sql +++ b/data/natality.sql @@ -33,4 +33,4 @@ CREATE TABLE `bigfunctions.us.natality` record_weight INT64 OPTIONS(description="1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area.") ) as -select * from `bigquery-public-data.samples.natality` tablesample system (10 percent) \ No newline at end of file +select * from `bigquery-public-data.samples.natality` tablesample system (10 percent) diff --git a/data/natality.yaml b/data/natality.yaml index 9eea26c1..657c58c8 100644 --- a/data/natality.yaml +++ b/data/natality.yaml @@ -134,4 +134,4 @@ schema: description: "Age of the father when the child was born." - name: record_weight type: INT64 - description: "1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area." \ No newline at end of file + description: "1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area." diff --git a/data/sales.csv b/data/sales.csv index ac4fc9b0..fa7fc3bb 100644 --- a/data/sales.csv +++ b/data/sales.csv @@ -996,4 +996,4 @@ 4,45,8,2023-01-13 1,62,10,2023-05-01 6,97,10,2023-03-30 -1,62,5,2023-01-06 \ No newline at end of file +1,62,5,2023-01-06 diff --git a/data/sales.yaml b/data/sales.yaml index 20470472..1f359073 100644 --- a/data/sales.yaml +++ b/data/sales.yaml @@ -14,4 +14,4 @@ schema: type: date description: "Sale date" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/data/sample_graph.yaml b/data/sample_graph.yaml index 22b5b74a..6509e182 100644 --- a/data/sample_graph.yaml +++ b/data/sample_graph.yaml @@ -8,4 +8,4 @@ schema: type: int64 description: "node id connected to node1" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/setup.py b/setup.py index fadd1168..966b02a2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import setuptools - VERSION = '0.7' From 898e912a75749d62d2a2398958b4ab917aaf0ee9 Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 16:01:06 +0900 Subject: [PATCH 03/12] Format Signed-off-by: Yu Ishikawa --- .github/workflows/deploy_pages.yml | 4 ++-- bigfun/datastore/index.yaml | 9 ++++----- bigfunctions/deduplicate_rows.yaml | 2 +- bigfunctions/get_latest_partition_timestamp.yaml | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy_pages.yml b/.github/workflows/deploy_pages.yml index 94c36e0e..69b70e18 100644 --- a/.github/workflows/deploy_pages.yml +++ b/.github/workflows/deploy_pages.yml @@ -8,8 +8,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - run: pip install -e . diff --git a/bigfun/datastore/index.yaml b/bigfun/datastore/index.yaml index 591b47f0..2ce19f82 100644 --- a/bigfun/datastore/index.yaml +++ b/bigfun/datastore/index.yaml @@ -1,6 +1,5 @@ indexes: - -- kind: bigfunction_call - properties: - - name: user_bigfunction_date - - name: row_count + - kind: bigfunction_call + properties: + - name: user_bigfunction_date + - name: row_count diff --git a/bigfunctions/deduplicate_rows.yaml b/bigfunctions/deduplicate_rows.yaml index ae860dc7..a3b1438b 100644 --- a/bigfunctions/deduplicate_rows.yaml +++ b/bigfunctions/deduplicate_rows.yaml @@ -15,7 +15,7 @@ examples: - description: "Returns table with duplicate rows removed." arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +-----+-----+ | id1 | id2 | +-----+-----+ diff --git a/bigfunctions/get_latest_partition_timestamp.yaml b/bigfunctions/get_latest_partition_timestamp.yaml index 90a46c5f..98defa09 100644 --- a/bigfunctions/get_latest_partition_timestamp.yaml +++ b/bigfunctions/get_latest_partition_timestamp.yaml @@ -15,7 +15,7 @@ examples: - description: "" arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +----------------------------+ | latest_partition_timestamp | +----------------------------+ From 61008d77f516adf60cfec2ed7f1571e45c9e8c85 Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 16:01:19 +0900 Subject: [PATCH 04/12] Update Signed-off-by: Yu Ishikawa --- .pre-commit-config.yaml | 24 +++++++++++++----------- .yamllint | 10 ++++++++++ bandit.yaml | 6 ++++++ 3 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 .yamllint create mode 100644 bandit.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0005dc20..5bfb8674 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,13 +23,14 @@ repos: rev: v0.10.0.1 hooks: - id: shellcheck - - repo: https://github.com/PyCQA/bandit - rev: 1.7.9 - hooks: - - id: bandit - name: bandit (python) - args: [--configfile, bandit.yaml, --severity-level, all] - additional_dependencies: [bandit] + # TODO enable the hook + # - repo: https://github.com/PyCQA/bandit + # rev: 1.7.9 + # hooks: + # - id: bandit + # name: bandit (python) + # args: [--configfile, bandit.yaml, --severity-level, all] + # additional_dependencies: [bandit] - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: @@ -49,7 +50,8 @@ repos: hooks: - id: isort name: isort (python) - - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta - hooks: - - id: hadolint-docker + # TODO enable the hook + # - repo: https://github.com/hadolint/hadolint + # rev: v2.13.0-beta + # hooks: + # - id: hadolint-docker diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..44c09792 --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +rules: + trailing-spaces: disable + key-duplicates: {} + colons: + max-spaces-before: 0 + max-spaces-after: 1 + indentation: + spaces: 2 + check-multi-line-strings: false diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 00000000..8cbd6620 --- /dev/null +++ b/bandit.yaml @@ -0,0 +1,6 @@ +# SEE https://bandit.readthedocs.io/en/latest/config.html +exclude_dirs: + - "tests" +skips: + # hardcoded_sql_expressions + - "B608" From 1a25c2e29305f2852187dc06d3b860095f286e6e Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 16:03:13 +0900 Subject: [PATCH 05/12] Add the workflow to run pre-commit Signed-off-by: Yu Ishikawa --- .github/workflows/lint.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..184cec8a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,24 @@ +name: lint + +on: + pull_request: + push: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + pip install -e .[dev] + - name: Run pre-commit + run: | + pre-commit run --all-files From 9a525abd6cb97cbecda39c22ffaf5212c2cd7c57 Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 16:12:34 +0900 Subject: [PATCH 06/12] Revert "Format" This reverts commit 898e912a75749d62d2a2398958b4ab917aaf0ee9. --- .github/workflows/deploy_pages.yml | 4 ++-- bigfun/datastore/index.yaml | 9 +++++---- bigfunctions/deduplicate_rows.yaml | 2 +- bigfunctions/get_latest_partition_timestamp.yaml | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy_pages.yml b/.github/workflows/deploy_pages.yml index 69b70e18..94c36e0e 100644 --- a/.github/workflows/deploy_pages.yml +++ b/.github/workflows/deploy_pages.yml @@ -8,8 +8,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 with: python-version: 3.x - run: pip install -e . diff --git a/bigfun/datastore/index.yaml b/bigfun/datastore/index.yaml index 2ce19f82..591b47f0 100644 --- a/bigfun/datastore/index.yaml +++ b/bigfun/datastore/index.yaml @@ -1,5 +1,6 @@ indexes: - - kind: bigfunction_call - properties: - - name: user_bigfunction_date - - name: row_count + +- kind: bigfunction_call + properties: + - name: user_bigfunction_date + - name: row_count diff --git a/bigfunctions/deduplicate_rows.yaml b/bigfunctions/deduplicate_rows.yaml index a3b1438b..ae860dc7 100644 --- a/bigfunctions/deduplicate_rows.yaml +++ b/bigfunctions/deduplicate_rows.yaml @@ -15,7 +15,7 @@ examples: - description: "Returns table with duplicate rows removed." arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +-----+-----+ | id1 | id2 | +-----+-----+ diff --git a/bigfunctions/get_latest_partition_timestamp.yaml b/bigfunctions/get_latest_partition_timestamp.yaml index 98defa09..90a46c5f 100644 --- a/bigfunctions/get_latest_partition_timestamp.yaml +++ b/bigfunctions/get_latest_partition_timestamp.yaml @@ -15,7 +15,7 @@ examples: - description: "" arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +----------------------------+ | latest_partition_timestamp | +----------------------------+ From 5f3380d44219d08fde452d63150050c8570a3ab2 Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Tue, 18 Jun 2024 16:16:46 +0900 Subject: [PATCH 07/12] Format Signed-off-by: Yu Ishikawa --- .github/workflows/deploy_pages.yml | 4 ++-- bigfun/datastore/index.yaml | 9 ++++----- bigfunctions/deduplicate_rows.yaml | 2 +- bigfunctions/get_latest_partition_timestamp.yaml | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy_pages.yml b/.github/workflows/deploy_pages.yml index 94c36e0e..69b70e18 100644 --- a/.github/workflows/deploy_pages.yml +++ b/.github/workflows/deploy_pages.yml @@ -8,8 +8,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - run: pip install -e . diff --git a/bigfun/datastore/index.yaml b/bigfun/datastore/index.yaml index 591b47f0..2ce19f82 100644 --- a/bigfun/datastore/index.yaml +++ b/bigfun/datastore/index.yaml @@ -1,6 +1,5 @@ indexes: - -- kind: bigfunction_call - properties: - - name: user_bigfunction_date - - name: row_count + - kind: bigfunction_call + properties: + - name: user_bigfunction_date + - name: row_count diff --git a/bigfunctions/deduplicate_rows.yaml b/bigfunctions/deduplicate_rows.yaml index ae860dc7..a3b1438b 100644 --- a/bigfunctions/deduplicate_rows.yaml +++ b/bigfunctions/deduplicate_rows.yaml @@ -15,7 +15,7 @@ examples: - description: "Returns table with duplicate rows removed." arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +-----+-----+ | id1 | id2 | +-----+-----+ diff --git a/bigfunctions/get_latest_partition_timestamp.yaml b/bigfunctions/get_latest_partition_timestamp.yaml index 90a46c5f..98defa09 100644 --- a/bigfunctions/get_latest_partition_timestamp.yaml +++ b/bigfunctions/get_latest_partition_timestamp.yaml @@ -15,7 +15,7 @@ examples: - description: "" arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +----------------------------+ | latest_partition_timestamp | +----------------------------+ From 6b13f760b0bffd6e6413995a321c8ff5b6f5b4b3 Mon Sep 17 00:00:00 2001 From: Unytics <111615732+unytics@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:54:59 +0200 Subject: [PATCH 08/12] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bfb8674..746d903b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,15 +14,10 @@ repos: - id: mypy additional_dependencies: - types-PyYAML - files: "^logingress/.*" - repo: https://github.com/rhysd/actionlint rev: v1.7.1 hooks: - id: actionlint-docker - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 - hooks: - - id: shellcheck # TODO enable the hook # - repo: https://github.com/PyCQA/bandit # rev: 1.7.9 @@ -41,15 +36,6 @@ repos: hooks: - id: yamllint args: [--strict, -c=.yamllint] - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 - hooks: - - id: shellcheck - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort (python) # TODO enable the hook # - repo: https://github.com/hadolint/hadolint # rev: v2.13.0-beta From a51435881f069deaa6729cd1bef861371bd569f2 Mon Sep 17 00:00:00 2001 From: Unytics <111615732+unytics@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:49:19 +0200 Subject: [PATCH 09/12] Update cli.py --- bigfun/cli.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/bigfun/cli.py b/bigfun/cli.py index f00bedf7..ddaeaed4 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -17,13 +17,14 @@ WEBSITE_CONFIG_FOLDER = THIS_FOLDER + '/website' CATEGORIES_DOC_TEMPLATE_FILENAME = f'{THIS_FOLDER}/templates/categories.md' -def load_config(config_filename): + +def get_config_value(name, config_filename): if os.path.exists(config_filename): with open(config_filename, encoding='utf-8') as file: - return yaml.safe_load(file) or {} - return {} - -def get_config_value(name, config, config_filename): + config = yaml.safe_load(file) or {} + else: + config = {} + if name in config: return config[name] @@ -38,6 +39,7 @@ def get_config_value(name, config, config_filename): yaml.dump(config, outfile, default_flow_style=False) return config[name] + def generate_doc(project, dataset): def init_docs_folder(): @@ -129,9 +131,8 @@ def test(ctx, bigfunction, config): """ Test BIGFUNCTION """ - config_filename = config - project = get_config_value('project_for_tests', config, config_filename) - dataset = get_config_value('dataset_for_tests', config, config_filename) + project = get_config_value('project_for_tests', config) + dataset = get_config_value('dataset_for_tests', config) bigfunction = bf.BigFunction(bigfunction, project=project, dataset=dataset) bigfunction.test() @@ -148,9 +149,8 @@ def deploy(ctx, bigfunction, project, dataset, config): Deploy the function defined in `bigfunctions/{BIGFUNCTION}.yaml` file. If BIGFUNCTION = 'ALL' then all bigfunctions contained in bigfunctions folder are deployed. """ - config_data = load_config(config) - project = project or get_config_value('project', config_data, config) - dataset = dataset or get_config_value('dataset', config_data, config) + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) datasets = [dataset.strip() for dataset in dataset.split(',')] bigfunctions = [bigfunction.strip() for bigfunction in bigfunction.split(',')] if bigfunction == 'ALL': @@ -181,10 +181,9 @@ def load_table(ctx, table, project, dataset, config): Create or replace bigquery table TABLE with data contained in `data/{TABLE}.csv`. If TABLE=ALL, then all tables defined in `data` folder are created. """ - config_data = load_config(config) from .load_table import load_table as upload_table - project = project or get_config_value('project', config_data, config) - dataset = dataset or get_config_value('dataset', config_data, config) + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) datasets = [dataset.strip() for dataset in dataset.split(',')] if table == 'ALL': tables = [f.replace('.yaml', '') for f in os.listdir(TABLES_FOLDER) if f.endswith('.yaml')] @@ -215,9 +214,8 @@ def generate(ctx, project, dataset, config): """ Generate markdown files for documentation from yaml bigfunctions files """ - config_data = load_config(config) - project = project or get_config_value('project', config_data, config) - dataset = dataset or get_config_value('dataset', config_data, config) + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) generate_doc(project, dataset) os.system('mkdocs build') @@ -231,9 +229,8 @@ def serve(ctx, project, dataset, config): """ Serve docs locally on http://localhost:8000 """ - config_data = load_config(config) - project = project or get_config_value('project', config_data, config) - dataset = dataset or get_config_value('dataset', config_data, config) + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) generate_doc(project, dataset) class EventHandler(RegexMatchingEventHandler): From 11ba04791dc0fa7af6841ca059679b64b2afae71 Mon Sep 17 00:00:00 2001 From: Unytics <111615732+unytics@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:54:42 +0200 Subject: [PATCH 10/12] Update cli.py --- bigfun/cli.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/bigfun/cli.py b/bigfun/cli.py index ddaeaed4..5fc305c7 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -102,15 +102,13 @@ def generate_bigfunctions_list_markdown(bigfunctions): help_headers_color='yellow', help_options_color='cyan' ) -@click.pass_context -def cli(ctx): - ctx.ensure_object(dict) +def cli(): + pass @cli.command() @click.argument('bigfunction') -@click.pass_context -def get(ctx, bigfunction): +def get(bigfunction): """ Download BIGFUNCTION yaml file from unytics/bigfunctions github repo """ @@ -126,8 +124,7 @@ def get(ctx, bigfunction): @cli.command() @click.argument('bigfunction') @click.option('--config', default='config.yaml', help='Path to the config file') -@click.pass_context -def test(ctx, bigfunction, config): +def test(bigfunction, config): """ Test BIGFUNCTION """ @@ -142,8 +139,7 @@ def test(ctx, bigfunction, config): @click.option('--project', help='Google Cloud project where the function will be deployed') @click.option('--dataset', help='BigQuery dataset name where the function will be deployed') @click.option('--config', default='config.yaml', help='Path to the config file') -@click.pass_context -def deploy(ctx, bigfunction, project, dataset, config): +def deploy(bigfunction, project, dataset, config): """ Deploy BIGFUNCTION @@ -175,8 +171,7 @@ def deploy(ctx, bigfunction, project, dataset, config): @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') @click.option('--config', default='config.yaml', help='Path to the config file') -@click.pass_context -def load_table(ctx, table, project, dataset, config): +def load_table(table, project, dataset, config): """ Create or replace bigquery table TABLE with data contained in `data/{TABLE}.csv`. If TABLE=ALL, then all tables defined in `data` folder are created. @@ -209,8 +204,7 @@ def docs(): @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') @click.option('--config', default='config.yaml', help='Path to the config file') -@click.pass_context -def generate(ctx, project, dataset, config): +def generate(project, dataset, config): """ Generate markdown files for documentation from yaml bigfunctions files """ @@ -224,8 +218,7 @@ def generate(ctx, project, dataset, config): @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') @click.option('--config', default='config.yaml', help='Path to the config file') -@click.pass_context -def serve(ctx, project, dataset, config): +def serve(project, dataset, config): """ Serve docs locally on http://localhost:8000 """ From 68ad89d6f784cbbf55e5693d1fdd4a55137ca373 Mon Sep 17 00:00:00 2001 From: Unytics <111615732+unytics@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:05:42 +0200 Subject: [PATCH 11/12] Update cli.py --- bigfun/cli.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/bigfun/cli.py b/bigfun/cli.py index 5fc305c7..e690b0dc 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -16,15 +16,17 @@ THIS_FOLDER = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') WEBSITE_CONFIG_FOLDER = THIS_FOLDER + '/website' CATEGORIES_DOC_TEMPLATE_FILENAME = f'{THIS_FOLDER}/templates/categories.md' - +CONFIGS = {} def get_config_value(name, config_filename): - if os.path.exists(config_filename): - with open(config_filename, encoding='utf-8') as file: - config = yaml.safe_load(file) or {} - else: - config = {} - + if config_filename not in CONFIGS: + if os.path.exists(config_filename): + with open(config_filename, encoding='utf-8') as file: + CONFIGS[config_filename] = yaml.safe_load(file) or {} + else: + CONFIGS[config_filename] = {} + + config = CONFIGS[config_filename] if name in config: return config[name] From c8f477b878eee67abe67fba3ee3872c7de5bdf61 Mon Sep 17 00:00:00 2001 From: Unytics <111615732+unytics@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:07:33 +0200 Subject: [PATCH 12/12] Update cli.py --- bigfun/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bigfun/cli.py b/bigfun/cli.py index e690b0dc..9aff8df5 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -238,6 +238,4 @@ def on_any_event(self, event): # observer.start() # bf.generate_doc(project, dataset) os.system('mkdocs serve') - # observer.start() - # bf.generate_doc(project, dataset) - os.system('mkdocs serve') +