diff --git a/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml b/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml index 25fa8d05..4eb92a97 100644 --- a/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/0_new_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to suggest a new BigFunction! - + ## πŸ™ Thanks for taking the time to suggest a new BigFunction! + Please fill the below fields to submit your idea. - type: checkboxes id: idea_does_not_exist attributes: label: Check the idea has not already been suggested - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find my idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Anew-bigfunction) required: true diff --git a/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml b/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml index 20fc9ac4..feb1dba5 100644 --- a/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/1_bug_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to report a bug in a BigFunction! - + ## πŸ™ Thanks for taking the time to report a bug in a BigFunction! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check the bug has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the bug in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) required: true @@ -21,7 +21,7 @@ body: id: title_is_ok attributes: label: Edit `function_name` and the short error description in title above - description: Edit the `function_name` and the short error description in the issue title above. + description: Edit the `function_name` and the short error description in the issue title above. options: - label: I wrote the correct function name and a short error description in the title above required: true diff --git a/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml b/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml index fc593a8b..f3e7026c 100644 --- a/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml +++ b/.github/ISSUE_TEMPLATE/2_improve_bigfunction.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to help us improve! - + ## πŸ™ Thanks for taking the time to help us improve! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check your idea has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfunction) required: true @@ -21,7 +21,7 @@ body: id: title_is_ok attributes: label: Edit `function_name` and the short idea description in title above - description: Edit the `function_name` and the short idea description in the issue title above. + description: Edit the `function_name` and the short idea description in the issue title above. options: - label: I wrote the correct function name and a short idea description in the title above required: true diff --git a/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml b/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml index 77b6bfe7..36cadab0 100644 --- a/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml +++ b/.github/ISSUE_TEMPLATE/3_bug_bigfun_cli.yaml @@ -6,14 +6,14 @@ body: - type: markdown attributes: value: | - ## πŸ™ Thanks for taking the time to report a bug of `bigfun` Command-Line-Interface! - + ## πŸ™ Thanks for taking the time to report a bug of `bigfun` Command-Line-Interface! + Please fill the below fields to submit it. - type: checkboxes id: idea_does_not_exist attributes: label: Check the bug has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the bug in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Abug-bigfun-CLI) required: true diff --git a/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml b/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml index 418fc22f..4f3f2cff 100644 --- a/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml +++ b/.github/ISSUE_TEMPLATE/4_improve_bigfun_cli.yaml @@ -7,13 +7,13 @@ body: attributes: value: | ## πŸ™ Thanks for taking the time to help us improve! - + Please fill the below fields to submit your idea. - type: checkboxes id: idea_does_not_exist attributes: label: Check your idea has not already been reported - description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. + description: Check in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) if your idea has not already been suggested. If it has, please don't create a new issue but add a comment on the existing one such as `+1` to upvote it. options: - label: I could not find the idea in [existing issues](https://github.com/unytics/bigfunctions/issues?q=is%3Aissue+is%3Aopen+label%3Aimprove-bigfun-CLI) required: true diff --git a/.github/workflows/deploy_pages.yml b/.github/workflows/deploy_pages.yml index 94c36e0e..69b70e18 100644 --- a/.github/workflows/deploy_pages.yml +++ b/.github/workflows/deploy_pages.yml @@ -8,8 +8,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - run: pip install -e . diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..184cec8a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,24 @@ +name: lint + +on: + pull_request: + push: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install dependencies + run: | + pip install -e .[dev] + - name: Run pre-commit + run: | + pre-commit run --all-files diff --git a/.gitignore b/.gitignore index 6337186d..801a9af0 100644 --- a/.gitignore +++ b/.gitignore @@ -144,4 +144,4 @@ site/ TEST venv -.venv/ \ No newline at end of file +.venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..746d903b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-json + - id: check-toml + - id: check-yaml + - id: detect-private-key + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + additional_dependencies: + - types-PyYAML + - repo: https://github.com/rhysd/actionlint + rev: v1.7.1 + hooks: + - id: actionlint-docker + # TODO enable the hook + # - repo: https://github.com/PyCQA/bandit + # rev: 1.7.9 + # hooks: + # - id: bandit + # name: bandit (python) + # args: [--configfile, bandit.yaml, --severity-level, all] + # additional_dependencies: [bandit] + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + args: [--strict, -c=.yamllint] + # TODO enable the hook + # - repo: https://github.com/hadolint/hadolint + # rev: v2.13.0-beta + # hooks: + # - id: hadolint-docker diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..44c09792 --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +rules: + trailing-spaces: disable + key-duplicates: {} + colons: + max-spaces-before: 0 + max-spaces-after: 1 + indentation: + spaces: 2 + check-multi-line-strings: false diff --git a/README.md b/README.md index 1fe13f3f..d137482a 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Commands: ### 5.3 Create you first function πŸ‘· -Functions are defined as yaml files under `bigfunctions` folder. To create your first function locally, the easiest is to download an existing yaml file of unytics/bigfunctions Github repo. +Functions are defined as yaml files under `bigfunctions` folder. To create your first function locally, the easiest is to download an existing yaml file of unytics/bigfunctions Github repo. For instance to download `is_email_valid.yaml` into bigfunctions folder, do: @@ -229,4 +229,3 @@ BigFunctions is fully open-source. Any contribution is more than welcome πŸ€—! - diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 00000000..8cbd6620 --- /dev/null +++ b/bandit.yaml @@ -0,0 +1,6 @@ +# SEE https://bandit.readthedocs.io/en/latest/config.html +exclude_dirs: + - "tests" +skips: + # hardcoded_sql_expressions + - "B608" diff --git a/bigfun/bigfunctions.py b/bigfun/bigfunctions.py index 5a740b57..15fe3b44 100644 --- a/bigfun/bigfunctions.py +++ b/bigfun/bigfunctions.py @@ -1,14 +1,16 @@ -import tempfile -import re +import functools +import json import os +import re import shutil -import json -import functools +import tempfile -import yaml import jinja2 +import yaml -from .utils import BigQuery, CloudRun, handle_error, print_success, print_info, print_command, build_and_upload_npm_package, merge_dict +from .utils import (BigQuery, CloudRun, build_and_upload_npm_package, + handle_error, merge_dict, print_command, print_info, + print_success) BIGFUNCTIONS_FOLDER = 'bigfunctions' DEFAULT_CONFIG_FILENAME = './config.yaml' diff --git a/bigfun/cli.py b/bigfun/cli.py index d881e43a..9aff8df5 100644 --- a/bigfun/cli.py +++ b/bigfun/cli.py @@ -1,31 +1,34 @@ +import multiprocessing import os import shutil -import multiprocessing +import click import jinja2 import yaml -import click from click_help_colors import HelpColorsGroup -from watchdog.observers import Observer from watchdog.events import RegexMatchingEventHandler +from watchdog.observers import Observer from . import bigfunctions as bf from . import utils - TABLES_FOLDER = 'data' THIS_FOLDER = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') WEBSITE_CONFIG_FOLDER = THIS_FOLDER + '/website' CATEGORIES_DOC_TEMPLATE_FILENAME = f'{THIS_FOLDER}/templates/categories.md' -CONFIG_FILENAME = 'config.yaml' -CONFIG = {} -if os.path.exists(CONFIG_FILENAME): - CONFIG = yaml.safe_load(open(CONFIG_FILENAME, encoding='utf-8').read()) or {} +CONFIGS = {} +def get_config_value(name, config_filename): + if config_filename not in CONFIGS: + if os.path.exists(config_filename): + with open(config_filename, encoding='utf-8') as file: + CONFIGS[config_filename] = yaml.safe_load(file) or {} + else: + CONFIGS[config_filename] = {} -def get_config_value(name): - if name in CONFIG: - return CONFIG[name] + config = CONFIGS[config_filename] + if name in config: + return config[name] text, default = { 'project': ("Default GCP project where to deploy bigfunctions", None), @@ -33,10 +36,10 @@ def get_config_value(name): 'project_for_tests': ("Default GCP project where to deploy bigfunctions for TESTING purposes", None), 'dataset_for_tests': ("Default dataset where to deploy bigfunctions for TESTING purposes", None), # eu,us,asia_east1,asia_east2,asia_northeast1,asia_northeast2,asia_northeast3,asia_south1,asia_southeast1,australia_southeast1,europe_north1,europe_west1,europe_west2,europe_west3,europe_west4,europe_west6,northamerica_northeast1,southamerica_east1,us_central1,us_east1,us_east4,us_west1,us_west2 }[name] - CONFIG[name] = click.prompt(text, default=default) - with open(CONFIG_FILENAME, 'w', encoding='utf-8') as outfile: - yaml.dump(CONFIG, outfile, default_flow_style=False) - return CONFIG[name] + config[name] = click.prompt(text, default=default) + with open(config_filename, 'w', encoding='utf-8') as outfile: + yaml.dump(config, outfile, default_flow_style=False) + return config[name] def generate_doc(project, dataset): @@ -108,9 +111,9 @@ def cli(): @cli.command() @click.argument('bigfunction') def get(bigfunction): - ''' + """ Download BIGFUNCTION yaml file from unytics/bigfunctions github repo - ''' + """ if not os.path.isdir('bigfunctions'): os.makedirs('bigfunctions') url = f'https://raw.githubusercontent.com/unytics/bigfunctions/main/bigfunctions/{bigfunction}.yaml' @@ -122,12 +125,13 @@ def get(bigfunction): @cli.command() @click.argument('bigfunction') -def test(bigfunction): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +def test(bigfunction, config): + """ Test BIGFUNCTION - ''' - project = get_config_value('project_for_tests') - dataset = get_config_value('dataset_for_tests') + """ + project = get_config_value('project_for_tests', config) + dataset = get_config_value('dataset_for_tests', config) bigfunction = bf.BigFunction(bigfunction, project=project, dataset=dataset) bigfunction.test() @@ -136,14 +140,15 @@ def test(bigfunction): @click.argument('bigfunction') @click.option('--project', help='Google Cloud project where the function will be deployed') @click.option('--dataset', help='BigQuery dataset name where the function will be deployed') -def deploy(bigfunction, project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +def deploy(bigfunction, project, dataset, config): + """ Deploy BIGFUNCTION Deploy the function defined in `bigfunctions/{BIGFUNCTION}.yaml` file. If BIGFUNCTION = 'ALL' then all bigfunctions contained in bigfunctions folder are deployed. - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) datasets = [dataset.strip() for dataset in dataset.split(',')] bigfunctions = [bigfunction.strip() for bigfunction in bigfunction.split(',')] if bigfunction == 'ALL': @@ -163,19 +168,19 @@ def deploy(bigfunction, project, dataset): ) - @cli.command() @click.argument('table') @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def load_table(table, project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +def load_table(table, project, dataset, config): + """ Create or replace bigquery table TABLE with data contained in `data/{TABLE}.csv`. If TABLE=ALL, then all tables defined in `data` folder are created. - ''' + """ from .load_table import load_table as upload_table - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) datasets = [dataset.strip() for dataset in dataset.split(',')] if table == 'ALL': tables = [f.replace('.yaml', '') for f in os.listdir(TABLES_FOLDER) if f.endswith('.yaml')] @@ -191,35 +196,36 @@ def load_table(table, project, dataset): @cli.group() def docs(): - ''' + """ Generate, serve and publish documentation - ''' + """ pass @docs.command() @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def generate(project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +def generate(project, dataset, config): + """ Generate markdown files for documentation from yaml bigfunctions files - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) generate_doc(project, dataset) os.system('mkdocs build') - @docs.command() @click.option('--project', help='Google Cloud project where the table is created') @click.option('--dataset', help='BigQuery dataset name where the table is created') -def serve(project, dataset): - ''' +@click.option('--config', default='config.yaml', help='Path to the config file') +def serve(project, dataset, config): + """ Serve docs locally on http://localhost:8000 - ''' - project = project or get_config_value('project') - dataset = dataset or get_config_value('dataset') + """ + project = project or get_config_value('project', config) + dataset = dataset or get_config_value('dataset', config) generate_doc(project, dataset) class EventHandler(RegexMatchingEventHandler): @@ -232,3 +238,4 @@ def on_any_event(self, event): # observer.start() # bf.generate_doc(project, dataset) os.system('mkdocs serve') + diff --git a/bigfun/datastore/index.yaml b/bigfun/datastore/index.yaml index bc37fb50..2ce19f82 100644 --- a/bigfun/datastore/index.yaml +++ b/bigfun/datastore/index.yaml @@ -1,7 +1,5 @@ indexes: - -- kind: bigfunction_call - properties: - - name: user_bigfunction_date - - name: row_count - + - kind: bigfunction_call + properties: + - name: user_bigfunction_date + - name: row_count diff --git a/bigfun/load_table.py b/bigfun/load_table.py index 198776c8..5ed5f603 100644 --- a/bigfun/load_table.py +++ b/bigfun/load_table.py @@ -22,4 +22,3 @@ def load_table(table): job_config = google.cloud.bigquery.job.LoadJobConfig(**load_config) bigquery.load_table_from_file(file, table, job_config=job_config).result() print_success('successfully loaded ' + table) - diff --git a/bigfun/templates/bookmarklet.js b/bigfun/templates/bookmarklet.js index 224de348..dc74451b 100644 --- a/bigfun/templates/bookmarklet.js +++ b/bigfun/templates/bookmarklet.js @@ -121,7 +121,3 @@ loadFunnelGraphJs(); loadChartJs(); loadGoogleChart(); setInterval(run, 100); - - - - diff --git a/bigfun/templates/categories.md b/bigfun/templates/categories.md index c0fcd755..9ebc2eaf 100644 --- a/bigfun/templates/categories.md +++ b/bigfun/templates/categories.md @@ -45,4 +45,3 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** {% endfor %} {% endfor %} - diff --git a/bigfun/templates/function_py.py b/bigfun/templates/function_py.py index 3b28e173..a07719ac 100644 --- a/bigfun/templates/function_py.py +++ b/bigfun/templates/function_py.py @@ -1,14 +1,13 @@ -import time import re -import json import datetime +import json +import time import traceback import uuid -from flask import Flask, request, jsonify import google.auth import google.cloud.error_reporting - +from flask import Flask, jsonify, request error_reporter = google.cloud.error_reporting.Client() app = Flask(__name__) diff --git a/bigfun/templates/function_py.sql b/bigfun/templates/function_py.sql index effbcb0c..e2f51507 100644 --- a/bigfun/templates/function_py.sql +++ b/bigfun/templates/function_py.sql @@ -5,4 +5,4 @@ options ( endpoint = '{{ remote_endpoint }}', user_defined_context = [("dataset_location", "{{ dataset_location }}")] {% if max_batching_rows %}, max_batching_rows = {{ max_batching_rows }}{% endif %} -); \ No newline at end of file +); diff --git a/bigfun/templates/function_py_test.py b/bigfun/templates/function_py_test.py index 7722b200..ec9df754 100644 --- a/bigfun/templates/function_py_test.py +++ b/bigfun/templates/function_py_test.py @@ -13,4 +13,4 @@ def run(): {{ code | indent(4) }} result = run() -print(result) \ No newline at end of file +print(result) diff --git a/bigfun/templates/procedure_test.sql b/bigfun/templates/procedure_test.sql index 5709e768..f0c401db 100644 --- a/bigfun/templates/procedure_test.sql +++ b/bigfun/templates/procedure_test.sql @@ -1,3 +1,3 @@ {% if bigfunction.type == 'procedure' %}call{% elif bigfunction.type == 'table_function' %}select * from{% else %}select{% endif %} {{ dataset }}.{{ bigfunction.name }}({% for argument in example.arguments %}{{ argument | replace('{BIGFUNCTIONS_DATASET}', dataset) | replace('\n', '\n ') }}{% if not loop.last %}, {% endif %}{% endfor %}){% if bigfunction.type == 'procedure' %};{% elif 'output' in bigfunction and bigfunction.type != 'table_function' %} as {{ bigfunction.output.name }}{% endif %} {% if bigfunction.type == 'procedure' and bigfunction.template %}select html from bigfunction_result;{% endif %} -{%- if bigfunction.type == 'procedure' and example.output %}select * from bigfunction_result;{% endif %} \ No newline at end of file +{%- if bigfunction.type == 'procedure' and example.output %}select * from bigfunction_result;{% endif %} diff --git a/bigfun/utils.py b/bigfun/utils.py index 9b18d225..5d915343 100644 --- a/bigfun/utils.py +++ b/bigfun/utils.py @@ -1,20 +1,18 @@ -import tempfile -import shutil +import math import os -import sys +import shutil import subprocess -import math +import sys +import tempfile import urllib.request - +import click import google.api_core.exceptions import google.auth.exceptions import google.cloud.bigquery import google.cloud.bigquery_connection_v1 import google.cloud.storage import google.iam.v1.policy_pb2 -import click - def print_color(msg): @@ -366,4 +364,4 @@ def download(url, destination_filename): try: urllib.request.urlretrieve(url, destination_filename) except Exception as e: - handle_error(f'Could not download file at url `{url}`. Reason: {e}') \ No newline at end of file + handle_error(f'Could not download file at url `{url}`. Reason: {e}') diff --git a/bigfun/website/theme_overrides/main.html b/bigfun/website/theme_overrides/main.html index 539c0399..681f76e3 100644 --- a/bigfun/website/theme_overrides/main.html +++ b/bigfun/website/theme_overrides/main.html @@ -225,7 +225,3 @@

Supercharge BigQuery
with BigFunctions

{% endblock %} - - - - diff --git a/bigfunctions/README.md b/bigfunctions/README.md index d4072c82..10a421c6 100644 --- a/bigfunctions/README.md +++ b/bigfunctions/README.md @@ -66,7 +66,7 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** | `us-west2` | `bigfunctions.us_west2` | | `us-west3` | `bigfunctions.us_west3` | | `us-west4` | `bigfunctions.us_west4` | - + @@ -269,6 +269,3 @@ BigFunctions are open-source BigQuery routines that give you **SQL-superpowers** - [timestamp_from_unix_date_time(unix_date_time, date_time_part)](timestamp_from_unix_date_time.md): Interprets `unix_date_time` as the number of `date_time_part` since `1970-01-01 00:00:00 UTC`. - [timestamp_to_unix_date_time(timestamp_expression, date_time_part)](timestamp_to_unix_date_time.md): Returns the number of `date_time_part` since `1970-01-01 00:00:00 UTC`. - [upsert(query_or_table_or_view, destination_table, insertion_mode, primary_keys, recency_field)](upsert.md): Merges `query_or_table_or_view` into the `destination_table`. - - - diff --git a/bigfunctions/ask_ai.yaml b/bigfunctions/ask_ai.yaml index 239f63a7..1d4c45e9 100644 --- a/bigfunctions/ask_ai.yaml +++ b/bigfunctions/ask_ai.yaml @@ -102,4 +102,3 @@ requirements: | quotas: max_rows_per_user_per_day: 1000 max_rows_per_query: 10 - diff --git a/bigfunctions/deduplicate_rows.yaml b/bigfunctions/deduplicate_rows.yaml index ae860dc7..a3b1438b 100644 --- a/bigfunctions/deduplicate_rows.yaml +++ b/bigfunctions/deduplicate_rows.yaml @@ -15,7 +15,7 @@ examples: - description: "Returns table with duplicate rows removed." arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +-----+-----+ | id1 | id2 | +-----+-----+ diff --git a/bigfunctions/deidentify.yaml b/bigfunctions/deidentify.yaml index 4bcd8e48..fd5b6b9b 100644 --- a/bigfunctions/deidentify.yaml +++ b/bigfunctions/deidentify.yaml @@ -88,4 +88,4 @@ code: | requirements: | google-cloud-dlp quotas: - max_rows_per_user_per_day: 10000 \ No newline at end of file + max_rows_per_user_per_day: 10000 diff --git a/bigfunctions/detect_sensitive_info.yaml b/bigfunctions/detect_sensitive_info.yaml index ab45eb46..5d587749 100644 --- a/bigfunctions/detect_sensitive_info.yaml +++ b/bigfunctions/detect_sensitive_info.yaml @@ -67,4 +67,4 @@ code: | ] return json.dumps(results, ensure_ascii=False) if results else None requirements: | - google-cloud-dlp \ No newline at end of file + google-cloud-dlp diff --git a/bigfunctions/exchange_rate.yaml b/bigfunctions/exchange_rate.yaml index 0b2a228c..6cde3080 100644 --- a/bigfunctions/exchange_rate.yaml +++ b/bigfunctions/exchange_rate.yaml @@ -57,4 +57,4 @@ code: | requirements: | yfinance cloud_run: - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/explore_column.yaml b/bigfunctions/explore_column.yaml index 000d8b10..8599a7f3 100644 --- a/bigfunctions/explore_column.yaml +++ b/bigfunctions/explore_column.yaml @@ -226,4 +226,3 @@ template: | - diff --git a/bigfunctions/explore_dataset.yaml b/bigfunctions/explore_dataset.yaml index 0f213c27..89cb4825 100644 --- a/bigfunctions/explore_dataset.yaml +++ b/bigfunctions/explore_dataset.yaml @@ -102,5 +102,3 @@ template: |-
{% endfor %} - - diff --git a/bigfunctions/explore_events.yaml b/bigfunctions/explore_events.yaml index 143a5559..4070c2f1 100644 --- a/bigfunctions/explore_events.yaml +++ b/bigfunctions/explore_events.yaml @@ -252,4 +252,3 @@ template: | } })(); - diff --git a/bigfunctions/explore_funnels.yaml b/bigfunctions/explore_funnels.yaml index f019ef9a..700bc686 100644 --- a/bigfunctions/explore_funnels.yaml +++ b/bigfunctions/explore_funnels.yaml @@ -247,6 +247,3 @@ template: | })(); - - - diff --git a/bigfunctions/export_to_datastore.yaml b/bigfunctions/export_to_datastore.yaml index f84d3452..1b4c450e 100644 --- a/bigfunctions/export_to_datastore.yaml +++ b/bigfunctions/export_to_datastore.yaml @@ -86,4 +86,4 @@ cloud_run: concurrency: 16 max_instances: 10 quotas: - max_rows_per_user_per_day: 1000000 \ No newline at end of file + max_rows_per_user_per_day: 1000000 diff --git a/bigfunctions/find_greater_value.yaml b/bigfunctions/find_greater_value.yaml index 644c1359..01e9dc8a 100644 --- a/bigfunctions/find_greater_value.yaml +++ b/bigfunctions/find_greater_value.yaml @@ -39,4 +39,4 @@ code: | where value >= x order by offset limit 1 - ) \ No newline at end of file + ) diff --git a/bigfunctions/find_lower_value.yaml b/bigfunctions/find_lower_value.yaml index 901e7902..91cbb488 100644 --- a/bigfunctions/find_lower_value.yaml +++ b/bigfunctions/find_lower_value.yaml @@ -39,4 +39,4 @@ code: | where value <= x order by offset limit 1 - ) \ No newline at end of file + ) diff --git a/bigfunctions/find_value.yaml b/bigfunctions/find_value.yaml index ae280129..1cf70831 100644 --- a/bigfunctions/find_value.yaml +++ b/bigfunctions/find_value.yaml @@ -26,4 +26,4 @@ examples: - "[3, 4]" - "7" output: "null" -code: (select offset from unnest(arr) as x with offset where x = value order by offset limit 1) \ No newline at end of file +code: (select offset from unnest(arr) as x with offset where x = value order by offset limit 1) diff --git a/bigfunctions/generate_face_embedding.yaml b/bigfunctions/generate_face_embedding.yaml index 80ba96f7..b51935e8 100644 --- a/bigfunctions/generate_face_embedding.yaml +++ b/bigfunctions/generate_face_embedding.yaml @@ -66,4 +66,4 @@ quotas: cloud_run: memory: 2048Mi concurrency: 1 - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/get_json.yaml b/bigfunctions/get_json.yaml index 140acb5e..94b21721 100644 --- a/bigfunctions/get_json.yaml +++ b/bigfunctions/get_json.yaml @@ -25,4 +25,4 @@ code: | url, json_set(headers, '$.Content-Type', 'application/json') ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/get_latest_partition_timestamp.yaml b/bigfunctions/get_latest_partition_timestamp.yaml index 7c3af0f3..d96f4e41 100644 --- a/bigfunctions/get_latest_partition_timestamp.yaml +++ b/bigfunctions/get_latest_partition_timestamp.yaml @@ -15,7 +15,7 @@ examples: - description: "" arguments: - '"my_project.my_dataset.my_table"' - output: | + output: | +----------------------------+ | latest_partition_timestamp | +----------------------------+ diff --git a/bigfunctions/get_value.yaml b/bigfunctions/get_value.yaml index 3c97989c..a8891a35 100644 --- a/bigfunctions/get_value.yaml +++ b/bigfunctions/get_value.yaml @@ -31,4 +31,4 @@ examples: - "[struct('a' as key, 8 as value), struct('a' as key, 9 as value)]" - "'a'" output: "8" -code: (select value from unnest(key_value_items) item where item.key = search_key limit 1) \ No newline at end of file +code: (select value from unnest(key_value_items) item where item.key = search_key limit 1) diff --git a/bigfunctions/get_view_history.yaml b/bigfunctions/get_view_history.yaml index 25529a25..3644f59f 100644 --- a/bigfunctions/get_view_history.yaml +++ b/bigfunctions/get_view_history.yaml @@ -45,4 +45,4 @@ code: | to_json(struct( fully_qualified_view as fully_qualified_view )) - ); \ No newline at end of file + ); diff --git a/bigfunctions/gregorian2hijri.yaml b/bigfunctions/gregorian2hijri.yaml index dcdfe7d3..ee503ac0 100644 --- a/bigfunctions/gregorian2hijri.yaml +++ b/bigfunctions/gregorian2hijri.yaml @@ -4,7 +4,7 @@ author: name: Fahed Sabellioglu url: https://stackoverflow.com/users/23326889/fahed-sabellioglu avatar_url: "https://lh3.googleusercontent.com/a/ACg8ocLdYt_9sKc80I-jaH_0VVz6zaSpYZ3-wV3XiNh8Z8QBiA=k-s256" -description: | +description: | Convert Gregorian Date to Hijri Date (taken from [here](https://stackoverflow.com/questions/78072960/convert-dates-gregorian-to-hijri-bigquery#answer-78079872)) arguments: - name: gregorian_date diff --git a/bigfunctions/html2pdf.yaml b/bigfunctions/html2pdf.yaml index 6c0cbba4..44b186a0 100644 --- a/bigfunctions/html2pdf.yaml +++ b/bigfunctions/html2pdf.yaml @@ -55,4 +55,3 @@ cloud_run: memory: 1024Mi concurrency: 1 max_instances: 10 - diff --git a/bigfunctions/ip2asn.yaml b/bigfunctions/ip2asn.yaml index a2d44b16..a4634808 100644 --- a/bigfunctions/ip2asn.yaml +++ b/bigfunctions/ip2asn.yaml @@ -72,4 +72,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2continent.yaml b/bigfunctions/ip2continent.yaml index b81e836d..a3b58064 100644 --- a/bigfunctions/ip2continent.yaml +++ b/bigfunctions/ip2continent.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2continent_name.yaml b/bigfunctions/ip2continent_name.yaml index 0f82a7e0..526974d3 100644 --- a/bigfunctions/ip2continent_name.yaml +++ b/bigfunctions/ip2continent_name.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2country.yaml b/bigfunctions/ip2country.yaml index 202d4ef8..4312eabd 100644 --- a/bigfunctions/ip2country.yaml +++ b/bigfunctions/ip2country.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip2country_name.yaml b/bigfunctions/ip2country_name.yaml index b0515b00..f27961cd 100644 --- a/bigfunctions/ip2country_name.yaml +++ b/bigfunctions/ip2country_name.yaml @@ -74,4 +74,3 @@ cloud_run: add_volume: '"name=storage,type=cloud-storage,bucket=bigfunctions_storage"' add_volume_mount: '"volume=storage,mount-path=/storage"' execution-environment: gen2 - diff --git a/bigfunctions/ip_range2ip_networks.yaml b/bigfunctions/ip_range2ip_networks.yaml index becc99f2..93f32573 100644 --- a/bigfunctions/ip_range2ip_networks.yaml +++ b/bigfunctions/ip_range2ip_networks.yaml @@ -34,4 +34,4 @@ code: | requirements: | netaddr cloud_run: - max_instances: 10 \ No newline at end of file + max_instances: 10 diff --git a/bigfunctions/json2excel.yaml b/bigfunctions/json2excel.yaml index 4e5cb2b0..f4013c11 100644 --- a/bigfunctions/json2excel.yaml +++ b/bigfunctions/json2excel.yaml @@ -26,4 +26,4 @@ code: | xlsx.utils.book_append_sheet(wb, ws, "Sheet1"); return xlsx.write(wb, {type: "base64", bookType: "xlsx"}); npm_packages: - - xlsx@0.18.5 \ No newline at end of file + - xlsx@0.18.5 diff --git a/bigfunctions/json2xml.yaml b/bigfunctions/json2xml.yaml index 3645a178..985b3c1a 100644 --- a/bigfunctions/json2xml.yaml +++ b/bigfunctions/json2xml.yaml @@ -34,4 +34,4 @@ code: | return null } npm_packages: - - fast-xml-parser@4.2.7 \ No newline at end of file + - fast-xml-parser@4.2.7 diff --git a/bigfunctions/json_items.yaml b/bigfunctions/json_items.yaml index 870032aa..2b5abf01 100644 --- a/bigfunctions/json_items.yaml +++ b/bigfunctions/json_items.yaml @@ -34,4 +34,4 @@ code: | for (const i in obj) { arr.push({key: i, value: obj[i] }); } - return arr \ No newline at end of file + return arr diff --git a/bigfunctions/json_keys.yaml b/bigfunctions/json_keys.yaml index f703f22a..1115c151 100644 --- a/bigfunctions/json_keys.yaml +++ b/bigfunctions/json_keys.yaml @@ -26,4 +26,4 @@ code: | catch { throw('could not parse ' + json_string) } - return Object.keys(obj) \ No newline at end of file + return Object.keys(obj) diff --git a/bigfunctions/json_query.yaml b/bigfunctions/json_query.yaml index 3602905b..6234f1fd 100644 --- a/bigfunctions/json_query.yaml +++ b/bigfunctions/json_query.yaml @@ -50,4 +50,4 @@ code: | } return JSON.stringify(jmespath.search(obj, query)); npm_packages: - - jmespath@0.16.0 \ No newline at end of file + - jmespath@0.16.0 diff --git a/bigfunctions/json_schema.yaml b/bigfunctions/json_schema.yaml index 4dd247e7..9a57d4da 100644 --- a/bigfunctions/json_schema.yaml +++ b/bigfunctions/json_schema.yaml @@ -68,4 +68,4 @@ code: | } const schema = getSchema(data); - return schema \ No newline at end of file + return schema diff --git a/bigfunctions/json_values.yaml b/bigfunctions/json_values.yaml index a3a5ea19..204334f5 100644 --- a/bigfunctions/json_values.yaml +++ b/bigfunctions/json_values.yaml @@ -26,4 +26,4 @@ code: | catch { throw('could not parse ' + json_string) } - return Object.values(obj) \ No newline at end of file + return Object.values(obj) diff --git a/bigfunctions/last_value.yaml b/bigfunctions/last_value.yaml index abce9f70..285f4ea6 100644 --- a/bigfunctions/last_value.yaml +++ b/bigfunctions/last_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 2, 3]" output: "3" -code: arr[ordinal(array_length(arr))] \ No newline at end of file +code: arr[ordinal(array_length(arr))] diff --git a/bigfunctions/levenshtein.yaml b/bigfunctions/levenshtein.yaml index e8d7638c..64f127db 100644 --- a/bigfunctions/levenshtein.yaml +++ b/bigfunctions/levenshtein.yaml @@ -22,4 +22,4 @@ examples: code: | return js_levenshtein(string1 || '', string2 || ''); npm_packages: - - js-levenshtein@1.1.6 \ No newline at end of file + - js-levenshtein@1.1.6 diff --git a/bigfunctions/list_bigquery_resources_in_current_project.yaml b/bigfunctions/list_bigquery_resources_in_current_project.yaml index 99eaf89a..e4fed5c3 100644 --- a/bigfunctions/list_bigquery_resources_in_current_project.yaml +++ b/bigfunctions/list_bigquery_resources_in_current_project.yaml @@ -288,4 +288,4 @@ code: | ), regexp_replace(id, r'[^a-zA-Z]', ' ') as broken_id, from joined - ''', '{PROJECT}', @@project_id); \ No newline at end of file + ''', '{PROJECT}', @@project_id); diff --git a/bigfunctions/max_value.yaml b/bigfunctions/max_value.yaml index d79efb26..fd296f95 100644 --- a/bigfunctions/max_value.yaml +++ b/bigfunctions/max_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 4, 3]" output: "4" -code: (select max(value) from unnest(arr) value) \ No newline at end of file +code: (select max(value) from unnest(arr) value) diff --git a/bigfunctions/min_value.yaml b/bigfunctions/min_value.yaml index e5dc6e10..f26f369e 100644 --- a/bigfunctions/min_value.yaml +++ b/bigfunctions/min_value.yaml @@ -18,4 +18,4 @@ examples: arguments: - "[1, 4, 3]" output: "1" -code: (select min(value) from unnest(arr) value) \ No newline at end of file +code: (select min(value) from unnest(arr) value) diff --git a/bigfunctions/parse_date.yaml b/bigfunctions/parse_date.yaml index 09d35724..c7f3f47c 100644 --- a/bigfunctions/parse_date.yaml +++ b/bigfunctions/parse_date.yaml @@ -61,4 +61,3 @@ code: | ) else null end - diff --git a/bigfunctions/parse_url.yaml b/bigfunctions/parse_url.yaml index 4af28fa5..ed15efe6 100644 --- a/bigfunctions/parse_url.yaml +++ b/bigfunctions/parse_url.yaml @@ -25,4 +25,4 @@ code: | regexp_extract(url, r'\?(.*)') as query, regexp_extract(url, r'#(.*)') as ref, regexp_extract(url, r'^([a-za-z]+)://') as protocol - ) \ No newline at end of file + ) diff --git a/bigfunctions/percentile_value.yaml b/bigfunctions/percentile_value.yaml index 8377c1b9..af545fe2 100644 --- a/bigfunctions/percentile_value.yaml +++ b/bigfunctions/percentile_value.yaml @@ -81,4 +81,4 @@ code: | ) end end - ) \ No newline at end of file + ) diff --git a/bigfunctions/phone_number_info.yaml b/bigfunctions/phone_number_info.yaml index 2b072b34..bd597d75 100644 --- a/bigfunctions/phone_number_info.yaml +++ b/bigfunctions/phone_number_info.yaml @@ -173,4 +173,4 @@ code: | isNonGeographic: phoneNumber.isNonGeographic(), }; npm_packages: - - libphonenumber-js/max@1.10.51 \ No newline at end of file + - libphonenumber-js/max@1.10.51 diff --git a/bigfunctions/post.yaml b/bigfunctions/post.yaml index 6afda3bb..b2f34dc4 100644 --- a/bigfunctions/post.yaml +++ b/bigfunctions/post.yaml @@ -46,4 +46,4 @@ code: | requirements: | requests quotas: - max_rows_per_query: 10 \ No newline at end of file + max_rows_per_query: 10 diff --git a/bigfunctions/precision_recall_auc.yaml b/bigfunctions/precision_recall_auc.yaml index 658936f7..4b25f983 100644 --- a/bigfunctions/precision_recall_auc.yaml +++ b/bigfunctions/precision_recall_auc.yaml @@ -29,4 +29,4 @@ code: | select (recall - lag(recall) over (order by recall)) * (precision + lag(precision) over (order by recall)) / 2 as auc_contrib from {BIGFUNCTIONS_DATASET}.precision_recall_curve(predictions) ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/precision_recall_curve.yaml b/bigfunctions/precision_recall_curve.yaml index 34545ee6..eb755d1a 100644 --- a/bigfunctions/precision_recall_curve.yaml +++ b/bigfunctions/precision_recall_curve.yaml @@ -56,4 +56,4 @@ code: | from unnested union all select 1, 0 - order by recall desc \ No newline at end of file + order by recall desc diff --git a/bigfunctions/prophet.yaml b/bigfunctions/prophet.yaml index ce1626ad..b66f5935 100644 --- a/bigfunctions/prophet.yaml +++ b/bigfunctions/prophet.yaml @@ -39,4 +39,4 @@ requirements: | prophet cloud_run: max_instances: 20 - concurrency: 1 \ No newline at end of file + concurrency: 1 diff --git a/bigfunctions/remove_accents.yaml b/bigfunctions/remove_accents.yaml index 44ea9f70..548237d1 100644 --- a/bigfunctions/remove_accents.yaml +++ b/bigfunctions/remove_accents.yaml @@ -16,4 +16,4 @@ examples: arguments: - "'VoilΓ  !'" output: Voila ! -code: regexp_replace(normalize(str, nfd), r"\pm", '') \ No newline at end of file +code: regexp_replace(normalize(str, nfd), r"\pm", '') diff --git a/bigfunctions/remove_extra_whitespaces.yaml b/bigfunctions/remove_extra_whitespaces.yaml index c5d33cfb..42eb1ab7 100644 --- a/bigfunctions/remove_extra_whitespaces.yaml +++ b/bigfunctions/remove_extra_whitespaces.yaml @@ -18,4 +18,4 @@ examples: arguments: - "'Hi Madison and Mateusz!\\n How are you doing?'" output: Hi Madison and Mateusz! How are you doing? -code: trim(regexp_replace(str, r'\s\s+', ' ')) \ No newline at end of file +code: trim(regexp_replace(str, r'\s\s+', ' ')) diff --git a/bigfunctions/remove_words.yaml b/bigfunctions/remove_words.yaml index 1599725f..805af9f9 100644 --- a/bigfunctions/remove_words.yaml +++ b/bigfunctions/remove_words.yaml @@ -20,4 +20,4 @@ examples: - "['can', 'eat']" output: "I candies" code: | - (select regexp_replace(string, r'\b(' || array_to_string(words_to_remove, '|') || r')\b', '')) \ No newline at end of file + (select regexp_replace(string, r'\b(' || array_to_string(words_to_remove, '|') || r')\b', '')) diff --git a/bigfunctions/roc_auc.yaml b/bigfunctions/roc_auc.yaml index dffc2e1e..8a9b03c9 100644 --- a/bigfunctions/roc_auc.yaml +++ b/bigfunctions/roc_auc.yaml @@ -33,4 +33,4 @@ code: | select (false_positive_rate - lag(false_positive_rate) over (order by false_positive_rate)) * (true_positive_rate + lag(true_positive_rate) over (order by false_positive_rate)) / 2 as auc_contrib from {BIGFUNCTIONS_DATASET}.roc_curve(predictions) ) - ) \ No newline at end of file + ) diff --git a/bigfunctions/roc_curve.yaml b/bigfunctions/roc_curve.yaml index c4746653..4567d6f0 100644 --- a/bigfunctions/roc_curve.yaml +++ b/bigfunctions/roc_curve.yaml @@ -59,4 +59,4 @@ code: | from unnested union all select 0, 0 - order by false_positive_rate, true_positive_rate \ No newline at end of file + order by false_positive_rate, true_positive_rate diff --git a/bigfunctions/send_mail.yaml b/bigfunctions/send_mail.yaml index 0d441bc7..31f0c69f 100644 --- a/bigfunctions/send_mail.yaml +++ b/bigfunctions/send_mail.yaml @@ -507,4 +507,4 @@ secrets: description: "Sendgrid API Key" documentation_link: https://docs.sendgrid.com/for-developers/sending-email/api-getting-started quotas: - max_rows_per_query: 10 \ No newline at end of file + max_rows_per_query: 10 diff --git a/bigfunctions/sort_values_desc.yaml b/bigfunctions/sort_values_desc.yaml index 77328c64..f0601147 100644 --- a/bigfunctions/sort_values_desc.yaml +++ b/bigfunctions/sort_values_desc.yaml @@ -16,4 +16,4 @@ examples: arguments: - "[1, 4, 3]" output: "[4, 3, 1]" -code: array(select value from unnest(arr) as value order by value desc) \ No newline at end of file +code: array(select value from unnest(arr) as value order by value desc) diff --git a/bigfunctions/translate.yaml b/bigfunctions/translate.yaml index 27c6a742..8490b31b 100644 --- a/bigfunctions/translate.yaml +++ b/bigfunctions/translate.yaml @@ -19,7 +19,7 @@ examples: - '"Salut πŸ‘‹ Florian. Merci d''avoir contribuΓ© !"' - "'en'" output: "Hello πŸ‘‹ Florian. Thanks for contributing!" -init_code: | +init_code: | from google.cloud import translate_v2 as translate translator = translate.Client() code: | diff --git a/bigfunctions/translated_month_name.yaml b/bigfunctions/translated_month_name.yaml index cc5bfb7f..ff65c652 100644 --- a/bigfunctions/translated_month_name.yaml +++ b/bigfunctions/translated_month_name.yaml @@ -30,4 +30,4 @@ code: | where lower(locale) = lower(language) and month_nb = extract(month from date) - ) \ No newline at end of file + ) diff --git a/bigfunctions/upload_to_gsheet.yaml b/bigfunctions/upload_to_gsheet.yaml index c05278d4..8feefdf2 100644 --- a/bigfunctions/upload_to_gsheet.yaml +++ b/bigfunctions/upload_to_gsheet.yaml @@ -113,4 +113,4 @@ quotas: max_rows_per_query: 1 cloud_run: service_account: 749389685934-compute@developer.gserviceaccount.com - memory: 512Mi \ No newline at end of file + memory: 512Mi diff --git a/bigfunctions/xml2json.yaml b/bigfunctions/xml2json.yaml index 2dfe4b30..1a2cd8fe 100644 --- a/bigfunctions/xml2json.yaml +++ b/bigfunctions/xml2json.yaml @@ -33,4 +33,4 @@ code: | } return null; npm_packages: - - fast-xml-parser@4.1.3 \ No newline at end of file + - fast-xml-parser@4.1.3 diff --git a/bigfunctions/xml_extract.yaml b/bigfunctions/xml_extract.yaml index 1b1d83c6..c6003f6b 100644 --- a/bigfunctions/xml_extract.yaml +++ b/bigfunctions/xml_extract.yaml @@ -41,4 +41,4 @@ code: | } npm_packages: - xmldom@0.6.0 - - xpath@0.0.32 \ No newline at end of file + - xpath@0.0.32 diff --git a/data/README.md b/data/README.md index ac0e5212..f2b2dd43 100644 --- a/data/README.md +++ b/data/README.md @@ -321,4 +321,3 @@ with open('translated_months.csv', 'w', encoding='utf-8') as out: --- - diff --git a/data/holidays.yaml b/data/holidays.yaml index 96d7a883..3b42929e 100644 --- a/data/holidays.yaml +++ b/data/holidays.yaml @@ -19,4 +19,4 @@ schema: type: date description: "public holiday local date" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/data/natality.sql b/data/natality.sql index bbd1030a..0cd6d5dc 100644 --- a/data/natality.sql +++ b/data/natality.sql @@ -33,4 +33,4 @@ CREATE TABLE `bigfunctions.us.natality` record_weight INT64 OPTIONS(description="1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area.") ) as -select * from `bigquery-public-data.samples.natality` tablesample system (10 percent) \ No newline at end of file +select * from `bigquery-public-data.samples.natality` tablesample system (10 percent) diff --git a/data/natality.yaml b/data/natality.yaml index 9eea26c1..657c58c8 100644 --- a/data/natality.yaml +++ b/data/natality.yaml @@ -134,4 +134,4 @@ schema: description: "Age of the father when the child was born." - name: record_weight type: INT64 - description: "1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area." \ No newline at end of file + description: "1 or 2, where 1 is a row from a full-reporting area, and 2 is a row from a 50% sample area." diff --git a/data/sales.csv b/data/sales.csv index ac4fc9b0..fa7fc3bb 100644 --- a/data/sales.csv +++ b/data/sales.csv @@ -996,4 +996,4 @@ 4,45,8,2023-01-13 1,62,10,2023-05-01 6,97,10,2023-03-30 -1,62,5,2023-01-06 \ No newline at end of file +1,62,5,2023-01-06 diff --git a/data/sales.yaml b/data/sales.yaml index 20470472..1f359073 100644 --- a/data/sales.yaml +++ b/data/sales.yaml @@ -14,4 +14,4 @@ schema: type: date description: "Sale date" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/data/sample_graph.yaml b/data/sample_graph.yaml index 22b5b74a..6509e182 100644 --- a/data/sample_graph.yaml +++ b/data/sample_graph.yaml @@ -8,4 +8,4 @@ schema: type: int64 description: "node id connected to node1" load_config: - source_format: CSV \ No newline at end of file + source_format: CSV diff --git a/setup.py b/setup.py index 7ae30138..0f7ecd48 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,11 @@ 'click', 'click-help-colors', ], + extras_require={ + 'dev': [ + 'pre-commit>=3,<4', + ], + }, entry_points={ 'console_scripts': [ 'bigfun = bigfun.cli:cli',