diff --git a/.github/workflows/db_export.yaml b/.github/workflows/db_export.yaml index 411b7dd2..84b1a9b1 100644 --- a/.github/workflows/db_export.yaml +++ b/.github/workflows/db_export.yaml @@ -44,8 +44,11 @@ jobs: - name: Run Pipeline env: - BASIC_AUTHS: ${{ secrets.BASIC_AUTHS }} + HDX_SITE: ${{ env.HDX_SITE }} HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }} + PREPREFIX: ${{ secrets.HDX_PIPELINE_PREPREFIX }} + USER_AGENT: ${{ secrets.USER_AGENT }} + BASIC_AUTHS: ${{ secrets.BASIC_AUTHS }} run: python3.11 -m hapi.pipelines.app -db "postgresql+psycopg://postgres:postgres@localhost:5432/hapi" - name: Dump PostgreSQL Views diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc0fdb7..4c3d0f48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.10.2] - 2024-09-24 + +### Changed + +- Fix how pipeline appears in MixPanel + ## [0.10.1] - 2024-09-20 ### Changed diff --git a/requirements.txt b/requirements.txt index 3bf1551b..5b5e32d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,13 +44,13 @@ filelock==3.16.1 # via virtualenv frictionless==5.17.1 # via hdx-python-utilities -google-auth==2.34.0 +google-auth==2.35.0 # via # google-auth-oauthlib # gspread google-auth-oauthlib==1.2.1 # via gspread -greenlet==3.1.0 +greenlet==3.1.1 # via sqlalchemy gspread==6.1.2 # via hdx-python-scraper diff --git a/src/hapi/pipelines/app/__main__.py b/src/hapi/pipelines/app/__main__.py index 83aae4dc..7d8b22c4 100755 --- a/src/hapi/pipelines/app/__main__.py +++ b/src/hapi/pipelines/app/__main__.py @@ -3,6 +3,7 @@ import argparse import logging from os import getenv +from os.path import expanduser, join from typing import Dict, Optional from hapi_schema.views import prepare_hapi_views @@ -39,12 +40,6 @@ def parse_args(): parser = argparse.ArgumentParser(description="HAPI pipelines") - parser.add_argument("-hk", "--hdx-key", default=None, help="HDX api key") - parser.add_argument("-ua", "--user-agent", default=None, help="user agent") - parser.add_argument("-pp", "--preprefix", default=None, help="preprefix") - parser.add_argument( - "-hs", "--hdx-site", default=None, help="HDX site to use" - ) parser.add_argument( "-db", "--db-uri", default=None, help="Database connection string" ) @@ -172,20 +167,6 @@ def main( if __name__ == "__main__": args = parse_args() - hdx_key = args.hdx_key - if hdx_key is None: - hdx_key = getenv("HDX_KEY") - user_agent = args.user_agent - if user_agent is None: - user_agent = getenv("USER_AGENT") - if user_agent is None: - user_agent = "hapi-pipelines" - preprefix = args.preprefix - if preprefix is None: - preprefix = getenv("PREPREFIX") - hdx_site = args.hdx_site - if hdx_site is None: - hdx_site = getenv("HDX_SITE", "prod") db_uri = args.db_uri if db_uri is None: db_uri = getenv("DB_URI") @@ -233,10 +214,8 @@ def main( project_config_dict = add_defaults(project_config_dict) facade( main, - hdx_key=hdx_key, - user_agent=user_agent, - preprefix=preprefix, - hdx_site=hdx_site, + user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"), + user_agent_lookup=lookup, project_config_dict=project_config_dict, db_uri=db_uri, db_params=args.db_params,