From bdbda06e3e3271a3cba2cad1952896928af348f2 Mon Sep 17 00:00:00 2001 From: Charlie Henry Date: Fri, 23 Feb 2024 16:01:40 -0600 Subject: [PATCH 1/3] update logging --- README.md | 10 ++++++++-- issues_to_socrata.py | 6 ++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62f0e92..3cd166c 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,18 @@ A bot that creates github issues from our Knack-based intake form. ## Get it going -1. Configure these environmental variables, which you can grab from Airflow: +1. Configure these environmental variables, which you can grab from 1Password: - KNACK_DTS_PORTAL_SERVICE_BOT_USERNAME - KNACK_DTS_PORTAL_SERVICE_BOT_PASSWORD - KNACK_API_KEY - KNACK_APP_ID - GITHUB_ACCESS_TOKEN +- ZENHUB_ACCESS_TOKEN +- SOCRATA_ENDPOINT +- SOCRATA_API_KEY_ID +- SOCRATA_API_KEY_SECRET +- SOCRATA_APP_TOKEN +- SOCRATA_RESOURCE_ID (of the Socrata dataset for issues) 2. Pull the docker image (`atddocker/atd-service-bot`) or install the package dependencies: `pip install -r requirements.txt` @@ -16,7 +22,7 @@ A bot that creates github issues from our Knack-based intake form. ## How it works -The bot runs on Airflow and fetches new service requests from our Knack app. It geneterates a github issue and applies labels and assignees based on the definitions in `config/config.py`. With the github issue successfully created, the bot submits an "edit record" form in Knack, which sets the record's `github_transmission_status` to `sent`. The form submit also triggers email notifications to the requester and to our staff. +The bot runs on Airflow and fetches new service requests from our Knack app. It generates a github issue and applies labels and assignees based on the definitions in `config/config.py`. With the github issue successfully created, the bot submits an "edit record" form in Knack, which sets the record's `github_transmission_status` to `sent`. The form submit also triggers email notifications to the requester and to our staff. ## How not to break the bot diff --git a/issues_to_socrata.py b/issues_to_socrata.py index e31c882..4966cf4 100644 --- a/issues_to_socrata.py +++ b/issues_to_socrata.py @@ -124,7 +124,7 @@ def main(): logging.info("Converting timestamps...") convert_timestamps(issues) - logging.info("Fetching zenhub data...") + logging.info("Fetching Zenhub data...") zenhub_metadata = get_zenhub_metadata(WORKSPACE_ID, ZENHUB_ACCESS_TOKEN, REPO["id"]) zenhub_metadata_index = create_zenhub_metadata_index(zenhub_metadata) @@ -149,13 +149,15 @@ def main(): logging.info(f"Uploading to Socrata...") first_chunk = True + count_chunks = 0 for chunk in chunks(issues, 1000): if first_chunk: # completely replace dataset to ensure deleted issues are flushed client.replace(SOCRATA_RESOURCE_ID, issues) first_chunk = False client.upsert(SOCRATA_RESOURCE_ID, issues) - logging.info(f"{len(chunk)} processed") + count_chunks += 1 + logging.info(f"{len(chunk) * count_chunks} processed") if __name__ == "__main__": From cb3869fbda5e3c4d23fc8a47b0f69474dad28ab7 Mon Sep 17 00:00:00 2001 From: Charlie Henry Date: Fri, 23 Feb 2024 16:04:24 -0600 Subject: [PATCH 2/3] different style --- issues_to_socrata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/issues_to_socrata.py b/issues_to_socrata.py index 4966cf4..4ca2a4e 100644 --- a/issues_to_socrata.py +++ b/issues_to_socrata.py @@ -149,15 +149,16 @@ def main(): logging.info(f"Uploading to Socrata...") first_chunk = True - count_chunks = 0 + count_processed = 0 for chunk in chunks(issues, 1000): if first_chunk: # completely replace dataset to ensure deleted issues are flushed client.replace(SOCRATA_RESOURCE_ID, issues) first_chunk = False client.upsert(SOCRATA_RESOURCE_ID, issues) - count_chunks += 1 - logging.info(f"{len(chunk) * count_chunks} processed") + count_processed += len(chunk) + logging.info(f"{len(chunk)} processed of {len(issues)}") + logging.info(f"Done uploading issues to Socrata") if __name__ == "__main__": From 2f6ac68ab17d309444f776141034e462df3dd6ad Mon Sep 17 00:00:00 2001 From: Charlie Henry Date: Fri, 23 Feb 2024 16:08:17 -0600 Subject: [PATCH 3/3] wrong var --- issues_to_socrata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issues_to_socrata.py b/issues_to_socrata.py index 4ca2a4e..ac9d753 100644 --- a/issues_to_socrata.py +++ b/issues_to_socrata.py @@ -157,7 +157,7 @@ def main(): first_chunk = False client.upsert(SOCRATA_RESOURCE_ID, issues) count_processed += len(chunk) - logging.info(f"{len(chunk)} processed of {len(issues)}") + logging.info(f"{count_processed} processed of {len(issues)}") logging.info(f"Done uploading issues to Socrata")