Merge pull request #64 from cityofaustin/add-pipeline-field

Add Pipeline value from Zenhub to the DTS Portal's Knack App

.dockerignore

@@ -1,2 +1,3 @@
 .git
 env_file
+.env

.gitignore

@@ -1,3 +1,5 @@
+.vscode
+
 # custom
 .DS_Store
 env.list

README.md

@@ -1,9 +1,11 @@
 # atd-service-bot
+
 A bot that creates github issues from our Knack-based intake form.
 
 ## Get it going
 
-1. Configure these environmental variables, which you can grab from 1Password:
+Place the following environment variables in `.env`, which you can grab from 1Password:
+
 - KNACK_DTS_PORTAL_SERVICE_BOT_USERNAME
 - KNACK_DTS_PORTAL_SERVICE_BOT_PASSWORD
 - KNACK_API_KEY
@@ -16,9 +18,22 @@ A bot that creates github issues from our Knack-based intake form.
 - SOCRATA_APP_TOKEN
 - SOCRATA_RESOURCE_ID (of the Socrata dataset for issues)
 
-2. Pull the docker image (`atddocker/atd-service-bot`) or install the package dependencies: `pip install -r requirements.txt`
+* Run this command to build the docker container:
+
+```bash
+docker compose build
+```
+
+- Run this command to be dropped into a development environment that simulates the
+  the environment that the docker container / program will be in when it's kicked off by
+  airflow.
 
-3. Run `python intake.py`
+```bash
+docker compose run service-bot
+```
+
+- While inside the shell provided by the container, you can run the scripts, and you
+  are able to continue to edit them outside of the container because they are bind-mounted in.
 
 ## How it works
 
@@ -29,9 +44,11 @@ The bot runs on Airflow and fetches new service requests from our Knack app. It
 Keeping our bot happy is contingent on not changing the information our bot expects to process.
 
 You must update `config/config.py` if you change any of these things in the DTS Knack app:
+
 - Workgroup names
 - Any pre-defined choice-list options (impact, need, application, workgroup, etc)
 
 ...or if you change any of these things on github:
+
 - repo names
-- labels
+- labels

docker-compose.yaml

@@ -0,0 +1,10 @@
+version: "3.8"
+
+services:
+  service-bot:
+    build: .
+    env_file:
+      - .env
+    volumes:
+      - .:/app
+    entrypoint: /bin/bash

env_template

@@ -0,0 +1,11 @@
+SOCRATA_RESOURCE_ID=
+ZENHUB_ACCESS_TOKEN=
+GITHUB_ACCESS_TOKEN=
+SOCRATA_ENDPOINT=
+SOCRATA_API_KEY_ID=
+SOCRATA_API_KEY_SECRET=
+SOCRATA_APP_TOKEN=
+KNACK_API_KEY=
+KNACK_APP_ID=
+KNACK_DTS_PORTAL_SERVICE_BOT_USERNAME=
+KNACK_DTS_PORTAL_SERVICE_BOT_PASSWORD=

gh_index_issues_to_dts_portal.py

@@ -1,72 +1,138 @@
 #!/usr/bin/env python3
+
 """ Create or update "Index" issues in the DTS Portal from Github.
 
 We use the DTS portal to track our project (aka "Index" issue) scoring. This
 script keeps the issue titles in the DTS portal in sync with Github by fetching these
 issues from the atd-data-tech repo and either creating new project records in the DTS
-portal or updating existing project records if their title does not match the title of
-the issue on Github."""
+portal or updating existing project records if their title or pipeline status does 
+not match the title of the issue on Github."""
+
 import logging
 import os
 import sys
 
 from github import Github
+import requests
 import knackpy
 
 
-def build_payload(project_records, project_issues, title_field, issue_number_field):
+ZENHUB_REPO = {"id": 140626918, "name": "cityofaustin/atd-data-tech"}
+WORKSPACE_ID = "5caf7dc6ecad11531cc418ef"
+ZENHUB_ACCESS_TOKEN = os.environ["ZENHUB_ACCESS_TOKEN"]
+KNACK_API_KEY = os.environ["KNACK_API_KEY"]
+KNACK_APP_ID = os.environ["KNACK_APP_ID"]
+GITHUB_ACCESS_TOKEN = os.environ["GITHUB_ACCESS_TOKEN"]
+REPO = "cityofaustin/atd-data-tech"
+KNACK_OBJ = "object_30"
+KNACK_TITLE_FIELD = "field_538"
+KNACK_ISSUE_NUMBER_FIELD = "field_492"
+KNACK_PIPELINE_FIELD = "field_649"  # production
+
+
+def get_zenhub_metadata(workspace_id, token, repo_id, timeout=60):
+    """
+    Fetch Zenhub metadata for a given repo.
+    """
+    url = f"https://api.zenhub.com/p2/workspaces/{workspace_id}/repositories/{repo_id}/board"
+    params = {"access_token": token}
+    res = requests.get(url, params=params, timeout=timeout)
+    res.raise_for_status()
+    return res.json()
+
+
+def find_pipeline_by_issue(data, issue_number):
+    """
+    Find the pipeline for a given issue number in the Zenhub metadata.
+    Return None if none found.
+    """
+    for pipeline in data["pipelines"]:
+        for issue in pipeline["issues"]:
+            if issue["issue_number"] == issue_number:
+                return pipeline["name"]
+    return None
+
+
+def find_knack_record_by_issue(knack_records, issue_number):
+    """
+    Find a knack record by issue number.
+    Return None if none found.
+    """
+    for record in knack_records:
+        if record[KNACK_ISSUE_NUMBER_FIELD] == issue_number:
+            return record
+    return None
+
+
+def build_payload(project_records, project_issues):
+    """
+    Build a payload to update knack records based on github issues and Zenhub metadata.
+    Take care to create the payload for each issue so that it will work as a create or
+    update call depending on if the record already exists in the Knack app.
+    """
+    zenhub_metadata = get_zenhub_metadata(
+        WORKSPACE_ID, ZENHUB_ACCESS_TOKEN, ZENHUB_REPO["id"]
+    )
+
     payload = []
-    for issue in project_issues:
-        # search for a corresponding Knack record for each project issue
-        matched = False
-        for record in project_records:
-            issue_number_knack = record[issue_number_field]
-            if not issue_number_knack:
-                continue
-            if issue_number_knack == issue.number:
-                matched = True
-                # matching Knack record found, so check if the titles match
-                title_knack = record[title_field]
-                if title_knack != issue.title:
-                    # records without matching title will be updated with github issue
-                    # title
-                    payload.append({"id": record["id"], title_field: issue.title})
-                    break
-        if not matched:
-            # this issue needs a new project record created in Knack
-            payload.append({issue_number_field: issue.number, title_field: issue.title})
+    for issue in project_issues:  # iterate over gh issues
+        pipeline = find_pipeline_by_issue(zenhub_metadata, issue.number)
+
+        # ZH metadata does not include closed issues
+        if issue.state == "closed":
+            pipeline = "Closed"
+
+        knack_record = find_knack_record_by_issue(project_records, issue.number)
+
+        if knack_record:
+            issue_payload = {"id": knack_record["id"]}
+            title_knack = knack_record[KNACK_TITLE_FIELD]
+            pipeline_knack = knack_record[KNACK_PIPELINE_FIELD]
+
+            if title_knack != issue.title:
+                issue_payload[KNACK_TITLE_FIELD] = issue.title
+            if pipeline_knack != pipeline:
+                issue_payload[KNACK_PIPELINE_FIELD] = pipeline
+            if title_knack != issue.title or pipeline_knack != pipeline:
+                payload.append(issue_payload)
+        else:
+            issue_payload = {
+                KNACK_ISSUE_NUMBER_FIELD: issue.number,
+                KNACK_TITLE_FIELD: issue.title,
+            }
+            if pipeline is not None:
+                issue_payload[KNACK_PIPELINE_FIELD] = pipeline
+            payload.append(issue_payload)
     return payload
 
 
 def main():
     logging.info("Starting...")
-    KNACK_API_KEY = os.environ["KNACK_API_KEY"]
-    KNACK_APP_ID = os.environ["KNACK_APP_ID"]
-    GITHUB_ACCESS_TOKEN = os.environ["GITHUB_ACCESS_TOKEN"]
-    REPO = "cityofaustin/atd-data-tech"
-    KNACK_OBJ = "object_30"
-    KNACK_TITLE_FIELD = "field_538"
-    KNACK_ISSUE_NUMBER_FIELD = "field_492"
 
+    # setup and get the knack records
     app = knackpy.App(app_id=KNACK_APP_ID, api_key=KNACK_API_KEY)
     project_records = app.get(KNACK_OBJ)
 
+    # setup an instance of our github client
     g = Github(GITHUB_ACCESS_TOKEN)
     repo = g.get_repo(REPO)
 
+    # iterate over the github client's issues and build our working data
     project_issues_paginator = repo.get_issues(state="all", labels=["Project Index"])
     project_issues = [issue for issue in project_issues_paginator]
 
+    # build the payload out of the github and knack state of the data
     knack_payload = build_payload(
-        project_records, project_issues, KNACK_TITLE_FIELD, KNACK_ISSUE_NUMBER_FIELD
+        project_records,
+        project_issues,
     )
 
+    # iterate over the payload issuing an update or create as needed per issue
+    # into knack. Report the status to be logged in airflow.
     logging.info(f"Creating/updating {len(knack_payload)} issues")
-
     for record in knack_payload:
         method = "update" if record.get("id") else "create"
         app.record(data=record, method=method, obj=KNACK_OBJ)
-
     logging.info(f"{len(knack_payload)} records processed.")
 
 

requirements.txt

@@ -1,4 +1,4 @@
 knackpy==1.0.*
 pygithub==1.53.*
 requests==2.24.*
-sodapy==2.1.*
+sodapy==2.1.*