Add prescription refresh workflow for image analysis handler

prescriptions-refresh-job/base/argo-workflows/prescriptions-refresh-image-analysis.yaml

@@ -0,0 +1,110 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: prescriptions-refresh-image-analysis
+spec:
+  templates:
+    - name: image-analysis
+      resubmitPendingPods: true
+      container:
+        name: image-analysis
+        image: prescriptions-refresh-job
+        env:
+          - name: THOTH_PRESCRIPTIONS_DRY_RUN
+            value: 1
+          - name: SENTRY_DSN
+            valueFrom:
+              secretKeyRef:
+                name: thoth
+                key: sentry-dsn
+          - name: GIT_SSH_COMMAND
+            # Needed to keep weak host verification for git clone.
+            value: "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+          - name: THOTH_DEPLOYMENT_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: thoth
+                key: deployment-name
+          - name: THOTH_USER_API_HOST
+            valueFrom:
+              configMapKeyRef:
+                name: prescription-refresh-job
+                key: user-api
+          - name: APP_SCRIPT
+            value: app.sh
+          - name: THOTH_PRESCRIPTIONS_REFRESH_RANDOMIZE
+            value: "1"
+          - name: THOTH_PRESCRIPTIONS_REFRESH_SUBCOMMAND
+            value: thoth-image-analysis
+          - name: THOTH_PRESCRIPTIONS_REFRESH_CONFIGURED_IMAGES
+            valueFrom:
+              configMapKeyRef:
+                key: configured-images
+                name: prescriptions-refresh-job
+          - name: THOTH_PRESCRIPTIONS_REFRESH_ML_PACKAGES
+            valueFrom:
+              configMapKeyRef:
+                key: configured-packages-ml
+                name: prescriptions-refresh-job
+          - name: THOTH_PRESCRIPTIONS_REFRESH_NLP_PACKAGES
+            valueFrom:
+              configMapKeyRef:
+                key: configured-packages-nlp
+                name: prescriptions-refresh-job
+          - name: THOTH_PRESCRIPTIONS_REFRESH_CV_PACKAGES
+            valueFrom:
+              configMapKeyRef:
+                key: configured-packages-cv
+                name: prescriptions-refresh-job
+          - name: GITHUB_PRIVATE_KEY_PATH
+            value: /opt/app-root/src/.github/github-privatekey
+          - name: THOTH_LOGGING_NO_JSON
+            valueFrom:
+              configMapKeyRef:
+                name: thoth
+                key: logging-no-json
+          - name: THOTH_PRESCRIPTIONS_REFRESH_GITHUB_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: kebechet
+                key: github-oauth-token
+          - name: THOTH_PRESCRIPTIONS_REFRESH_QUAY_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: thoth
+                key: quay-token
+          - name: GITHUB_APP_ID
+            valueFrom:
+              secretKeyRef:
+                name: kebechet
+                key: GITHUB_APP_ID
+          - name: THOTH_PRESCRIPTIONS_REFRESH_GITHUB_LABELS
+            value: bot
+          - name: THOTH_PRESCRIPTIONS_REFRESH_REPO
+            value: [email protected]:thoth-station/prescriptions.git
+          - name: THOTH_PRESCRIPTIONS_REFRESH_DEBUG
+            value: "1"
+          - name: PROMETHEUS_PUSHGATEWAY_HOST
+            valueFrom:
+              configMapKeyRef:
+                key: pushgateway-host
+                name: prometheus
+          - name: PROMETHEUS_PUSHGATEWAY_PORT
+            valueFrom:
+              configMapKeyRef:
+                key: pushgateway-port
+                name: prometheus
+        resources:
+          limits:
+            cpu: 250m
+            memory: 256Mi
+          requests:
+            cpu: 250m
+            memory: 256Mi
+        volumeMounts:
+          - name: ssh-config
+            mountPath: /opt/app-root/src/.ssh
+          - name: github-app-privatekey
+            mountPath: /opt/app-root/src/.github/
+            readOnly: true

prescriptions-refresh-job/base/argo-workflows/prescriptions-refresh.yaml

@@ -147,3 +147,9 @@ spec:
                 template: pypi-release
               continueOn:
                 failed: true
+            - name: image-analysis
+              templateRef:
+                name: prescriptions-refresh-image-analysis
+                template: image-analysis
+              continueOn:
+                failed: true

prescriptions-refresh-job/base/configmap.yaml

@@ -5,3 +5,19 @@ metadata:
   name: prescriptions-refresh-job
 data:
   configured-images: ""
+  configured-packages-ml: |
+    keras
+    scikit-learn
+    torch
+    tensorflow
+  configured-packages-nlp: |
+    gensim
+    nltk
+    spacy
+    transformers
+  configured-packages-cv: |
+    opencv-python
+    pillow
+    pytesseract
+    torchvision
+  user-api: ""

prescriptions-refresh-job/base/kustomization.yaml

@@ -6,6 +6,7 @@ commonLabels:
   app.kubernetes.io/managed-by: aicoe-thoth-devops
 resources:
   - argo-workflows/prescriptions-refresh-gh.yaml
+  - argo-workflows/prescriptions-refresh-image-analysis.yaml
   - argo-workflows/prescriptions-refresh-pg.yaml
   - argo-workflows/prescriptions-refresh-pypi.yaml
   - argo-workflows/prescriptions-refresh-quay.yaml

prescriptions-refresh-job/overlays/ocp4-stage/configmap.yaml

@@ -0,0 +1,23 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: prescriptions-refresh-job
+data:
+  configured-images: ""
+  configured-packages-ml: |
+    keras
+    scikit-learn
+    torch
+    tensorflow
+  configured-packages-nlp: |
+    gensim
+    nltk
+    spacy
+    transformers
+  configured-packages-cv: |
+    opencv-python
+    pillow
+    pytesseract
+    torchvision
+  user-api: "stage.thoth-station.ninja"

prescriptions-refresh-job/overlays/ocp4-stage/cronworkflow.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: prescriptions-refresh
+spec:
+  suspend: false

prescriptions-refresh-job/overlays/ocp4-stage/imagestreamtag.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: image.openshift.io/v1
+kind: ImageStream
+metadata:
+  name: prescriptions-refresh-job
+spec:
+  tags:
+    - name: latest
+      from:
+        kind: DockerImage
+        name: quay.io/thoth-station/prescriptions-refresh-job:v0.5.0
+      importPolicy: {}
+      referencePolicy:
+        type: Local

prescriptions-refresh-job/overlays/ocp4-stage/kustomization.yaml

@@ -0,0 +1,76 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+commonLabels:
+  app: thoth
+  component: prescriptions-refresh-job
+resources:
+  - ../../base
+  - thoth-notification.yaml
+patchesStrategicMerge:
+  - configmap.yaml
+  - imagestreamtag.yaml
+  - cronworkflow.yaml
+patches:
+  - patch: |-
+      - op: add
+        path: /metadata/namespace
+        value: "thoth-infra-stage"
+    target:
+      group: batch
+      version: v1
+      kind: Job
+      annotationSelector: "operation=chat-notification"
+  - patch: |-
+      - op: replace
+        path: /spec/templates/0/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/1/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/2/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/3/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/4/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/5/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+      - op: replace
+        path: /spec/templates/6/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: prescriptions-refresh-gh
+  - patch: |-
+      - op: replace
+        path: /spec/templates/0/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: prescriptions-refresh-scorecards
+  - patch: |-
+      - op: replace
+        path: /spec/templates/0/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: prescriptions-refresh-pg
+  - patch: |-
+      - op: replace
+        path: /spec/templates/0/container/image
+        value: image-registry.openshift-image-registry.svc:5000/thoth-middletier-stage/prescriptions-refresh-job:latest
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name:  prescriptions-refresh-quay

prescriptions-refresh-job/overlays/ocp4-stage/thoth-notification.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: prescriptions-refresh-job-chat-notification-success
+  annotations:
+    operation: chat-notification
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: HookSucceeded
+spec:
+  ttlSecondsAfterFinished: 5
+  backoffLimit: 2
+  template:
+    spec:
+      containers:
+        - name: chat-notification
+          image: registry.access.redhat.com/ubi8/ubi
+          command:
+            - "curl"
+            - "-X"
+            - "POST"
+            - "-H"
+            - "Content-Type: application/json; charset=UTF-8"
+            - "-d"
+            - "{'text':'Successfully synchronized *prescriptions-refresh-job* to *OCP4-STAGE*, see <https://argocd.operate-first.cloud/applications/stage-thoth-prescriptions-refresh-job|ArgoCD UI> 🚚'}"
+            - "$(THOTH_DEVOPS_WEBHOOK_URL)"
+          env:
+            - name: THOTH_DEVOPS_WEBHOOK_URL
+              valueFrom:
+                secretKeyRef:
+                  name: chat-notification
+                  key: thoth-devops
+      restartPolicy: Never
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: prescriptions-refresh-job-chat-notification-fail
+  annotations:
+    operation: chat-notification
+    argocd.argoproj.io/hook: SyncFail
+    argocd.argoproj.io/hook-delete-policy: HookSucceeded
+spec:
+  ttlSecondsAfterFinished: 5
+  backoffLimit: 2
+  template:
+    spec:
+      containers:
+        - name: chat-notification
+          image: registry.access.redhat.com/ubi8/ubi
+          command:
+            - "curl"
+            - "-X"
+            - "POST"
+            - "-H"
+            - "Content-Type: application/json; charset=UTF-8"
+            - "-d"
+            - "{'text':'🔥 *FAILED* to sync *prescriptions-refresh-job* to *OCP4-STAGE*, see <https://argocd.operate-first.cloud/applications/stage-thoth-prescriptions-refresh-job|ArgoCD UI>'}"
+            - "$(THOTH_DEVOPS_WEBHOOK_URL)"
+          env:
+            - name: THOTH_DEVOPS_WEBHOOK_URL
+              valueFrom:
+                secretKeyRef:
+                  name: chat-notification
+                  key: thoth-devops
+      restartPolicy: Never

prescriptions-refresh-job/overlays/test/configmap.yaml

@@ -0,0 +1,23 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: prescriptions-refresh-job
+data:
+  configured-images: ""
+  configured-packages-ml: |
+    keras
+    scikit-learn
+    torch
+    tensorflow
+  configured-packages-nlp: |
+    gensim
+    nltk
+    spacy
+    transformers
+  configured-packages-cv: |
+    opencv-python
+    pillow
+    pytesseract
+    torchvision
+  user-api: "test.thoth-station.ninja"

prescriptions-refresh-job/overlays/test/kustomization.yaml

@@ -9,6 +9,7 @@ resources:
 patchesStrategicMerge:
   - imagestreamtag.yaml
   - cronworkflow.yaml
+  - configmap.yaml
 patches:
   - patch: |-
       - op: replace