Merge remote-tracking branch 'origin/johnfreeman/daq-deliverables_iss…

…ue105_standard_image' into develop

Conflicts:
	ers-dbwriter/dbwriter.py

.gitignore

@@ -0,0 +1 @@
+*~

config-service/configservice.yml → config-service/config-service-deployment.yaml

@@ -3,11 +3,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   labels:
-    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/audit: baseline
     pod-security.kubernetes.io/audit-version: latest
-    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce: baseline # unified image runs as root :(
     pod-security.kubernetes.io/enforce-version: latest
-    pod-security.kubernetes.io/warn: restricted
+    pod-security.kubernetes.io/warn: baseline
     pod-security.kubernetes.io/warn-version: latest
   name: daqconfig
 ---
@@ -35,7 +35,11 @@ spec:
         app.kubernetes.io/component: config-utility
     spec:
       containers:
-      - env:
+      - image: ghcr.io/dune-daq/microservices:latest
+        name: daqconfig-service
+        env:
+        - name: MICROSERVICE
+          value: config-service
         - name: MONGO_HOST
           valueFrom:
             secretKeyRef:
@@ -61,8 +65,6 @@ spec:
             secretKeyRef:
               key: database
               name: daqconfig-mongodb-svcbind-0
-        image: ghcr.io/dune-daq/daqconfig-service:v1.0.1
-        name: daqconfig-service
         ports:
         - containerPort: 5003
           name: http
@@ -77,8 +79,6 @@ spec:
             drop:
             - ALL
           runAsGroup: 11000
-          runAsNonRoot: true
-          runAsUser: 11000
           seccompProfile:
             type: RuntimeDefault
       securityContext:

config-service/entrypoint.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-echo "You should probably define env vars for:"
-echo " MONGO_HOST, MONGO_PORT, MONGO_USER, MONGO_PASS, MONGO_DBNAME"
+cd $(dirname $0)
+source ../entrypoint_functions.sh
+
+ensure_required_variables "MONGO_HOST MONGO_PORT MONGO_USER MONGO_PASS MONGO_DBNAME"
 
 exec gunicorn -b 0.0.0.0:5003 --workers=1 --worker-class=gevent --timeout 5000000000 --log-level=debug conf-service:app

dockerfiles/Dockerfile.microservices

@@ -0,0 +1,5 @@
+FROM ghcr.io/dune-daq/microservices_dependencies:latest
+
+RUN git clone https://github.com/DUNE-DAQ/microservices -b develop
+
+ENTRYPOINT ["/microservices/entrypoint.sh"]

dockerfiles/Dockerfile.microservices_dependencies

@@ -0,0 +1,43 @@
+FROM cern/alma9-base
+
+ARG ERSVERSION=dunedaq-v4.1.1  # For issue.proto from ers
+ARG ERSKAFKAVERSION=dunedaq-v4.1.1  # For ERSSubscriber.py from erskafka
+ARG LOCALPYDIR=/microservices_python
+
+# libaio and libnsl are needed when rpm is called on the Oracle client software 
+
+RUN yum clean all \
+  && yum -y install gcc make git unzip libaio libnsl libpq-devel libffi-devel python3-pip python3-wheel \
+  && yum clean all
+
+RUN curl -O https://download.oracle.com/otn_software/linux/instantclient/1919000/oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm \
+  && rpm -iv oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm
+
+COPY requirements.txt /
+RUN python3 -m pip install --upgrade setuptools && \
+  python3 -m pip install -r requirements.txt && \
+  python3 -m pip cache remove \*
+
+# elisa_client_api needed by the logbook microservice
+RUN git clone https://github.com/DUNE-DAQ/elisa_client_api.git && python3 -m pip install --upgrade setuptools && python3 -m pip install ./elisa_client_api 
+
+
+
+# protoc-24.3-linux-x86_64.zip is the latest zipfile available as of Sep-15-2023
+# See also https://grpc.io/docs/protoc-installation/#install-pre-compiled-binaries-any-os
+
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip && \
+  unzip protoc-24.3-linux-x86_64.zip && \
+  curl -O https://raw.githubusercontent.com/DUNE-DAQ/ers/$ERSVERSION/schema/ers/issue.proto && \
+  mkdir -p $LOCALPYDIR/ers && \
+  protoc --python_out=$LOCALPYDIR/ers issue.proto
+
+RUN mkdir -p $LOCALPYDIR/erskafka && \
+    curl https://raw.githubusercontent.com/DUNE-DAQ/erskafka/$ERSKAFKAVERSION/python/erskafka/ERSSubscriber.py -o $LOCALPYDIR/erskafka/ERSSubscriber.py
+
+ENV PYTHONPATH=$LOCALPYDIR:$PYTHONPATH
+
+# This ensures the container will run as non-root by default. Hat tip Pat Riehecky.
+# [Commented out so various entrypoint.sh scripts as of Sep-12-2023 continue to work, to be addressed later]
+# USER 60000:0
+

dockerfiles/requirements.txt

@@ -0,0 +1,32 @@
+aniso8601==9.0.1
+apispec==5.1.1
+click==8.1.2
+cx-Oracle==8.2.1
+dataclasses==0.6
+Flask==2.1.1
+Flask-Caching==2.0.2
+Flask-HTTPAuth==4.6.0
+flask-redis==0.4.0
+Flask-RESTful==0.3.9
+gevent==22.10.2
+greenlet==2.0.2
+gunicorn==20.1.0
+importlib-metadata==4.11.3
+influxdb==5.3.1
+itsdangerous==2.0.1
+Jinja2==3.1.1
+kafka-python==2.0.2
+MarkupSafe==2.0.1
+psycopg2-binary==2.9.7
+pymongo==4.0.2
+pytz==2022.1
+redis==3.5.3
+setuptools==39.2.0
+six==1.16.0
+typing_extensions==4.1.1
+Werkzeug==2.1.1
+zipp==3.8.0
+zope.event==4.6
+zope.interface==5.5.2
+protobuf==4.24.3
+

docs/README.md

@@ -0,0 +1,14 @@
+# microservices
+
+To run one of the provided microservices in this repo, the basic command is the following:
+```
+docker run --rm -e MICROSERVICE=<name of microservice> ghcr.io/dune-daq/microservices:39be
+```
+
+There are a couple of points to note:
+* The value of MICROSERVICE should be the name of a given microservice's subdirectory in this repo. As of Oct-6-2023, the available subdirectories are: `config-service`, `ers-dbwriter`, `logbook`, `opmon-dbwriter`, `runnumber-rest` and `runregistry-rest`. 
+* Most microservices require additional environment variables to be set, which can be passed using the usual docker syntax: `-e VARIABLE_NAME=<variable value>`
+* If you don't know what these additional environment variables are, you can just run the `docker` command as above without setting them; the container will exit out almost immediately but only after telling you what variables are missing
+* The `39be` tag for the image in the example above just refers to the first four characters of the git commit of the microservices repo whose `dockerfiles/Dockerfile.microservices` Docker file was used to create the image
+
+For details on a given microservice, look at its own README file (format is `docs/README_<microservice name>.md`). They may or may not be up to date, however.

entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+cd $(dirname $0)
+source ./entrypoint_functions.sh
+
+ensure_required_variables "MICROSERVICE"
+
+microservice_dir=$(dirname $0)/$MICROSERVICE
+
+if [[ ! -e ${microservice_dir}/entrypoint.sh ]]; then
+    echo "This script sees the MICROSERVICE environment variable set to \"$MICROSERVICE\" but is unable to find the corresponding entrypoint script \"${microservice_dir}/entrypoint.sh\"" >&2
+    exit 2
+fi
+
+cd $microservice_dir
+
+./entrypoint.sh
+
+retval=$?
+echo "Return value of call to ${microservice_dir}/entrypoint.sh is $retval"
+
+exit $retval

entrypoint_functions.sh

@@ -0,0 +1,24 @@
+
+function ensure_required_variables() {
+
+    vars_as_string=$1
+
+    IFS=' ' read -ra vars <<< "$vars_as_string"
+
+    missing_variable=false
+
+    for var in "${vars[@]}"; do
+
+	if [[ -v $var ]]; then
+	    echo "$var is defined as \"${!var}\"."
+	else
+	    echo "$var needs to be defined as an environment variable."
+	    missing_variable=true
+	fi
+    done
+
+    if $missing_variable ; then
+	echo "One or more required environment variables is undefined; exiting..." >&2
+	exit 3
+    fi
+}

ers-dbwriter/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cd $(dirname $0)
+source ../entrypoint_functions.sh
+
+ensure_required_variables "ERS_DBWRITER_HOST ERS_DBWRITER_PORT ERS_DBWRITER_USER ERS_DBWRITER_PASS ERS_DBWRITER_NAME ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER"
+
+python3 ./dbwriter.py

ers-dbwriter/ersdbwriter.yaml → ers-dbwriter/ers-dbwriter-deployment.yaml

@@ -2,10 +2,10 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  labels: # https://github.com/bitnami/charts/pull/17388
+  labels:
     pod-security.kubernetes.io/audit: baseline
     pod-security.kubernetes.io/audit-version: latest
-    pod-security.kubernetes.io/enforce: baseline
+    pod-security.kubernetes.io/enforce: baseline # unified image runs as root :(
     pod-security.kubernetes.io/enforce-version: latest
     pod-security.kubernetes.io/warn: baseline
     pod-security.kubernetes.io/warn-version: latest
@@ -14,7 +14,7 @@ metadata:
 # You must still deploy kafka and postgresql with their upstream manifests
 # and create a secret called ers-secret.
 # It MUST have the following keys defined:
-# ERS_DBWRITER_KAFKA_HOST, ERS_DBWRITER_KAFKA_PORT,
+# ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER
 # ERS_DBWRITER_HOST, ERS_DBWRITER_PORT
 # ERS_DBWRITER_NAME
 # ERS_DBWRITER_USER, ERS_DBWRITER_PASS
@@ -24,26 +24,53 @@ metadata:
   labels:
     app.kubernetes.io/app: ers-dbwriter
     app.kubernetes.io/component: ers-dbwriter
-  name: erskafka
+  name: ers-kafka-dbwriter
   namespace: ers
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: erskafka
+      app: erskafka-dbwriter
   template:
     metadata:
       labels:
-        app: erskafka
+        app: erskafka-dbwriter
         app.kubernetes.io/app: ers-dbwriter
         app.kubernetes.io/component: ers-dbwriter
     spec:
       containers:
-      - image: ghcr.io/dune-daq/pocket-ersdbwriter:latest
-        name: erskafka
-        envFrom:
-        - secretRef:
-            name: ers-secret
+      - image: ghcr.io/dune-daq/microservices:latest
+        name: erskafka-dbwriter
+        env:
+        - name: MICROSERVICE
+          value: ers-dbwriter
+        - name: ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER
+          value: dune-daq.kafka.svc.cluster.local:9092
+        - name: ERS_DBWRITER_HOST
+          valueFrom:
+            secretKeyRef:
+              key: host
+              name: ers-postgresql-svcbind-custom-user
+        - name: ERS_DBWRITER_PORT
+          valueFrom:
+            secretKeyRef:
+              key: port
+              name: ers-postgresql-svcbind-custom-user
+        - name: ERS_DBWRITER_USER
+          valueFrom:
+            secretKeyRef:
+              key: username
+              name: ers-postgresql-svcbind-custom-user
+        - name: ERS_DBWRITER_PASS
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: ers-postgresql-svcbind-custom-user
+        - name: ERS_DBWRITER_NAME
+          valueFrom:
+            secretKeyRef:
+              key: database
+              name: ers-postgresql-svcbind-custom-user
         resources:
           limits:
             memory: 1Gi
@@ -55,8 +82,6 @@ spec:
             drop:
             - ALL
           runAsGroup: 11000
-          runAsNonRoot: true
-          runAsUser: 11000
           seccompProfile:
             type: RuntimeDefault
       securityContext: