diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b25c15b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/README.md b/README.md deleted file mode 100644 index 2e73d91..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# microservices diff --git a/config-service/Dockerfile b/config-service/Dockerfile deleted file mode 100644 index 49f1177..0000000 --- a/config-service/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ghcr.io/dune-daq/c8-minimal:latest - -EXPOSE 5003 - -RUN yum clean all \ - && yum -y install python3 python3-pip python3-wheel \ - && yum clean all - -COPY conf-service.py configconfig.py requirements.txt / - -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install -r /requirements.txt - -COPY --chmod=755 entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] diff --git a/config-service/configservice.yml b/config-service/config-service-deployment.yaml similarity index 89% rename from config-service/configservice.yml rename to config-service/config-service-deployment.yaml index db568c3..998b7f3 100644 --- a/config-service/configservice.yml +++ b/config-service/config-service-deployment.yaml @@ -3,11 +3,11 @@ apiVersion: v1 kind: Namespace metadata: labels: - pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit: baseline pod-security.kubernetes.io/audit-version: latest - pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce: baseline # unified image runs as root :( pod-security.kubernetes.io/enforce-version: latest - pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn: baseline pod-security.kubernetes.io/warn-version: latest name: daqconfig --- @@ -35,7 +35,11 @@ spec: app.kubernetes.io/component: config-utility spec: containers: - - env: + - image: ghcr.io/dune-daq/microservices:latest + name: daqconfig-service + env: + - name: MICROSERVICE + value: config-service - name: MONGO_HOST valueFrom: secretKeyRef: @@ -61,8 +65,6 @@ spec: secretKeyRef: key: database name: daqconfig-mongodb-svcbind-0 - image: ghcr.io/dune-daq/daqconfig-service:v1.0.1 - name: daqconfig-service ports: - containerPort: 5003 name: http @@ -77,8 +79,6 @@ spec: drop: - ALL runAsGroup: 11000 - runAsNonRoot: true - runAsUser: 11000 seccompProfile: type: RuntimeDefault securityContext: diff --git a/config-service/entrypoint.sh b/config-service/entrypoint.sh index a15a923..f7713a2 100755 --- a/config-service/entrypoint.sh +++ b/config-service/entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/bash -echo "You should probably define env vars for:" -echo " MONGO_HOST, MONGO_PORT, MONGO_USER, MONGO_PASS, MONGO_DBNAME" +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "MONGO_HOST MONGO_PORT MONGO_USER MONGO_PASS MONGO_DBNAME" exec gunicorn -b 0.0.0.0:5003 --workers=1 --worker-class=gevent --timeout 5000000000 --log-level=debug conf-service:app diff --git a/config-service/requirements.txt b/config-service/requirements.txt deleted file mode 100644 index cd8e0d0..0000000 --- a/config-service/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -aniso8601==9.0.1 -apispec==5.1.1 -click==8.0.4 -dataclasses==0.8 -Flask==2.0.3 -Flask-Caching==1.10.1 -Flask-RESTful==0.3.9 -gevent==21.12.0 -greenlet==1.1.2 -gunicorn==20.1.0 -importlib-metadata==4.8.3 -itsdangerous==2.0.1 -Jinja2==3.0.3 -MarkupSafe==2.0.1 -pymongo==4.0.2 -pytz==2021.3 -six==1.16.0 -typing_extensions==4.1.1 -Werkzeug==2.0.3 -zipp==3.6.0 -zope.event==4.5.0 -zope.interface==5.4.0 diff --git a/dockerfiles/Dockerfile.microservices b/dockerfiles/Dockerfile.microservices new file mode 100644 index 0000000..d00bd1f --- /dev/null +++ b/dockerfiles/Dockerfile.microservices @@ -0,0 +1,5 @@ +FROM ghcr.io/dune-daq/microservices_dependencies:latest + +RUN git clone https://github.com/DUNE-DAQ/microservices -b develop + +ENTRYPOINT ["/microservices/entrypoint.sh"] diff --git a/dockerfiles/Dockerfile.microservices_dependencies b/dockerfiles/Dockerfile.microservices_dependencies new file mode 100644 index 0000000..df0e56d --- /dev/null +++ b/dockerfiles/Dockerfile.microservices_dependencies @@ -0,0 +1,43 @@ +FROM cern/alma9-base + +ARG ERSVERSION=dunedaq-v4.1.1 # For issue.proto from ers +ARG ERSKAFKAVERSION=dunedaq-v4.1.1 # For ERSSubscriber.py from erskafka +ARG LOCALPYDIR=/microservices_python + +# libaio and libnsl are needed when rpm is called on the Oracle client software + +RUN yum clean all \ + && yum -y install gcc make git unzip libaio libnsl libpq-devel libffi-devel python3-pip python3-wheel \ + && yum clean all + +RUN curl -O https://download.oracle.com/otn_software/linux/instantclient/1919000/oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm \ + && rpm -iv oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm + +COPY requirements.txt / +RUN python3 -m pip install --upgrade setuptools && \ + python3 -m pip install -r requirements.txt && \ + python3 -m pip cache remove \* + +# elisa_client_api needed by the logbook microservice +RUN git clone https://github.com/DUNE-DAQ/elisa_client_api.git && python3 -m pip install --upgrade setuptools && python3 -m pip install ./elisa_client_api + + + +# protoc-24.3-linux-x86_64.zip is the latest zipfile available as of Sep-15-2023 +# See also https://grpc.io/docs/protoc-installation/#install-pre-compiled-binaries-any-os + +RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip && \ + unzip protoc-24.3-linux-x86_64.zip && \ + curl -O https://raw.githubusercontent.com/DUNE-DAQ/ers/$ERSVERSION/schema/ers/issue.proto && \ + mkdir -p $LOCALPYDIR/ers && \ + protoc --python_out=$LOCALPYDIR/ers issue.proto + +RUN mkdir -p $LOCALPYDIR/erskafka && \ + curl https://raw.githubusercontent.com/DUNE-DAQ/erskafka/$ERSKAFKAVERSION/python/erskafka/ERSSubscriber.py -o $LOCALPYDIR/erskafka/ERSSubscriber.py + +ENV PYTHONPATH=$LOCALPYDIR:$PYTHONPATH + +# This ensures the container will run as non-root by default. Hat tip Pat Riehecky. +# [Commented out so various entrypoint.sh scripts as of Sep-12-2023 continue to work, to be addressed later] +# USER 60000:0 + diff --git a/dockerfiles/requirements.txt b/dockerfiles/requirements.txt new file mode 100644 index 0000000..c119429 --- /dev/null +++ b/dockerfiles/requirements.txt @@ -0,0 +1,32 @@ +aniso8601==9.0.1 +apispec==5.1.1 +click==8.1.2 +cx-Oracle==8.2.1 +dataclasses==0.6 +Flask==2.1.1 +Flask-Caching==2.0.2 +Flask-HTTPAuth==4.6.0 +flask-redis==0.4.0 +Flask-RESTful==0.3.9 +gevent==22.10.2 +greenlet==2.0.2 +gunicorn==20.1.0 +importlib-metadata==4.11.3 +influxdb==5.3.1 +itsdangerous==2.0.1 +Jinja2==3.1.1 +kafka-python==2.0.2 +MarkupSafe==2.0.1 +psycopg2-binary==2.9.7 +pymongo==4.0.2 +pytz==2022.1 +redis==3.5.3 +setuptools==39.2.0 +six==1.16.0 +typing_extensions==4.1.1 +Werkzeug==2.1.1 +zipp==3.8.0 +zope.event==4.6 +zope.interface==5.5.2 +protobuf==4.24.3 + diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..bf09fd8 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,14 @@ +# microservices + +To run one of the provided microservices in this repo, the basic command is the following: +``` +docker run --rm -e MICROSERVICE= ghcr.io/dune-daq/microservices:39be +``` + +There are a couple of points to note: +* The value of MICROSERVICE should be the name of a given microservice's subdirectory in this repo. As of Oct-6-2023, the available subdirectories are: `config-service`, `ers-dbwriter`, `logbook`, `opmon-dbwriter`, `runnumber-rest` and `runregistry-rest`. +* Most microservices require additional environment variables to be set, which can be passed using the usual docker syntax: `-e VARIABLE_NAME=` +* If you don't know what these additional environment variables are, you can just run the `docker` command as above without setting them; the container will exit out almost immediately but only after telling you what variables are missing +* The `39be` tag for the image in the example above just refers to the first four characters of the git commit of the microservices repo whose `dockerfiles/Dockerfile.microservices` Docker file was used to create the image + +For details on a given microservice, look at its own README file (format is `docs/README_.md`). They may or may not be up to date, however. diff --git a/config-service/README.md b/docs/README_config-service.md similarity index 100% rename from config-service/README.md rename to docs/README_config-service.md diff --git a/ers-dbwriter/README.md b/docs/README_ers-dbwriter.md similarity index 100% rename from ers-dbwriter/README.md rename to docs/README_ers-dbwriter.md diff --git a/logbook/README.md b/docs/README_logbook.md similarity index 100% rename from logbook/README.md rename to docs/README_logbook.md diff --git a/runnumber-rest/README.md b/docs/README_runnumber-rest.md similarity index 100% rename from runnumber-rest/README.md rename to docs/README_runnumber-rest.md diff --git a/runregistry-rest/README.md b/docs/README_runregistry-rest.md similarity index 100% rename from runregistry-rest/README.md rename to docs/README_runregistry-rest.md diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..9625826 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/bash + + +cd $(dirname $0) +source ./entrypoint_functions.sh + +ensure_required_variables "MICROSERVICE" + +microservice_dir=$(dirname $0)/$MICROSERVICE + +if [[ ! -e ${microservice_dir}/entrypoint.sh ]]; then + echo "This script sees the MICROSERVICE environment variable set to \"$MICROSERVICE\" but is unable to find the corresponding entrypoint script \"${microservice_dir}/entrypoint.sh\"" >&2 + exit 2 +fi + +cd $microservice_dir + +./entrypoint.sh + +retval=$? +echo "Return value of call to ${microservice_dir}/entrypoint.sh is $retval" + +exit $retval diff --git a/entrypoint_functions.sh b/entrypoint_functions.sh new file mode 100644 index 0000000..de7cef3 --- /dev/null +++ b/entrypoint_functions.sh @@ -0,0 +1,24 @@ + +function ensure_required_variables() { + + vars_as_string=$1 + + IFS=' ' read -ra vars <<< "$vars_as_string" + + missing_variable=false + + for var in "${vars[@]}"; do + + if [[ -v $var ]]; then + echo "$var is defined as \"${!var}\"." + else + echo "$var needs to be defined as an environment variable." + missing_variable=true + fi + done + + if $missing_variable ; then + echo "One or more required environment variables is undefined; exiting..." >&2 + exit 3 + fi +} diff --git a/ers-dbwriter/Dockerfile b/ers-dbwriter/Dockerfile deleted file mode 100644 index 0c40e7a..0000000 --- a/ers-dbwriter/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM dunedaq/c8-minimal - -RUN yum -y install python3-pip python3-devel libpq-devel gcc \ - && yum clean all - -COPY requirements.txt / -COPY dbwriter.py / - -RUN pip3 install -r requirements.txt - -ENTRYPOINT ["python3", "dbwriter.py"] diff --git a/ers-dbwriter/entrypoint.sh b/ers-dbwriter/entrypoint.sh new file mode 100755 index 0000000..4d13e11 --- /dev/null +++ b/ers-dbwriter/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "ERS_DBWRITER_HOST ERS_DBWRITER_PORT ERS_DBWRITER_USER ERS_DBWRITER_PASS ERS_DBWRITER_NAME ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER" + +python3 ./dbwriter.py diff --git a/ers-dbwriter/ersdbwriter.yaml b/ers-dbwriter/ers-dbwriter-deployment.yaml similarity index 50% rename from ers-dbwriter/ersdbwriter.yaml rename to ers-dbwriter/ers-dbwriter-deployment.yaml index 6ea0014..540134c 100644 --- a/ers-dbwriter/ersdbwriter.yaml +++ b/ers-dbwriter/ers-dbwriter-deployment.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: Namespace metadata: - labels: # https://github.com/bitnami/charts/pull/17388 + labels: pod-security.kubernetes.io/audit: baseline pod-security.kubernetes.io/audit-version: latest - pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce: baseline # unified image runs as root :( pod-security.kubernetes.io/enforce-version: latest pod-security.kubernetes.io/warn: baseline pod-security.kubernetes.io/warn-version: latest @@ -14,7 +14,7 @@ metadata: # You must still deploy kafka and postgresql with their upstream manifests # and create a secret called ers-secret. # It MUST have the following keys defined: -# ERS_DBWRITER_KAFKA_HOST, ERS_DBWRITER_KAFKA_PORT, +# ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER # ERS_DBWRITER_HOST, ERS_DBWRITER_PORT # ERS_DBWRITER_NAME # ERS_DBWRITER_USER, ERS_DBWRITER_PASS @@ -24,26 +24,53 @@ metadata: labels: app.kubernetes.io/app: ers-dbwriter app.kubernetes.io/component: ers-dbwriter - name: erskafka + name: ers-kafka-dbwriter namespace: ers spec: replicas: 1 selector: matchLabels: - app: erskafka + app: erskafka-dbwriter template: metadata: labels: - app: erskafka + app: erskafka-dbwriter app.kubernetes.io/app: ers-dbwriter app.kubernetes.io/component: ers-dbwriter spec: containers: - - image: ghcr.io/dune-daq/pocket-ersdbwriter:latest - name: erskafka - envFrom: - - secretRef: - name: ers-secret + - image: ghcr.io/dune-daq/microservices:latest + name: erskafka-dbwriter + env: + - name: MICROSERVICE + value: ers-dbwriter + - name: ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER + value: dune-daq.kafka.svc.cluster.local:9092 + - name: ERS_DBWRITER_HOST + valueFrom: + secretKeyRef: + key: host + name: ers-postgresql-svcbind-custom-user + - name: ERS_DBWRITER_PORT + valueFrom: + secretKeyRef: + key: port + name: ers-postgresql-svcbind-custom-user + - name: ERS_DBWRITER_USER + valueFrom: + secretKeyRef: + key: username + name: ers-postgresql-svcbind-custom-user + - name: ERS_DBWRITER_PASS + valueFrom: + secretKeyRef: + key: password + name: ers-postgresql-svcbind-custom-user + - name: ERS_DBWRITER_NAME + valueFrom: + secretKeyRef: + key: database + name: ers-postgresql-svcbind-custom-user resources: limits: memory: 1Gi @@ -55,8 +82,6 @@ spec: drop: - ALL runAsGroup: 11000 - runAsNonRoot: true - runAsUser: 11000 seccompProfile: type: RuntimeDefault securityContext: diff --git a/ers-dbwriter/requirements.txt b/ers-dbwriter/requirements.txt deleted file mode 100644 index 88139c8..0000000 --- a/ers-dbwriter/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -kafka-python -psycopg2-binary diff --git a/logbook/Dockerfile b/logbook/Dockerfile deleted file mode 100644 index d0fc7c4..0000000 --- a/logbook/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM cern/cc7-base AS builder - -RUN yum clean all \ - && yum update -y - -#Some utilities needed to compile python -RUN yum install gcc -y \ - && yum install make -y \ - && yum install openssl-devel libffi-devel bzip2-devel -y - -RUN curl https://www.python.org/ftp/python/3.9.10/Python-3.9.10.tgz > Python-3.9.10.tgz -RUN tar -zxvf Python-3.9.10.tgz - -WORKDIR /Python-3.9.10 -RUN ./configure --enable-optimizations \ - && make altinstall \ - && /usr/local/bin/python3.9 -m pip install --upgrade pip -WORKDIR / - -RUN yum install perl-Authen-Krb5.x86_64 -y \ - && yum install perl-WWW-CERNSSO-Auth.noarch -y - -COPY *.py elisaconf.json requirements.txt / -COPY elisa_client_api /elisa_client_api -COPY cern-get-sso-cookie /usr/local/bin - -RUN pip install -r /requirements.txt \ - && pip install ./elisa_client_api \ - && mkdir -p ./logfiles - -EXPOSE 5005 -CMD ["python3.9", "logbook.py"] diff --git a/logbook/entrypoint.sh b/logbook/entrypoint.sh new file mode 100755 index 0000000..47f78b7 --- /dev/null +++ b/logbook/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "USERNAME PASSWORD HARDWARE" + +mkdir -p ./logfiles +cp ./cern-get-sso-cookie /usr/local/bin + +python3 ./logbook.py + diff --git a/logbook/logbook-deployment.yaml b/logbook/logbook-deployment.yaml new file mode 100644 index 0000000..e8bfc07 --- /dev/null +++ b/logbook/logbook-deployment.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: baseline # unified image runs as root :( + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: baseline + pod-security.kubernetes.io/warn-version: latest + name: logbook +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/app: logbook + app.kubernetes.io/component: logbook + name: logbook + namespace: logbook +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/app: logbook + app.kubernetes.io/component: logbook + template: + metadata: + labels: + app.kubernetes.io/app: logbook + app.kubernetes.io/component: logbook + spec: + containers: + - image: ghcr.io/dune-daq/microservices:latest + name: logbook + env: + - name: MICROSERVICE + value: logbook + - name: HARDWARE + value: XXXXXXXXXXXXX + - name: USERNAME + valueFrom: + secretKeyRef: + key: YYYY + name: XXXXX + - name: PASSWORD + valueFrom: + secretKeyRef: + key: YYYY + name: XXXXX + resources: + limits: + memory: 1Gi + requests: + memory: 8Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 11000 + seccompProfile: + type: RuntimeDefault + securityContext: + fsGroup: 11000 diff --git a/logbook/requirements.txt b/logbook/requirements.txt deleted file mode 100644 index 339acee..0000000 --- a/logbook/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -Flask==2.0.2 -Flask-Caching==1.10.1 -Flask-HTTPAuth==4.4.0 -flask-redis==0.4.0 -Flask-RESTful==0.3.9 diff --git a/opmon-dbwriter/Dockerfile b/opmon-dbwriter/Dockerfile deleted file mode 100644 index cc11b5a..0000000 --- a/opmon-dbwriter/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM dunedaq/c8-minimal - -RUN yum -y install python3-pip python3-devel libpq-devel gcc \ - && yum clean all - -COPY requirements.txt / -COPY kafka-to-influx.py / - -RUN pip3 install -r requirements.txt - -ENTRYPOINT ["python3", "kafka-to-influx.py"] diff --git a/opmon-dbwriter/entrypoint.sh b/opmon-dbwriter/entrypoint.sh new file mode 100755 index 0000000..63a6b62 --- /dev/null +++ b/opmon-dbwriter/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "" + +python3 ./kafka-to-influx.py diff --git a/opmon-dbwriter/requirements.txt b/opmon-dbwriter/requirements.txt deleted file mode 100644 index ef6317d..0000000 --- a/opmon-dbwriter/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -kafka-python -click -influxdb - diff --git a/runnumber-rest/Dockerfile b/runnumber-rest/Dockerfile deleted file mode 100644 index 5d32716..0000000 --- a/runnumber-rest/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM cern/cc7-base - -RUN yum clean all \ - && yum -y install python3-pip \ - && yum -y install python3-devel \ - && yum --enablerepo=cernonly -y install oracle-instantclient12.1-devel \ - && yum --enablerepo=cernonly -y install oracle-instantclient-tnsnames.ora \ - && yum clean all - -COPY authentication.py / -COPY credentials.py / -COPY rest.py / -COPY backend.py / -COPY queries.py / -COPY requirements.txt / - -RUN pip3 install --upgrade pip -RUN pip3 install -r /requirements.txt - -ENV LD_LIBRARY_PATH=/usr/lib/oracle/12.1/client64/lib/ - -COPY entrypoint.sh / -RUN ["chmod", "+x", "/entrypoint.sh"] -ENTRYPOINT ["/entrypoint.sh"] diff --git a/runnumber-rest/entrypoint.sh b/runnumber-rest/entrypoint.sh index d15ceeb..8d89e56 100755 --- a/runnumber-rest/entrypoint.sh +++ b/runnumber-rest/entrypoint.sh @@ -1,7 +1,15 @@ #!/bin/bash -sed -i "s/dburi\='Secret from Kubernetes\!'/dburi\='${RNURI}'/g" /credentials.py -sed -i "s/user\='Secret from Kubernetes\!'/user\='${RNUSER}'/g" /credentials.py -sed -i "s/password\='Secret from Kubernetes\!'/password\='${RNPASS}'/g" /credentials.py + +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "RNURI RNUSER RNPASS" + +export LD_LIBRARY_PATH=/usr/lib/oracle/12.1/client64/lib/:$LD_LIBRARY_PATH + +sed -i "s/dburi\='Secret from Kubernetes\!'/dburi\='${RNURI}'/g" credentials.py +sed -i "s/user\='Secret from Kubernetes\!'/user\='${RNUSER}'/g" credentials.py +sed -i "s/password\='Secret from Kubernetes\!'/password\='${RNPASS}'/g" credentials.py exec gunicorn -b 0.0.0.0:5000 --workers=1 --worker-class=gevent --timeout 5000000000 --log-level=debug rest:app diff --git a/runnumber-rest/requirements.txt b/runnumber-rest/requirements.txt deleted file mode 100644 index e072a75..0000000 --- a/runnumber-rest/requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -aniso8601==9.0.1 -click==8.0.1 -cx-Oracle==8.2.1 -dataclasses==0.8 -Flask==2.0.2 -Flask-HTTPAuth==4.4.0 -Flask-RESTful==0.3.9 -gevent==21.8.0 -gunicorn==20.1.0 -importlib-metadata==4.8.1 -itsdangerous==2.0.1 -Jinja2==3.0.2 -MarkupSafe==2.0.1 -pytz==2021.3 -setuptools==39.2.0 -six==1.16.0 -typing-extensions==3.10.0.2 -Werkzeug==2.0.2 -zipp==3.6.0 diff --git a/runnumber-rest/runnumber-rest-deployment.yaml b/runnumber-rest/runnumber-rest-deployment.yaml new file mode 100644 index 0000000..b2b55a0 --- /dev/null +++ b/runnumber-rest/runnumber-rest-deployment.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: baseline # unified image runs as root :( + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: baseline + pod-security.kubernetes.io/warn-version: latest + name: runservices +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/app: runnumber-rest + app.kubernetes.io/component: runnumber-rest + name: runnumber-rest + namespace: {{ DUNE_runservices.namespace }} +spec: + minReadySeconds: 5 + selector: + matchLabels: + app.kubernetes.io/app: runnumber-rest + app.kubernetes.io/component: runnumber-rest + template: + metadata: + labels: + app.kubernetes.io/app: runnumber-rest + app.kubernetes.io/component: runnumber-rest + spec: + containers: + - image: ghcr.io/dune-daq/microservices:latest + name: runnumber-rest + env: + - name: MICROSERVICE + value: runnumber-rest + envFrom: + - secretRef: + name: run-db-secret + ports: + - containerPort: 5000 + protocol: TCP + resources: + limits: + memory: 1Gi + requests: + memory: 8Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + securityContext: + allowPrivilegeEscalation: false + runAsGroup: 11000 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + securityContext: + fsGroup: 11000 diff --git a/runregistry-rest/Dockerfile b/runregistry-rest/Dockerfile deleted file mode 100644 index 0fdb47e..0000000 --- a/runregistry-rest/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# this should move to github container registry at some point -# and should probably be based on one of our baseline containers -FROM docker.io/dunedaq/pocket-runregistry-rest:0.2 - -#RUN yum clean all \ -# && yum -y install python3-devel \ -# && yum -y install python3-pip \ -# && yum -y install python3-wheel \ -# && yum --enablerepo=cernonly -y install oracle-instantclient12.1-devel \ -# && yum --enablerepo=cernonly -y install oracle-instantclient-tnsnames.ora \ -# && yum clean all - -# ENV LD_LIBRARY_PATH=/usr/lib/oracle/12.1/client64/lib/ - -COPY authentication.py credentials.py rest.py requirements.txt / -COPY ./backends/. /backends/ - -RUN mkdir --mode=1777 /uploads - -RUN pip3 install --upgrade pip -RUN pip3 install -r /requirements.txt - -COPY --chmod=755 entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] diff --git a/runregistry-rest/credentials.py b/runregistry-rest/credentials.py index 4417d23..72009cf 100644 --- a/runregistry-rest/credentials.py +++ b/runregistry-rest/credentials.py @@ -8,9 +8,9 @@ username = os.environ.get("DB_USERNAME", "") password = os.environ.get("DB_PASSWORD", "") else: # is oracle? - dburi = "FIX ME to be a Secret from Kubernetes with good defaults!" + dburi = os.environ.get("DB_URI", None) port = os.environ.get("DB_PORT", 1521) - database = "set default database name here" + database = os.environ.get("DB_NAME", "runregistry") username = os.environ.get("DB_USERNAME", "") password = os.environ.get("DB_PASSWORD", "") diff --git a/runregistry-rest/entrypoint.sh b/runregistry-rest/entrypoint.sh index 5d3b27c..4972050 100755 --- a/runregistry-rest/entrypoint.sh +++ b/runregistry-rest/entrypoint.sh @@ -1,6 +1,14 @@ #!/bin/bash -echo "You should probably define env vars for:" -echo " DB_HOSTNAME, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD" + +cd $(dirname $0) +source ../entrypoint_functions.sh + +ensure_required_variables "DB_HOSTNAME DB_PORT DB_NAME DB_USERNAME DB_PASSWORD" + + +mkdir --mode=1777 /uploads + +export LD_LIBRARY_PATH=/usr/lib/oracle/12.1/client64/lib/:$LD_LIBRARY_PATH exec gunicorn -b 0.0.0.0:5005 --workers=1 --worker-class=gevent --timeout 5000000000 --log-level=debug rest:app diff --git a/runregistry-rest/requirements.txt b/runregistry-rest/requirements.txt deleted file mode 100644 index 1d64435..0000000 --- a/runregistry-rest/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -aniso8601==9.0.1 -click==8.0.1 -#cx-Oracle==8.2.1 -#dataclasses==0.8 -Flask==2.0.2 -Flask-Caching==1.10.1 -Flask-HTTPAuth==4.4.0 -flask-redis==0.4.0 -Flask-RESTful==0.3.9 -gevent==21.8.0 -gunicorn==20.1.0 -importlib-metadata==4.8.1 -itsdangerous==2.0.1 -Jinja2==3.0.2 -MarkupSafe==2.0.1 -pytz==2021.3 -redis==3.5.3 -setuptools==39.2.0 -six==1.16.0 -typing-extensions==3.10.0.2 -Werkzeug==2.0.2 -zipp==3.6.0 -psycopg2 -flask-redis diff --git a/runregistry-rest/runregistry-rest-deployment.yaml b/runregistry-rest/runregistry-rest-deployment.yaml new file mode 100644 index 0000000..d1e072d --- /dev/null +++ b/runregistry-rest/runregistry-rest-deployment.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: baseline # unified image runs as root :( + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: baseline + pod-security.kubernetes.io/warn-version: latest + name: runservices +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/app: runregistry-rest + app.kubernetes.io/component: runregistry-rest + name: runregistry-rest + namespace: runservices +spec: + selector: + matchLabels: + app.kubernetes.io/app: runregistry-rest + app.kubernetes.io/component: runregistry-rest + minReadySeconds: 5 + template: + metadata: + labels: + app.kubernetes.io/app: runregistry-rest + app.kubernetes.io/component: runregistry-rest + spec: + containers: + - image: ghcr.io/dune-daq/microservices:latest + name: runregistry-rest + env: + - name: MICROSERVICE + value: runregistry-rest + envFrom: + - secretRef: + name: run-db-secret + +# - name: RGDB +# value: postgres +# - name: DB_USERNAME +# valueFrom: +# secretKeyRef: +# key: username +# name: runservices-postgresql-svcbind-custom-user +# - name: DB_PASSWORD +# valueFrom: +# secretKeyRef: +# key: password +# name: runservices-postgresql-svcbind-custom-user +# - name: DB_HOSTNAME +# valueFrom: +# secretKeyRef: +# key: host +# name: runservices-postgresql-svcbind-custom-user +# - name: DB_PORT +# valueFrom: +# secretKeyRef: +# key: port +# name: runservices-postgresql-svcbind-custom-user +# - name: DB_NAME +# valueFrom: +# secretKeyRef: +# key: database +# name: runservices-postgresql-svcbind-custom-user + ports: + - containerPort: 5005 + name: http + protocol: TCP + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + resources: + limits: + memory: 1Gi + requests: + memory: 8Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 11000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /uploads + name: uploads-volume + volumes: # persistance is not required at this time + - name: uploads-volume + emptyDir: + sizeLimit: 20Gi + securityContext: + fsGroup: 11000 diff --git a/runregistry-rest/runregistry.yml b/runregistry-rest/runregistry.yml deleted file mode 100644 index 0cbee51..0000000 --- a/runregistry-rest/runregistry.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -# You must still deploy your database with its manifests from upstream -# and create a secret called runcounter-postgresql-svcbind-custom-user -# containing keys: host, port, dbname, username, password -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/app: runregistry-rest - app.kubernetes.io/component: runregistry-rest - name: runregistry-rest - namespace: runcounter -spec: - selector: - matchLabels: - app.kubernetes.io/app: runregistry-rest - app.kubernetes.io/component: runregistry-rest - minReadySeconds: 5 - template: - metadata: - labels: - app.kubernetes.io/app: runregistry-rest - app.kubernetes.io/component: runregistry-rest - spec: - containers: - - env: - - name: RGDB - value: postgres - - name: DB_USERNAME - valueFrom: - secretKeyRef: - key: username - name: runcounter-postgresql-svcbind-custom-user - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: runcounter-postgresql-svcbind-custom-user - - name: DB_HOSTNAME - valueFrom: - secretKeyRef: - key: host - name: runcounter-postgresql-svcbind-custom-user - - name: DB_PORT - valueFrom: - secretKeyRef: - key: port - name: runcounter-postgresql-svcbind-custom-user - - name: DB_NAME - valueFrom: - secretKeyRef: - key: database - name: runcounter-postgresql-svcbind-custom-user - image: ghcr.io/dune-daq/pocket-runregistry-rest:v0.0.8 - name: runregistry-rest - ports: - - containerPort: 5005 - name: http - protocol: TCP - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - resources: - limits: - memory: 1Gi - requests: - memory: 8Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 11000 - runAsNonRoot: true - runAsUser: 11000 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /uploads - name: uploads-volume - volumes: # persistance is not required at this time - - name: uploads-volume - emptyDir: - sizeLimit: 20Gi - securityContext: - fsGroup: 11000 ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/app: runregistry-rest - app.kubernetes.io/component: runregistry-rest - name: runregistry-rest - namespace: runcounter -spec: - ports: - - name: http - port: 5005 - protocol: TCP - targetPort: 5005 - selector: - app.kubernetes.io/app: runregistry-rest - app.kubernetes.io/component: runregistry-rest - type: ClusterIP