-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/johnfreeman/daq-deliverables_iss…
…ue105_standard_image' into develop Conflicts: ers-dbwriter/dbwriter.py
- Loading branch information
Showing
38 changed files
with
469 additions
and
328 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
*~ |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
#!/bin/bash | ||
|
||
echo "You should probably define env vars for:" | ||
echo " MONGO_HOST, MONGO_PORT, MONGO_USER, MONGO_PASS, MONGO_DBNAME" | ||
cd $(dirname $0) | ||
source ../entrypoint_functions.sh | ||
|
||
ensure_required_variables "MONGO_HOST MONGO_PORT MONGO_USER MONGO_PASS MONGO_DBNAME" | ||
|
||
exec gunicorn -b 0.0.0.0:5003 --workers=1 --worker-class=gevent --timeout 5000000000 --log-level=debug conf-service:app |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
FROM ghcr.io/dune-daq/microservices_dependencies:latest | ||
|
||
RUN git clone https://github.com/DUNE-DAQ/microservices -b develop | ||
|
||
ENTRYPOINT ["/microservices/entrypoint.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
FROM cern/alma9-base | ||
|
||
ARG ERSVERSION=dunedaq-v4.1.1 # For issue.proto from ers | ||
ARG ERSKAFKAVERSION=dunedaq-v4.1.1 # For ERSSubscriber.py from erskafka | ||
ARG LOCALPYDIR=/microservices_python | ||
|
||
# libaio and libnsl are needed when rpm is called on the Oracle client software | ||
|
||
RUN yum clean all \ | ||
&& yum -y install gcc make git unzip libaio libnsl libpq-devel libffi-devel python3-pip python3-wheel \ | ||
&& yum clean all | ||
|
||
RUN curl -O https://download.oracle.com/otn_software/linux/instantclient/1919000/oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm \ | ||
&& rpm -iv oracle-instantclient19.19-basic-19.19.0.0.0-1.el9.x86_64.rpm | ||
|
||
COPY requirements.txt / | ||
RUN python3 -m pip install --upgrade setuptools && \ | ||
python3 -m pip install -r requirements.txt && \ | ||
python3 -m pip cache remove \* | ||
|
||
# elisa_client_api needed by the logbook microservice | ||
RUN git clone https://github.com/DUNE-DAQ/elisa_client_api.git && python3 -m pip install --upgrade setuptools && python3 -m pip install ./elisa_client_api | ||
|
||
|
||
|
||
# protoc-24.3-linux-x86_64.zip is the latest zipfile available as of Sep-15-2023 | ||
# See also https://grpc.io/docs/protoc-installation/#install-pre-compiled-binaries-any-os | ||
|
||
RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip && \ | ||
unzip protoc-24.3-linux-x86_64.zip && \ | ||
curl -O https://raw.githubusercontent.com/DUNE-DAQ/ers/$ERSVERSION/schema/ers/issue.proto && \ | ||
mkdir -p $LOCALPYDIR/ers && \ | ||
protoc --python_out=$LOCALPYDIR/ers issue.proto | ||
|
||
RUN mkdir -p $LOCALPYDIR/erskafka && \ | ||
curl https://raw.githubusercontent.com/DUNE-DAQ/erskafka/$ERSKAFKAVERSION/python/erskafka/ERSSubscriber.py -o $LOCALPYDIR/erskafka/ERSSubscriber.py | ||
|
||
ENV PYTHONPATH=$LOCALPYDIR:$PYTHONPATH | ||
|
||
# This ensures the container will run as non-root by default. Hat tip Pat Riehecky. | ||
# [Commented out so various entrypoint.sh scripts as of Sep-12-2023 continue to work, to be addressed later] | ||
# USER 60000:0 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
aniso8601==9.0.1 | ||
apispec==5.1.1 | ||
click==8.1.2 | ||
cx-Oracle==8.2.1 | ||
dataclasses==0.6 | ||
Flask==2.1.1 | ||
Flask-Caching==2.0.2 | ||
Flask-HTTPAuth==4.6.0 | ||
flask-redis==0.4.0 | ||
Flask-RESTful==0.3.9 | ||
gevent==22.10.2 | ||
greenlet==2.0.2 | ||
gunicorn==20.1.0 | ||
importlib-metadata==4.11.3 | ||
influxdb==5.3.1 | ||
itsdangerous==2.0.1 | ||
Jinja2==3.1.1 | ||
kafka-python==2.0.2 | ||
MarkupSafe==2.0.1 | ||
psycopg2-binary==2.9.7 | ||
pymongo==4.0.2 | ||
pytz==2022.1 | ||
redis==3.5.3 | ||
setuptools==39.2.0 | ||
six==1.16.0 | ||
typing_extensions==4.1.1 | ||
Werkzeug==2.1.1 | ||
zipp==3.8.0 | ||
zope.event==4.6 | ||
zope.interface==5.5.2 | ||
protobuf==4.24.3 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# microservices | ||
|
||
To run one of the provided microservices in this repo, the basic command is the following: | ||
``` | ||
docker run --rm -e MICROSERVICE=<name of microservice> ghcr.io/dune-daq/microservices:39be | ||
``` | ||
|
||
There are a couple of points to note: | ||
* The value of MICROSERVICE should be the name of a given microservice's subdirectory in this repo. As of Oct-6-2023, the available subdirectories are: `config-service`, `ers-dbwriter`, `logbook`, `opmon-dbwriter`, `runnumber-rest` and `runregistry-rest`. | ||
* Most microservices require additional environment variables to be set, which can be passed using the usual docker syntax: `-e VARIABLE_NAME=<variable value>` | ||
* If you don't know what these additional environment variables are, you can just run the `docker` command as above without setting them; the container will exit out almost immediately but only after telling you what variables are missing | ||
* The `39be` tag for the image in the example above just refers to the first four characters of the git commit of the microservices repo whose `dockerfiles/Dockerfile.microservices` Docker file was used to create the image | ||
|
||
For details on a given microservice, look at its own README file (format is `docs/README_<microservice name>.md`). They may or may not be up to date, however. |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
#!/bin/bash | ||
|
||
|
||
cd $(dirname $0) | ||
source ./entrypoint_functions.sh | ||
|
||
ensure_required_variables "MICROSERVICE" | ||
|
||
microservice_dir=$(dirname $0)/$MICROSERVICE | ||
|
||
if [[ ! -e ${microservice_dir}/entrypoint.sh ]]; then | ||
echo "This script sees the MICROSERVICE environment variable set to \"$MICROSERVICE\" but is unable to find the corresponding entrypoint script \"${microservice_dir}/entrypoint.sh\"" >&2 | ||
exit 2 | ||
fi | ||
|
||
cd $microservice_dir | ||
|
||
./entrypoint.sh | ||
|
||
retval=$? | ||
echo "Return value of call to ${microservice_dir}/entrypoint.sh is $retval" | ||
|
||
exit $retval |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
|
||
function ensure_required_variables() { | ||
|
||
vars_as_string=$1 | ||
|
||
IFS=' ' read -ra vars <<< "$vars_as_string" | ||
|
||
missing_variable=false | ||
|
||
for var in "${vars[@]}"; do | ||
|
||
if [[ -v $var ]]; then | ||
echo "$var is defined as \"${!var}\"." | ||
else | ||
echo "$var needs to be defined as an environment variable." | ||
missing_variable=true | ||
fi | ||
done | ||
|
||
if $missing_variable ; then | ||
echo "One or more required environment variables is undefined; exiting..." >&2 | ||
exit 3 | ||
fi | ||
} |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/bin/bash | ||
|
||
cd $(dirname $0) | ||
source ../entrypoint_functions.sh | ||
|
||
ensure_required_variables "ERS_DBWRITER_HOST ERS_DBWRITER_PORT ERS_DBWRITER_USER ERS_DBWRITER_PASS ERS_DBWRITER_NAME ERS_DBWRITER_KAFKA_BOOTSTRAP_SERVER" | ||
|
||
python3 ./dbwriter.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.