Release/1.1.20231129

.dockerignore

@@ -1,3 +1,4 @@
 scripts
 venv
 data
+**.DS_Store

.env

@@ -10,3 +10,13 @@ KEYCLOAK_ADMIN_PASSWORD=password
 KEYCLOAK_CLIENT_SECRET="QJiOGn09eCEfnqAmcPP2l4vMU8grlmVQ"
 REDIRECT_URIS=http://${HOSTNAME}/docs/oauth2-redirect
 POST_LOGOUT_REDIRECT_URIS=http://${HOSTNAME}/aiod-auth/realms/aiod/protocol/openid-connect/logout
+
+#ELASTICSEARCH
+ES_USER=elastic
+ES_PASSWORD=changeme
+ES_DISCOVERY_TYPE=single-node
+ES_ROLE="edit_aiod_resources"
+ES_JAVA_OPTS="-Xmx256m -Xms256m"
+
+#LOGSTASH
+LS_JAVA_OPTS="-Xmx256m -Xms256m"

.github/workflows/dockerized-tests.yml

@@ -23,11 +23,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Build the docker image
-        run: docker build --tag ai4eu_server_demo:latest -f Dockerfile .
+        run: docker build --tag aiod_metadata_catalogue:latest -f Dockerfile .
 
       - name: Run docker container and pytest tests
         run: |
-          docker run -e KEYCLOAK_CLIENT_SECRET="mocked_secret" --entrypoint "" ai4eu_server_demo sh -c "pip install \".[dev]\" && pytest tests -s"
+          docker run -e KEYCLOAK_CLIENT_SECRET="mocked_secret" --entrypoint "" aiod_metadata_catalogue sh -c "pip install \".[dev]\" && pytest tests -s"
 
 
 

.gitignore

@@ -1,7 +1,18 @@
 # Project Specific
+
 # data/ is intended for database data from the mysql container
 data/
 
+# Generated Logstash configuration
+logstash/config/config/logstash.yml
+logstash/config/config/pipelines.yml
+logstash/config/pipeline
+logstash/config/sql
+
+
+
+
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -113,6 +124,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+**.DS_Store
 
 # Spyder project settings
 .spyderproject
@@ -135,4 +147,4 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-.vscode
+.vscode

.pre-commit-config.yaml

@@ -11,9 +11,10 @@ repos:
         name: mypy src
         files: src/.*
         additional_dependencies:
-          - types-requests
           - types-python-dateutil
           - types-pytz
+          - types-requests
+          - types-setuptools
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 OpenML Labs
+Copyright (c) 2023 AI On Demand
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -58,6 +58,11 @@ For development:
 - Additional 'mysqlclient' dependencies. Please have a look at [their installation instructions]
   (https://github.com/PyMySQL/mysqlclient#install).
 
+## Production environment
+
+For production environments elasticsearch recommends -Xss4G and -Xmx8G for the JVM settings.\
+This parameters can be defined in the .env file.
+See the [elasticsearch guide](https://www.elastic.co/guide/en/logstash/current/jvm-settings.html).
 
 ## Installation
 
@@ -67,40 +72,22 @@ Information on how to install Docker is found in [their documentation](https://d
 
 ### Using docker compose
 ```bash
-docker compose up
+docker compose --profile examples up
 ```
+
 starts the MYSQL Server, the REST API, Keycloak for Identy and access management and Nginx for reverse proxing. \
 Once started, you should be able to visit the REST API server at: http://localhost and Keycloak at http://localhost/aiod-auth \
 To authenticate to the REST API swagger interface the predefined user is: user, and password: password \
 To authenticate as admin to Keycloak the predefined user is: admin and password: password \
 To use a different DNS hostname replace localhost with it in .env and src/config.toml \
 This configuration is intended for development, DO NOT use it in production. 
 
-Use the following instructions if you prefer to use Docker instead of Docker Compose.
-
-### Starting a MySQL Server
-
-We use the default [MySQL Docker image](https://hub.docker.com/_/mysql).
-By default, the database is stored within the docker container and will thus be deleted when the container is removed.
-Instructions on using a persistent storage can be found at the end of this section.
-
-First, we define a docker network to allow our server to be reachable from other docker containers:
-
-```bash
-docker network create sql-network
-```
-
-Then, start the MySQL Server:
-
+To turn if off again, use 
 ```bash
-docker run -e MYSQL_ROOT_PASSWORD=ok --name sqlserver --network sql-network -d mysql
+docker compose --profile examples down
 ```
 
-That's all! You should be able to connect to the server now, though no database is present yet:
-
-```bash
-docker run -it --network sql-network --rm mysql mysql -hsqlserver -uroot -pok
-```
+To connect to the database use `./scripts/database-connect.sql`.
 
 ```bash
 mysql> SHOW DATABASES;
@@ -115,48 +102,18 @@ mysql> SHOW DATABASES;
 4 rows in set (0.03 sec)
 ```
 
-For ease of use, you can also make use of the script in `scripts/run_mysql_server.sh`.
-
-#### Persistent Storage
-
-The data is persistent when simply stopping and restarting the server:
-
-```bash
-docker stop sqlserver
-docker start sqlserver
-```
-
-However, all data is lost when the container is deleted.
-To ensure data can persist even if the container is deleted, allow the Docker container to write to a directory on the
-host machine.
-To do that, mount a volume by adding `-v /ABSOLUTE/PATH/TO/HOST/DIR:/var/lib/mysql` to the docker command that starts
-the server
-(it's also possible to create a docker
-volume ([docs](https://docs.docker.com/engine/reference/commandline/run/#mount-volume--v---read-only))).
-If you want to use a path within this repository directory, we recommend naming the directory `data` since then it will
-automatically be ignored by git.
-For more information, see "Where to Store Data" in the linked documentation.
-
-### Starting the REST API
-
-The repository provides a Dockerfile to run the REST API in a containerized environment.
+Now, you can visit the server from your browser at `localhost:8000/docs`.
 
-#### Using the Docker container
-
-First, build the docker image from the dockerfile:
+#### Using connectors
+You can specify different connectors using
 
 ```bash
-docker build --tag ai4eu_server_demo:latest -f Dockerfile .
+docker compose --profile examples --profile huggingface-datasets --profile openml-datasets up -d
+docker compose --profile examples --profile huggingface-datasets --profile openml-datasets down
 ```
 
-then create a container from that image, remember to forward the port and connect to the right 
-docker network (alternatively you can use the script at `scripts/run_apiserver.sh`)
-
-```bash
-docker run --network sql-network -it --rm -p 8000:8000 --name apiserver ai4eu_server_demo
-```
-
-At this point you should be able to visit the server from your browser at `localhost:8000/docs`.
+Make sure you use the same profile for `up` and `down`, otherwise some containers might keep 
+running.
 
 #### Local Installation
 
@@ -199,7 +156,7 @@ The `--reload` argument will automatically restart the app if changes are made t
 3. Run using DevContainer (see next subsection)
 
 ### Authentication
-Currently, the code is on default coupled with a keycloak running on test.openml.org. To make 
+Currently, the code is by default running using the local Keycloak. To make 
 this work, you need to set an environment variable. You can do this by setting the 
 `KEYCLOAK_CLIENT_SECRET` in `src/.env`.
 
@@ -208,9 +165,6 @@ this work, you need to set an environment variable. You can do this by setting t
 KEYCLOAK_CLIENT_SECRET=[SECRET]
 ```
 
-Please ask Jos van der Velde ([email protected]) for the keycloak secret, and to give your 
-user the correct roles.
-
 Alternatively, you can connect to a different keycloak instance by modifying `src/.env`. EGI 
 Checkin can for instance be used on a deployed instance - not on local host. Marco Rorro is the 
 go-to person to request the usage of the EGI Checkin.
@@ -255,11 +209,6 @@ You can change this behavior through parameters of the script:
 Following the installation instructions above, the server may be reached at `127.0.0.1:8000`.
 REST API documentation is automatically built and can be viewed at `127.0.0.1:8000/docs`.
 
-#### Editing Files Locally
-
-While developing the server it is often convenient to edit the files on the host machine.
-To avoid rebuilding the docker container each time you edit the files, you can mount the host files into the container.
-Mount your repository's `src` directory into the container's `/app` directory by adding `-v $(pwd)/src:/app`.
 
 #### Automatically Restart on Change
 

connectors/fill-examples.sh

@@ -4,6 +4,10 @@ python3 connectors/synchronization.py \
   -c connectors.example.example.ExampleCaseStudyConnector \
   -w /opt/connectors/data/example/case_study
 
+python3 connectors/synchronization.py \
+  -c connectors.example.example.ExampleContactConnector \
+  -w /opt/connectors/data/example/contact
+
 python3 connectors/synchronization.py \
   -c connectors.example.example.ExampleComputationalAssetConnector \
   -w /opt/connectors/data/example/computational_asset
@@ -12,10 +16,6 @@ python3 connectors/synchronization.py \
   -c connectors.example.example.ExampleEducationalResourceConnector \
   -w /opt/connectors/data/example/educational_resource
 
-python3 connectors/synchronization.py \
-  -c connectors.example.example.ExampleEventConnector \
-  -w /opt/connectors/data/example/event
-
 python3 connectors/synchronization.py \
   -c connectors.example.example.ExampleExperimentConnector \
   -w /opt/connectors/data/example/experiment
@@ -36,6 +36,10 @@ python3 connectors/synchronization.py \
   -c connectors.example.example.ExamplePersonConnector \
   -w /opt/connectors/data/example/person
 
+python3 connectors/synchronization.py \
+  -c connectors.example.example.ExampleEventConnector \
+  -w /opt/connectors/data/example/event
+
 python3 connectors/synchronization.py \
   -c connectors.example.example.ExampleProjectConnector \
   -w /opt/connectors/data/example/project
@@ -82,6 +86,10 @@ python3 connectors/synchronization.py \
   -c connectors.example.enum.EnumConnectorOrganisationType \
   -w /opt/connectors/data/enum/organisation_type
 
+python3 connectors/synchronization.py \
+  -c connectors.example.enum.EnumConnectorNewsCategory \
+  -w /opt/connectors/data/enum/news_category
+
 python3 connectors/synchronization.py \
   -c connectors.example.enum.EnumConnectorStatus \
-  -w /opt/connectors/data/enum/status
+  -w /opt/connectors/data/enum/status

connectors/huggingface/datasets.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 WORK_DIR=/opt/connectors/data/huggingface/dataset
+mkdir -p $WORK_DIR
 
 python3 connectors/synchronization.py \
   -c connectors.huggingface.huggingface_dataset_connector.HuggingFaceDatasetConnector \

connectors/openml/Dockerfile

@@ -1,4 +1,4 @@
-FROM ai4eu_server
+FROM aiod_metadata_catalogue
 
 COPY cron /etc/cron.d/aiod
 COPY datasets.sh /opt/connectors/script/datasets.sh

connectors/openml/datasets.sh

@@ -15,6 +15,6 @@ echo $(date -u) "Starting synchronization..."
 PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
       -c $CONNECTOR \
       -w $WORK_DIR \
-      --from-identifier 4500 \
-      --save-every 100 > ${WORK_DIR}/connector.log 2>&1
+      --from-identifier 1 \
+      --save-every 100 >> ${WORK_DIR}/connector.log 2>&1
 echo $(date -u) "Synchronization Done."

connectors/zenodo/Dockerfile

@@ -1,4 +1,4 @@
-FROM ai4eu_server
+FROM aiod_metadata_catalogue
 
 COPY cron /etc/cron.d/aiod
 COPY datasets.sh /opt/connectors/script/datasets.sh

connectors/zenodo/datasets.sh

@@ -16,5 +16,5 @@ PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
       -c $CONNECTOR \
       -w $WORK_DIR \
       --from-date "2023-08-01" \
-      --save-every 100 > ${WORK_DIR}/connector.log 2>&1
+      --save-every 100 >> ${WORK_DIR}/connector.log 2>&1
 echo $(date -u) "Synchronization Done."

deletion/Dockerfile

@@ -0,0 +1,12 @@
+FROM aiod_metadata_catalogue
+
+COPY cron /etc/cron.d/aiod
+COPY delete_old.sh /opt/deletion/script/delete_old.sh
+COPY entry.sh /opt/deletion/script/entry.sh
+
+USER root
+RUN apt -y install cron
+RUN chmod +x /etc/cron.d/aiod /opt/deletion/script/delete_old.sh
+RUN crontab /etc/cron.d/aiod
+
+WORKDIR /app

deletion/cron

@@ -0,0 +1 @@
+40 * * * * bash /opt/deletion/script/delete_old.sh >> /opt/deletion/data/deletion-cron.log 2>&1

deletion/delete_old.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+WORK_DIR=/opt/deletion/data/
+
+another_instance()
+{
+    echo $(date -u) "This script is already running in a different thread."
+    exit 1
+}
+exec 9< "$0"
+flock -n -x 9 || another_instance
+
+echo $(date -u) "Starting deletion..."
+PYTHONPATH=/app /usr/local/bin/python3 /app/database/deletion/hard_delete.py \
+      --time-threshold-minutes 10080 > ${WORK_DIR}/deletion.log 2>&1
+echo $(date -u) "Deletion Done."

deletion/entry.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# If this directory does not exist, the cron job cannot log (and cannot run)
+mkdir -p /opt/deletion/data/
+
+# Run cron on the foreground with log level WARN
+/usr/sbin/cron -f -l 4