Merge pull request #43 from bcgsc/feature/dev-yml

Feature/dev yml

docker-compose.http.yml → demo/docker-compose.dev.yml

@@ -20,7 +20,7 @@ services:
         target: /orientdb/backup
         type: bind
   graphkb_api:
-    image: bcgsc/pori-graphkb-api:v3.13.3
+    image: bcgsc/pori-graphkb-api:latest
     ports:
       - 8080:8080
     environment:
@@ -44,15 +44,14 @@ services:
         type: bind
         read_only: true
   graphkb_client:
-    image: bcgsc/pori-graphkb-client:v4.2.2
+    image: bcgsc/pori-graphkb-client:latest
     environment:
       KEYCLOAK_REALM: PORI
       KEYCLOAK_CLIENT_ID: GraphKB
       # customize settings below
       API_BASE_URL: http://graphkb_api:8080
       KEYCLOAK_URL: http://localhost:8888/auth
       PUBLIC_PATH: /
-      IS_DEMO: 1 # Remove when non-demo deployment
     ports:
       - 5000:80
     depends_on:
@@ -61,7 +60,7 @@ services:
       - app-network
     restart: always
   ipr_db:
-    image: bcgsc/pori-ipr-demodb:v6.2.3
+    image: bcgsc/pori-ipr-demodb:latest
     restart: always
     environment:
       DATABASE_NAME: ipr_demo
@@ -92,7 +91,7 @@ services:
       - app-network
     restart: always
   ipr_api:
-    image: bcgsc/pori-ipr-api:v6.2.3
+    image: bcgsc/pori-ipr-api:latest
     ports:
       - 8081:8080
     environment:
@@ -120,15 +119,14 @@ services:
         type: bind
         read_only: true
   ipr_client:
-    image: bcgsc/pori-ipr-client:v6.2.1
+    image: bcgsc/pori-ipr-client:latest
     environment:
       KEYCLOAK_REALM: PORI
       # customize settings below
       API_BASE_URL: http://ipr_api:8081/api
       GRAPHKB_URL: http://graphkb_client:5000
       KEYCLOAK_URL: http://localhost:8888/auth
       PUBLIC_PATH: /
-      IS_DEMO: 1 # Remove when non-demo deployment
     ports:
       - 3000:80
     depends_on:

demo/kc_setup_default_users.sh

@@ -0,0 +1,44 @@
+
+export KEYCLOAK_ADMIN_USER=admin
+export KEYCLOAK_ADMIN_PASS=admin
+export KEYCLOAK_REALM=PORI
+export KEYCLOAK_URL=http://localhost:8888/auth
+export DEFAULT_PASSWORD=secret
+
+KEYFILE=keys/keycloak.key
+
+# Get the Admin user token
+auth_resp=$(curl -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "username=${KEYCLOAK_ADMIN_USER}" \
+    -d "password=${KEYCLOAK_ADMIN_PASS}" \
+    -d 'grant_type=password' \
+    -d 'client_id=admin-cli')
+
+token=$( echo $auth_resp | grep -o '"access_token":[^,][^,]*' | sed 's/^"access_token":\s*"//' | sed 's/"$//' )
+
+if [ "$token" = "" ];
+then
+    echo "FAILED to get authorization token"
+    exit 1
+fi
+
+# Add the default users with some generated passwords
+
+BODY_TEMPLATE='{"username": "<USERNAME>", "enabled": true, "credentials": [{"type": "password", "value": "<PASSWORD>", "temporary": false}], "realmRoles": ["IPR", "GraphKB"]}'
+
+for username in graphkb_admin graphkb_importer ipr_graphkb_link iprdemo colab_demo
+do
+    echo "Adding user: $username"
+    echo "setting user (${username}) with password (${DEFAULT_PASSWORD})"
+    body=${BODY_TEMPLATE/<USERNAME>/$username}
+    body=${body/<PASSWORD>/$DEFAULT_PASSWORD}
+
+    curl -X POST "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/users" \
+        -H 'Content-Type: application/json' \
+        -H "Accept: application/json" \
+        -H "Authorization: Bearer $token" \
+        -d "$body"
+done
+
+echo ""

demo/kc_setup_keyfile.sh

@@ -0,0 +1,43 @@
+
+export KEYCLOAK_ADMIN_USER=admin
+export KEYCLOAK_ADMIN_PASS=admin
+export KEYCLOAK_REALM=PORI
+export KEYCLOAK_URL=http://localhost:8888/auth
+export DEFAULT_PASSWORD=secret
+
+KEYFILE=keys/keycloak.key
+
+# Get the Admin user token
+auth_resp=$(curl -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "username=${KEYCLOAK_ADMIN_USER}" \
+    -d "password=${KEYCLOAK_ADMIN_PASS}" \
+    -d 'grant_type=password' \
+    -d 'client_id=admin-cli')
+
+token=$( echo $auth_resp | grep -o '"access_token":[^,][^,]*' | sed 's/^"access_token":\s*"//' | sed 's/"$//' )
+
+if [ "$token" = "" ];
+then
+    echo "FAILED to get authorization token"
+    exit 1
+fi
+
+# Now fetch the public key file
+resp=$(curl -X GET "${KEYCLOAK_URL}/admin/realms/${KEYCLOAK_REALM}/keys" \
+        -H 'Content-Type: application/json' \
+        -H "Accept: application/json" \
+        -H "Authorization: Bearer $token" )
+# echo $resp
+
+key=$( echo $resp | grep -o '"publicKey":[^,][^,]*' | sed 's/^"publicKey":\s*"//' | sed 's/"$//' )
+
+if [ ! -d "keys" ];
+then
+    mkdir keys
+fi
+
+echo "writing: $KEYFILE"
+echo "-----BEGIN PUBLIC KEY-----" > $KEYFILE
+echo "$key" >> $KEYFILE
+echo "-----END PUBLIC KEY-----" >> $KEYFILE

docs/developer_reference/getting_started.md

@@ -0,0 +1,57 @@
+# Getting Started
+
+For working on most of the PORI-related projects you will need to have a number of the components set up. For example, to work on the GraphKB API you will need both an OrientDB server and a Keycloak server already running.
+
+If your institution regularly works on PORI related projects then we reccommend setting up a development instance of the PORI platform which your developers can point their applications to. If you do not have access to something like this, then the easiest way to get the dependencies for whatever part of the PORI platform you are working on up and running is by running the development version of the docker compose configuration found in this repository: [docker-compose.dev.yml](https://github.com/bcgsc/pori/blob/master/docker-compose.dev.yml).
+
+```yaml title="docker-compose.dev.yml"
+--8<-- "./docker-compose.dev.yml"
+```
+
+## Start the Authentication Server
+
+First, set up a keycloak instance for development (like the regular set up but you can ignore the https certificates). Since we are not exposing this outside our network and are using it for development and testing only we pass admin/admin as the admin user credentials. You should pick something more secure for non-development or public installations.
+
+```bash
+docker run \
+    -e KEYCLOAK_USER=admin \
+    -e KEYCLOAK_PASSWORD=admin \
+    -p 8443:8334 \
+    -p 8888:8080 \
+    -d \
+    bcgsc/pori-auth:latest
+```
+
+You should now be able to view the browser-based administrative console by visiting [http://localhost:8888](http://localhost:8888) in your browser.
+
+### Download the Public Key File
+
+After the container is started you can go to the admin console GUI to add a users and download the realm's public key file. This must be done prior to starting the other containers.
+
+You can do this via the GUI as described in the main [install instructions](../install.md) or via a script using the keycloak REST API.
+
+```bash title="kc_setup_keyfile.sh"
+--8<-- "./kc_setup_keyfile.sh"
+```
+
+### Create the Default Users
+
+Next, create the users as specified in the main [install instructions](https://bcgsc.github.io/pori/install) or via the script below.
+
+```bash title="kc_setup_default_users.sh"
+--8<-- "./kc_setup_default_users.sh"
+```
+
+## Run docker-compose
+
+Once keycloak is set up you will need to create some directories for storing database data (so that is persists when you stop and restart your docker containers)
+
+```bash
+mkdir -p databases/{postgres,orientdb}/{backup,data}
+```
+
+Now you are ready to start up with the dev compose yml
+
+```bash
+docker-compose -f demo/docker-compose.dev.yml up -d
+```

docs/install.md

@@ -1,7 +1,7 @@
 
 # Install with Docker
 
-Since PORI is a production-ready, institution-level, scaleable platform, the simplest way to get the entire platform up and running from scratch is using [docker](https://www.docker.com/). Most of the servers are auto-started together with docker-compose but the keycloak container must be started and configured on its own first. The instructions below set up the platform with HTTPS and then use a reverse proxy to pick up the ports. This way you can omit the proxy step and run the platform with http when initially setting up and testing.
+Since PORI is a production-ready, institution-level, scaleable platform, the simplest way to get the entire platform up and running from scratch is using [docker](https://www.docker.com/). Most of the servers are auto-started together with docker-compose but the keycloak container must be started and configured on its own first. The instructions below set up the platform with HTTPS and then use a reverse proxy to pick up the ports. This way you can omit the proxy step and run the platform with http when initially setting up and testing. If you are a developer you may wish to look at the [getting started section](./developer_reference/getting_started.md) in the developers guide instead.
 
 Start by cloning this repository which contains the default docker compose config (docker-compose.yml)
 
@@ -58,6 +58,8 @@ CONTAINER ID   IMAGE                  COMMAND                  CREATED
 16ff3826c976   bcgsc/pori-auth:latest   "/opt/jboss/tools/do…"   About a minute ago   Up About a minute   8443/tcp, 0.0.0.0:8888->8080/tcp   strange_chaum
 ```
 
+### Download the Public Key File
+
 After the container is started you can go to the admin console GUI to add a users and download the realm's public key file. This must be done prior to starting the other containers.
 
 The public key file will need to be passed to the GraphKB API container at run time. Copy it from the Realms > Keys page which should look something like below
@@ -74,6 +76,10 @@ MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoteEI/Iu923I4Zqt8prxIx3ljGEecnrI+sWj
 
 Both the IPR and GraphKB API containers will use this ./keys folder, binding it into the container at run time (See volumes section of docker compose file).
 
+Note that for your convenience this can also be done with the following bash command
+
+### Adding Default Demo Users
+
 You will also want to add a couple of users to make things simpler to test. If you use the non-default demo passwords (RECCOMMENDED!) you will need to change the corresponding fields in the docker compose file. The names of these users can also be changed but it will require also adding them to the application databases.
 
 | Name             | Default in DB | Purpose                                                                                                                                              |

mkdocs.yml

@@ -14,6 +14,9 @@ markdown_extensions:
   - admonition
   - pymdownx.inlinehilite
   - pymdownx.extra
+  - pymdownx.snippets:
+      check_paths: True
+      base_path: demo
   - pymdownx.emoji:
       emoji_index: !!python/name:materialx.emoji.twemoji
       emoji_generator: !!python/name:materialx.emoji.to_svg

requirements.txt

@@ -10,11 +10,12 @@ click==7.1.2
 defusedxml==0.7.1
 entrypoints==0.3
 future==0.18.2
+ghp-import==2.0.2
 gitdb==4.0.7
 GitPython==3.1.17
 graphkb==1.5.4
 idna==2.10
-importlib-metadata==1.7.0
+importlib-metadata==4.10.1
 ipython-genutils==0.2.0
 Jinja2==2.11.3
 joblib==1.0.1
@@ -31,12 +32,13 @@ markdown-it-py==1.1.0
 markdown-refdocs==1.4.0
 MarkupSafe==1.1.1
 mdit-py-plugins==0.2.8
+mergedeep==1.3.4
 mistune==0.8.4
-mkdocs==1.1.2
+mkdocs==1.2.3
 mkdocs-awesome-pages-plugin==2.5.0
 mkdocs-exclude==1.0.2
 mkdocs-include-markdown-plugin==3.0.1
-mkdocs-material==7.1.3
+mkdocs-material==8.1.8
 mkdocs-material-extensions==1.0.1
 mkdocs-redirects==1.0.3
 mknotebooks==0.7.0
@@ -50,12 +52,16 @@ nltk==3.6.2
 packaging==20.9
 pandocfilters==1.4.3
 pathspec==0.8.1
-Pygments==2.8.1
-pymdown-extensions==8.1.1
+progressbar2==3.53.1
+Pygments==2.11.2
+pymdown-extensions==9.1
 pyparsing==2.4.7
 pyrsistent==0.17.3
 python-dateutil==2.8.1
+python-utils==2.5.6
+pytz==2021.1
 PyYAML==5.4.1
+pyyaml-env-tag==0.1
 pyzmq==22.1.0
 regex==2021.4.4
 requests==2.25.1
@@ -69,6 +75,7 @@ traitlets==5.0.5
 typed-ast==1.4.3
 typing-extensions==3.7.4.3
 urllib3==1.26.5
+watchdog==2.1.6
 wcmatch==8.2
 webencodings==0.5.1
 zipp==3.4.1