From 9d64ffad1c3c0ebbf0b5fd669041726abb426135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Tue, 8 Aug 2023 14:53:33 +0200 Subject: [PATCH 1/7] chore: refactor nginx.conf for CORS. feat: pass upload directly to PIMS. --- configs/nginx/etc/nginx/cors_params | 3 + configs/nginx/etc/nginx/nginx.conf.sample | 120 ++++++++++++---------- 2 files changed, 66 insertions(+), 57 deletions(-) create mode 100644 configs/nginx/etc/nginx/cors_params diff --git a/configs/nginx/etc/nginx/cors_params b/configs/nginx/etc/nginx/cors_params new file mode 100644 index 0000000..99dc9d4 --- /dev/null +++ b/configs/nginx/etc/nginx/cors_params @@ -0,0 +1,3 @@ +add_header 'Access-Control-Allow-Origin' '*' always; +add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always; +add_header 'Access-Control-Allow-Headers' 'Content-Type, Content-Range, Content-Disposition, Content-Description, date, dateFull, authorization,content-type-full' always; diff --git a/configs/nginx/etc/nginx/nginx.conf.sample b/configs/nginx/etc/nginx/nginx.conf.sample index cc5c7f0..153411d 100644 --- a/configs/nginx/etc/nginx/nginx.conf.sample +++ b/configs/nginx/etc/nginx/nginx.conf.sample @@ -14,7 +14,6 @@ # limitations under the License. # -load_module 'modules/ngx_http_upload_module.so'; worker_processes 1; daemon off; @@ -28,18 +27,34 @@ http { server_names_hash_bucket_size 128; server_tokens off; - log_format main '$scheme - $host - $remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; + # Use Docker's resolver to resolve container names when used with set and proxy_pass + resolver 127.0.0.11 valid=30s; - sendfile on; + log_format main '$scheme - h=$host - hh=$http_host - sn=$server_name - ra=$remote_addr - ru=$remote_user [$time_local] req="$request" ' + 's=$status bbs=$body_bytes_sent hr="$http_referer" ' + 'hua="$http_user_agent" hxff="$http_x_forwarded_for" hxfh="$http_x_forwarded_host"'; + sendfile on; keepalive_timeout 120; proxy_connect_timeout 75; large_client_header_buffers 4 16k; - error_log stderr debug; - access_log stdout; + error_log /var/log/nginx/error.log debug; + access_log /var/log/nginx/access.log main; + + upstream ims-pooled { + server $INTERNAL_URLS_IMS; + } + + # Default server. Useful when no server_name matched because Host is wrong but can be found in http_x_forwarded_host + # The downside is that when there's no matching server block at all for an URL that finds its way here, there's an infinite loop. + server { + listen 80 default_server; + location / { + proxy_pass http://127.0.0.1; + proxy_set_header Host $http_x_forwarded_host; + } + } server { client_max_body_size 0; @@ -47,30 +62,24 @@ http { server_name $URLS_CORE; location / { + include /etc/nginx/cors_params; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_WEB_UI; + + # Allow nginx to start without web_ui. Else it may crash on start saying "[emerg] host not found in upstream "web_ui"" + set $tmpwebuivar "$INTERNAL_URLS_WEB_UI"; + proxy_pass http://$tmpwebuivar; #proxy_http_version 1.1; #proxy_set_header Upgrade $http_upgrade; #proxy_set_header Connection "upgrade"; } - location /api { - proxy_intercept_errors off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; - } - location /monitoring { # check and logout + location ~ /(api|monitoring|login|logout|saml|saml2|server|session|custom-ui|static|restApiDoc|js/jsondoc|css/jsondoc) { proxy_intercept_errors off; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; - } - location ~ /(login|logout) { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; + set $tmpcorehost "$INTERNAL_URLS_CORE"; + proxy_pass http://$tmpcorehost; } location /ws { proxy_http_version 1.1; @@ -79,17 +88,11 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; + set $tmpcorehost "$INTERNAL_URLS_CORE"; + proxy_pass http://$tmpcorehost; } - location ~ /(server|session|custom-ui|static) { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; - } - location ~ /(restApiDoc|js/jsondoc|css/jsondoc) { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_CORE; + location /www/ { + alias /www/; } proxy_http_version 1.1; @@ -110,9 +113,17 @@ http { server_name $URLS_IMAGE_SERVER; location / { - add_header 'Access-Control-Allow-Origin' '*'; + if ($request_method = OPTIONS ) { + include /etc/nginx/cors_params; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + + include /etc/nginx/cors_params; proxy_set_header Host $host; - proxy_pass http://$INTERNAL_URLS_IMS; + set $tmpims1host ims-pooled; + proxy_pass http://$tmpims1host; } } server { @@ -121,41 +132,36 @@ http { listen 80; server_name $URLS_UPLOAD; + location / { + include /etc/nginx/cors_params; + + if ($request_method = OPTIONS ) { + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + return 444; + } + # Upload form should be submitted to this location location /upload { - proxy_read_timeout 600; - # Pass altered request body to this location - upload_pass @test; - # Store files to this directory - # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist - upload_store $UPLOAD_PATH; - - # Allow uploaded files to be read only by user - upload_store_access user:rw group:rw all:rw; - - # Set specified fields in request body - upload_set_form_field $upload_field_name.name "$upload_file_name"; - upload_set_form_field $upload_field_name.content_type "$upload_content_type"; - upload_set_form_field $upload_field_name.path "$upload_tmp_path"; - # Inform backend about hash and size of a file - upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5"; - upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size"; - upload_pass_form_field "^submit$|^description$"; + if ($request_method = OPTIONS ) { + include /etc/nginx/cors_params; + add_header 'Access-Control-Allow-Credentials' 'false' always; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } - upload_pass_args on; + proxy_read_timeout 600; add_header 'Access-Control-Allow-Credentials' 'false'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; add_header 'Access-Control-Allow-Headers' 'Content-Type, Content-Range, Content-Disposition, Content-Description, date, dateFull, authorization,content-type-full'; add_header 'Access-Control-Max-Age' '1728000'; add_header 'Access-Control-Allow-Origin' '*'; - } - # Pass altered request body to a backend - location @test { - add_header 'Access-Control-Allow-Origin' '*'; proxy_pass http://$INTERNAL_URLS_IMS; - proxy_read_timeout 600; } error_page 500 502 503 504 /50x.html; location = /50x.html { From b554ef6e14f33960852fbdc0c617d76809f86653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Thu, 17 Aug 2023 15:00:01 +0200 Subject: [PATCH 2/7] feat: better visibility for services_URI --- cytomine.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytomine.template b/cytomine.template index e51b232..eacbc95 100644 --- a/cytomine.template +++ b/cytomine.template @@ -39,7 +39,7 @@ global: MONGO: mongo:4.4.18-focal NGINX: cytomine/nginx:1.22.1-1.2.0 PIMS_CACHE: redis:7.0.8 - PIMS: cytomine/pims-ce-package:0.0.1 + PIMS: cytomine/pims-ce-package:0.0.2 POSTGIS: cytomine/postgis:15-3.3-alpine-1.2.1 RABBITMQ: rabbitmq:3.10 WEB_UI: cytomine/web_ui:2.6.1 From 1bacb8162b666edd60f18a0a6197d4afad12f28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Tue, 5 Sep 2023 15:51:44 +0200 Subject: [PATCH 3/7] feat: update pims with direct upload, latest services versions. --- cytomine.template | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cytomine.template b/cytomine.template index eacbc95..6917a2c 100644 --- a/cytomine.template +++ b/cytomine.template @@ -1,4 +1,4 @@ - +COPY --chmod=774 nginx-entrypoint.sh /nginx-entrypoint.sh # -------------------------------------------------- # # Update these to configure your instance global: @@ -36,13 +36,13 @@ global: constant: BIOFORMAT: cytomine/bioformat:v1.2.0 CORE: cytomine/core:4.4.0 - MONGO: mongo:4.4.18-focal - NGINX: cytomine/nginx:1.22.1-1.2.0 + MONGO: cytomine/mongo:0.0.1 + NGINX: cytomine/nginx:1.2.3 PIMS_CACHE: redis:7.0.8 - PIMS: cytomine/pims-ce-package:0.0.2 + PIMS: cytomine/pims-ce-package:0.0.3 POSTGIS: cytomine/postgis:15-3.3-alpine-1.2.1 RABBITMQ: rabbitmq:3.10 - WEB_UI: cytomine/web_ui:2.6.1 + WEB_UI: cytomine/web_ui:2.6.2 internal_docker_urls: # must match respective container hostnames from compose file constant: From 944552b8afec6ca9a578f5776ef3281ec0c0e1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Tue, 5 Sep 2023 15:55:36 +0200 Subject: [PATCH 4/7] fix: upload body_size --- configs/nginx/etc/nginx/nginx.conf.sample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/nginx/etc/nginx/nginx.conf.sample b/configs/nginx/etc/nginx/nginx.conf.sample index 153411d..2662c9d 100644 --- a/configs/nginx/etc/nginx/nginx.conf.sample +++ b/configs/nginx/etc/nginx/nginx.conf.sample @@ -49,6 +49,7 @@ http { # Default server. Useful when no server_name matched because Host is wrong but can be found in http_x_forwarded_host # The downside is that when there's no matching server block at all for an URL that finds its way here, there's an infinite loop. server { + client_max_body_size 0; listen 80 default_server; location / { proxy_pass http://127.0.0.1; @@ -62,7 +63,7 @@ http { server_name $URLS_CORE; location / { - include /etc/nginx/cors_params; + include /etc/nginx/cors_params; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; From 471078493891018c2365a8fbe199cd8fa76978b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Tue, 5 Sep 2023 16:02:47 +0200 Subject: [PATCH 5/7] fix: cytomine.template --- cytomine.template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cytomine.template b/cytomine.template index 6917a2c..75ceb78 100644 --- a/cytomine.template +++ b/cytomine.template @@ -1,5 +1,4 @@ -COPY --chmod=774 nginx-entrypoint.sh /nginx-entrypoint.sh -# -------------------------------------------------- # +# -------------------------------------------------- # # Update these to configure your instance global: urls: From f22a585bac8312812dd69106c955a622908e5a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gillard?= Date: Thu, 14 Sep 2023 09:55:57 +0200 Subject: [PATCH 6/7] feat: remove custom nginx config as it is already shipped in nginx container. fix: use pims with direct upload. --- configs/README.md | 75 ++++++++++ configs/nginx/etc/nginx/cors_params | 3 - configs/nginx/etc/nginx/nginx.conf.sample | 172 ---------------------- cytomine.template | 4 +- 4 files changed, 77 insertions(+), 177 deletions(-) create mode 100644 configs/README.md delete mode 100644 configs/nginx/etc/nginx/cors_params delete mode 100644 configs/nginx/etc/nginx/nginx.conf.sample diff --git a/configs/README.md b/configs/README.md new file mode 100644 index 0000000..12159f9 --- /dev/null +++ b/configs/README.md @@ -0,0 +1,75 @@ +# Configuration overrides folder +For each service, you can override any file within the container. +## File override +Under this `configs/` folder, just recreate the folder structure of the file from the root filesystem path and place a file there. +Example: override the `/etc/nginx/cors_params` of the `nginx` service. +``` +configs +└─ nginx <-- service name + └─ etc <-- /etc inside the container + └─ nginx + └─ cors_params <-- file +``` + +## File templating +Additionally, environment variables can be interpolated inside those files if they end with `.sample`. +For instance, `/etc/nginx/nginx.conf.sample` of the `nginx` service +```conf +... + server { + client_max_body_size 0; + listen 80; + server_name $URLS_CORE; +... +``` +Will become `/etc/nginx/nginx.conf` with `$URLS_CORE` replaced by `cytomine.local` +```conf +... + server { + client_max_body_size 0; + listen 80; + server_name cytomine.local +... +``` +# List of services and overrides examples +Here is a list of what can be defined in the `configs/` folder. +## core +Example: +``` +configs +└─ core <-- service name + └─ app <-- /app inside the container + └─ logback.xml <-- file +``` +See here what can be overridden https://github.com/cytomine/Cytomine-core-spring/blob/main/README.md#override-configuration + +## nginx +Example: +``` +configs +└─ nginx <-- service name + └─ etc <-- /etc inside the container + └─ nginx + └─ cors_params <-- file +``` +See here what can be overridden https://github.com/cytomine/Cytomine-nginx/tree/main/cm_configs_default + +## pims +Example: +``` +configs +└─ pims <-- service name + └─ app <-- /app inside the container + └─ logging.xml <-- file +``` +See here what can be overridden https://github.com/cytomine/pims + +## web_ui +Example: +``` +configs +└─ web_ui <-- service name + └─ app <-- /app inside the container + └─ configuration.json.sample <-- file +``` +See here what can be overridden https://github.com/cytomine/Cytomine-Web-UI diff --git a/configs/nginx/etc/nginx/cors_params b/configs/nginx/etc/nginx/cors_params deleted file mode 100644 index 99dc9d4..0000000 --- a/configs/nginx/etc/nginx/cors_params +++ /dev/null @@ -1,3 +0,0 @@ -add_header 'Access-Control-Allow-Origin' '*' always; -add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always; -add_header 'Access-Control-Allow-Headers' 'Content-Type, Content-Range, Content-Disposition, Content-Description, date, dateFull, authorization,content-type-full' always; diff --git a/configs/nginx/etc/nginx/nginx.conf.sample b/configs/nginx/etc/nginx/nginx.conf.sample deleted file mode 100644 index 2662c9d..0000000 --- a/configs/nginx/etc/nginx/nginx.conf.sample +++ /dev/null @@ -1,172 +0,0 @@ -# -# Copyright (c) 2009-2020. Authors: see NOTICE file. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -worker_processes 1; -daemon off; - -events { - worker_connections 1024; -} - -http { - include mime.types; - default_type application/octet-stream; - server_names_hash_bucket_size 128; - server_tokens off; - - # Use Docker's resolver to resolve container names when used with set and proxy_pass - resolver 127.0.0.11 valid=30s; - - log_format main '$scheme - h=$host - hh=$http_host - sn=$server_name - ra=$remote_addr - ru=$remote_user [$time_local] req="$request" ' - 's=$status bbs=$body_bytes_sent hr="$http_referer" ' - 'hua="$http_user_agent" hxff="$http_x_forwarded_for" hxfh="$http_x_forwarded_host"'; - - sendfile on; - keepalive_timeout 120; - proxy_connect_timeout 75; - - large_client_header_buffers 4 16k; - error_log /var/log/nginx/error.log debug; - access_log /var/log/nginx/access.log main; - - upstream ims-pooled { - server $INTERNAL_URLS_IMS; - } - - # Default server. Useful when no server_name matched because Host is wrong but can be found in http_x_forwarded_host - # The downside is that when there's no matching server block at all for an URL that finds its way here, there's an infinite loop. - server { - client_max_body_size 0; - listen 80 default_server; - location / { - proxy_pass http://127.0.0.1; - proxy_set_header Host $http_x_forwarded_host; - } - } - - server { - client_max_body_size 0; - listen 80; - server_name $URLS_CORE; - - location / { - include /etc/nginx/cors_params; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - - # Allow nginx to start without web_ui. Else it may crash on start saying "[emerg] host not found in upstream "web_ui"" - set $tmpwebuivar "$INTERNAL_URLS_WEB_UI"; - proxy_pass http://$tmpwebuivar; - - #proxy_http_version 1.1; - #proxy_set_header Upgrade $http_upgrade; - #proxy_set_header Connection "upgrade"; - } - location ~ /(api|monitoring|login|logout|saml|saml2|server|session|custom-ui|static|restApiDoc|js/jsondoc|css/jsondoc) { - proxy_intercept_errors off; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - set $tmpcorehost "$INTERNAL_URLS_CORE"; - proxy_pass http://$tmpcorehost; - } - location /ws { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - set $tmpcorehost "$INTERNAL_URLS_CORE"; - proxy_pass http://$tmpcorehost; - } - location /www/ { - alias /www/; - } - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - proxy_intercept_errors on; - - error_page 404 = /cytomine-page-not-found; - location /cytomine-page-not-found { - default_type "text/html"; - alias /opt/nginx/nginx-1.10.1/html/404.html; - } - } - server { - client_max_body_size 0; - listen 80; - server_name $URLS_IMAGE_SERVER; - - location / { - if ($request_method = OPTIONS ) { - include /etc/nginx/cors_params; - add_header Content-Length 0; - add_header Content-Type text/plain; - return 200; - } - - include /etc/nginx/cors_params; - proxy_set_header Host $host; - set $tmpims1host ims-pooled; - proxy_pass http://$tmpims1host; - } - } - server { - client_max_body_size 0; - proxy_request_buffering off; - listen 80; - server_name $URLS_UPLOAD; - - location / { - include /etc/nginx/cors_params; - - if ($request_method = OPTIONS ) { - add_header Content-Length 0; - add_header Content-Type text/plain; - return 200; - } - return 444; - } - - # Upload form should be submitted to this location - location /upload { - - if ($request_method = OPTIONS ) { - include /etc/nginx/cors_params; - add_header 'Access-Control-Allow-Credentials' 'false' always; - add_header Content-Length 0; - add_header Content-Type text/plain; - return 200; - } - - proxy_read_timeout 600; - - add_header 'Access-Control-Allow-Credentials' 'false'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'Content-Type, Content-Range, Content-Disposition, Content-Description, date, dateFull, authorization,content-type-full'; - add_header 'Access-Control-Max-Age' '1728000'; - add_header 'Access-Control-Allow-Origin' '*'; - proxy_pass http://$INTERNAL_URLS_IMS; - } - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root html; - } - } -} diff --git a/cytomine.template b/cytomine.template index 75ceb78..63ba713 100644 --- a/cytomine.template +++ b/cytomine.template @@ -36,9 +36,9 @@ global: BIOFORMAT: cytomine/bioformat:v1.2.0 CORE: cytomine/core:4.4.0 MONGO: cytomine/mongo:0.0.1 - NGINX: cytomine/nginx:1.2.3 + NGINX: cytomine/nginx:2.0.0 PIMS_CACHE: redis:7.0.8 - PIMS: cytomine/pims-ce-package:0.0.3 + PIMS: cytomine/pims-ce-package:0.2.0 POSTGIS: cytomine/postgis:15-3.3-alpine-1.2.1 RABBITMQ: rabbitmq:3.10 WEB_UI: cytomine/web_ui:2.6.2 From 597c152d881bad5264bb5049b659eef9af8779f3 Mon Sep 17 00:00:00 2001 From: Mohamed Eltahir Date: Wed, 20 Sep 2023 17:56:28 +0200 Subject: [PATCH 7/7] fix: :bug: add OME to list of prefixes to hide --- cytomine.template | 1 + 1 file changed, 1 insertion(+) diff --git a/cytomine.template b/cytomine.template index 63ba713..c5059c1 100644 --- a/cytomine.template +++ b/cytomine.template @@ -112,6 +112,7 @@ global: DICOM=DICOM. MIRAX=MIRAX. ICC_PROFILE=ICC_PROFILE. + OME=OME. services: default: