Merge branch 'RESTAPI-990-add-home-parameter' into 'master'

Restapi 990 add home parameter

See merge request firecrest/firecrest!273

CHANGELOG.md

@@ -5,11 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.14.1]
+## [1.15.0]
 
 ### Added
 
-- `/utilities/whoami` adds the `boolean` parameter "`groups`" which set on `true` returns a dictionary with `uid`, `gid`, and `groups`
+- Added the endpoints `GET /status/filesystems` and `GET /status/filesystems/<system>`, providing enhancement in terms of filesystem availability
+- The endpoint `/utilities/whoami` adds the `boolean` parameter "`groups`" which set on `true` returns a dictionary with `uid`, `gid`, and `groups`
+- Added the parameter `WORKLOAD_MANAGER` in `GET /status/parameters` to provide information on the resource and workload manager used for compute
 - Add F7T_LOG_TYPE to select logging to files or stdout.
 - Add F7T_GUNICORN_LOG for Gunicorn logs.
 - Add profiling midelware.
@@ -20,6 +22,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed check when submitted an empty batch file on `POST /compute/jobs/upload` 
 - Fixed error message when `GET /status/systems` encounters error in one filesystem
 
+### Changed
+
+- Upgraded `kong` gateway to [v3.6.0](https://docs.konghq.com/gateway/changelog/#3600)
+- Upgraded `cryptography`package to [v42.0.4](https://cryptography.io/en/latest/changelog/#v42-0-4)
+- Upgraded `paramiko` package to [v3.4.0](https://github.com/paramiko/paramiko/tree/3.4.0)
 
 ## [1.14.0]
 

deploy/demo/common/common.env

@@ -17,8 +17,6 @@ F7T_AUTH_ALLOWED_USERS=''
 # Keycloak scope for clients:
 F7T_AUTH_REQUIRED_SCOPE='firecrest'
 F7T_REALM_RSA_TYPE=RS256
-# scopes filtered for this Firecrest, eg  firecrest-tds.cscs.ch, firecrest-production.cscs.ch
-F7T_FIRECREST_SERVICE='firecrest.some.place'
 # AUTHENTICATION ROLE for FirecREST Service Accounts
 F7T_AUTH_ROLE='firecrest-sa'
 # DEBUG FLAG
@@ -57,11 +55,17 @@ F7T_KONG_URL=http://192.168.220.10:8000
 # list of systems
 #public name for systems, where users except to submit jobs and get files (list with ';')
 F7T_SYSTEMS_PUBLIC='cluster;cluster'
-# filesystems mounted in each system
-# ; separated for system (position related to SYSTEMS_PUBLIC) and for each filesystem mounted inside each system, separated with ","
-# example: let's suppose F7T_SYSTEMS_PUBLIC="cluster1;cluster2", cluster1 has "/home" and "/scratch", and cluster2 has mounted "/home":
-# F7T_FILESYSTEMS = "/home,/scratch;/home"
-F7T_FILESYSTEMS="/home;/home"
+# List of filesystems mounted in each system
+#F7T_FILESYSTEMS="{'cluster01': [{'name':'PROJECT','path':'/project','description':'Project Filesystem'}, 
+#                                {'name':'STORE', 'path':'/store', 'description':'Long term filesystem'}, 
+#                                {'name':'SCRATCH', 'path':'/scratch', 'description':'Fast Lustre filesystem'} ], 
+#                 'cluster02': [{'name':'PROJECT','path':'/project','description':'Project Filesystem'}, 
+#                                {'name':'STORE', 'path':'/store', 'description':'Long term filesystem'}, 
+#                                {'name':'SCRATCH', 'path':'/home', 'description':'Home filesystem'}] }"
+F7T_FILESYSTEMS="{
+                   'cluster': [{'name':'HOME', 'path':'/home', 'description':'Users home filesystem'} , 
+                               {'name':'SCRATCH', 'path':'/scratch', 'description':'Scratch filesystem'}] 
+                 } "  
 #internal machines that microservices connect to (in correlation with SYSTEMS_PUBLIC)
 F7T_SYSTEMS_INTERNAL_COMPUTE='192.168.220.12:22;192.168.220.12:22'
 F7T_SYSTEMS_INTERNAL_STORAGE='192.168.220.12:22;192.168.220.12:22'

deploy/demo/docker-compose.yml

@@ -244,7 +244,7 @@ services:
       - ./logs/keycloak:/opt/jboss/keycloak/standalone/log/:delegated
 
   kong:
-    image: kong:2.8.4
+    image: kong:3.6.0
     container_name: kong
     environment:
       - KONG_DATABASE=off

deploy/docker/base/requirements.txt

@@ -1,4 +1,4 @@
-cryptography==39.0.2
+cryptography==42.0.4
 markupsafe==2.1.2
 Flask==2.3.2
 PyJWT==2.4.0

deploy/docker/compute/requirements.txt

@@ -1,2 +1,2 @@
 -r ../base/requirements.txt
-paramiko==2.11.0
+paramiko==3.4.0

deploy/docker/reservations/requirements.txt

@@ -1,2 +1,2 @@
 -r ../base/requirements.txt
-paramiko==2.11.0
+paramiko==3.4.0

deploy/docker/status/requirements.txt

@@ -1,2 +1,2 @@
 -r ../base/requirements.txt
-paramiko==2.11.0
+paramiko==3.4.0

deploy/docker/storage/requirements.txt

@@ -1,5 +1,5 @@
 -r ../base/requirements.txt
 keystoneauth1==5.1.2
 lxml==4.9.1
-paramiko==2.11.0
+paramiko==3.4.0
 python-keystoneclient==5.0.0

deploy/docker/utilities/requirements.txt

@@ -1,2 +1,2 @@
 -r ../base/requirements.txt
-paramiko==2.11.0
+paramiko==3.4.0

deploy/k8s/compute/templates/cm.compute.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 data:
   F7T_COMPUTE_BASE_FS: "{{ .Values.F7T_COMPUTE_BASE_FS }}"
   F7T_COMPUTE_PORT: "{{ .Values.F7T_COMPUTE_PORT }}"
-  F7T_FIRECREST_SERVICE: "{{ .Values.F7T_FIRECREST_SERVICE }}"
   F7T_SPANK_PLUGIN_OPTION: "{{ .Values.F7T_SPANK_PLUGIN_OPTION }}"
   F7T_USE_SPANK_PLUGIN: "{{ .Values.F7T_USE_SPANK_PLUGIN }}"
 kind: ConfigMap

deploy/k8s/kong/templates/deploy.kong.yaml

@@ -19,7 +19,7 @@ spec:
         role: gateway
     spec:
       containers:
-      - image: kong:2.8.4
+      - image: kong:3.6.0
         env:
         - name: KONG_DECLARATIVE_CONFIG
           value: /kong/kong.yaml

deploy/k8s/values-dev.yaml

@@ -13,7 +13,10 @@ F7T_AUTH_ROLE: firecrest-sa
 F7T_AUTH_TOKEN_AUD: ""
 F7T_CERTIFICATOR_URL: https://svc-certificator:5010
 F7T_COMPUTE_URL: https://svc-compute:5006
-F7T_FILESYSTEMS: /home
+F7T_FILESYSTEMS: "{
+                   'cluster': [{'name':'HOME', 'path':'/home', 'description':'Users home filesystem'} , 
+                               {'name':'SCRATCH', 'path':'/scratch', 'description':'Scratch filesystem'}] 
+                 } "
 F7T_GUNICORN_SSL: --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 --ssl-version TLSv1_2 --keyfile /f7t-ssl.key --certfile /f7t-ssl.crt
 F7T_JAEGER_AGENT: svc-jaeger
 F7T_KONG_URL: http://svc-kong:8000

deploy/test-build/environment/common.env

@@ -17,8 +17,6 @@ F7T_AUTH_ALLOWED_USERS=''
 # Keycloak scope for clients:
 F7T_AUTH_REQUIRED_SCOPE=''
 F7T_REALM_RSA_TYPE=RS256
-# scopes filtered for this Firecrest, eg  firecrest-tds.cscs.ch, firecrest-production.cscs.ch
-F7T_FIRECREST_SERVICE=''
 # AUTHENTICATION ROLE for FirecREST Service Accounts
 F7T_AUTH_ROLE=''
 # DEBUG FLAG
@@ -55,8 +53,17 @@ F7T_SYSTEMS_PUBLIC='system01;system02'
 # filesystems mounted in each system
 # ; separated for system (position related to SYSTEMS_PUBLIC) and for each filesystem mounted inside each system, separated with ","
 # example: let's suppose SYSTEMS_PUBLIC="cluster1;cluster2", cluster1 has "/home" and "/scratch", and cluster2 has mounted "/home":
-# FILESYSTEMS = "/home,/scratch;/home"
-F7T_FILESYSTEMS="/home;/home"
+# List of filesystems mounted in each system
+#F7T_FILESYSTEMS="{'cluster01': [{'name':'PROJECT','path':'/project','description':'Project Filesystem'}, 
+#                                {'name':'STORE', 'path':'/store', 'description':'Long term filesystem'}, 
+#                                {'name':'SCRATCH', 'path':'/scratch', 'description':'Fast Lustre filesystem'} ], 
+#                 'cluster02': [{'name':'PROJECT','path':'/project','description':'Project Filesystem'}, 
+#                                {'name':'STORE', 'path':'/store', 'description':'Long term filesystem'}, 
+#                                {'name':'SCRATCH', 'path':'/home', 'description':'Home filesystem'}] }"
+F7T_FILESYSTEMS=F7T_FILESYSTEMS="{
+                   'cluster': [{'name':'HOME', 'path':'/home', 'description':'Users home filesystem'} , 
+                               {'name':'SCRATCH', 'path':'/scratch', 'description':'Scratch filesystem'}] 
+                 } " 
 #internal machines that microservices connect to (in correlation with SYSTEMS_PUBLIC)
 F7T_SYSTEMS_INTERNAL_COMPUTE='cluster'
 F7T_SYSTEMS_INTERNAL_STORAGE='cluster'

doc/openapi/firecrest-api.yaml

@@ -8,7 +8,7 @@ openapi: 3.0.0
 servers:
   - url: "/"
 info:
-  version: 1.14.1
+  version: 1.15.0
   title: FirecREST API
   description: >
     FirecREST platform, a RESTful Services Gateway to HPC resources, is a
@@ -107,6 +107,58 @@ paths:
                 $ref: '#/components/schemas/System'
         '404':
           description: System does not exists
+
+
+  '/status/filesystems':
+    get:
+      summary: List of filesystems allowed in the API
+      description: Returns list of filesystems enabled in the API that can be configured in environment files.
+      tags:
+        - Status
+      responses:
+        '200':
+          description: List of filesystems
+          content:
+            "application/json":
+              schema:
+                properties:
+                  description: 
+                    description: Description of the output
+                    type: string
+                    default: Filesystem information
+                  out:
+                    type: object
+                    $ref: '#/components/schemas/Filesystems'
+
+
+  '/status/filesystems/{machinename}':
+    parameters:
+      - name: machinename
+        in: path
+        description: System name
+        required: true
+        schema:
+          type: string
+    get:
+      summary: List of filesystems allowed in the API
+      description: Returns list of filesystems enabled in the API that can be configured in environment files.
+      tags:
+        - Status
+      responses:
+        '200':
+          description: List of filesystems enabled on {machinename} 
+          content:
+            "application/json":
+              schema:
+                properties:
+                  description: 
+                    description: Description of the output
+                    type: string
+                    default: Filesystem information
+                  out:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/Filesystem'
   '/status/parameters':
     get:
       summary: List of API parameters
@@ -120,7 +172,6 @@ paths:
             '*/*':
               schema:
                 $ref: '#/components/schemas/Parameters'
-
   '/utilities/ls':
     parameters:
       - in: header
@@ -2658,6 +2709,43 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Service'
+
+    Filesystem:
+      properties:
+        description:
+          type: string
+          description: Description of the filesystem
+        name:
+          type: string
+          description: Internal name for the filesystem (HOME, SCRATCH, etc)
+        path:
+          type: string
+          description: Absolute path of the filesystem
+        status:
+          type: string
+          description: Description of the current status of the filesystem
+          enum:
+            - available
+            - down
+        status_code:
+          type: integer
+          description: Status code of the filesystem
+          enum:
+            - 200
+            - 400            
+
+    Filesystems:
+      type: object
+      properties:
+        system:
+          type: array
+          description: System name
+          items:
+            type: array
+            items: 
+              $ref: '#/components/schemas/Filesystem'
+
+
     Job:
       type: object
       required:

doc/openapi/firecrest-developers-api.yaml

@@ -8,7 +8,7 @@ openapi: 3.0.0
 servers:
   - url: "/"
 info:
-  version: 1.14.1
+  version: 1.15.0
   title: FirecREST Developers API
   description: >
     This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers.
@@ -95,6 +95,58 @@ paths:
                 $ref: '#/components/schemas/System'
         '404':
           description: System does not exists
+
+
+  '/status/filesystems':
+    get:
+      summary: List of filesystems allowed in the API
+      description: Returns list of filesystems enabled in the API that can be configured in environment files.
+      tags:
+        - Status
+      responses:
+        '200':
+          description: List of filesystems
+          content:
+            "application/json":
+              schema:
+                properties:
+                  description: 
+                    description: Description of the output
+                    type: string
+                    default: Filesystem information
+                  output:
+                    type: object
+                    $ref: '#/components/schemas/Filesystems'
+
+
+  '/status/filesystems/{machinename}':
+    parameters:
+      - name: machinename
+        in: path
+        description: System name
+        required: true
+        schema:
+          type: string
+    get:
+      summary: List of filesystems allowed in the API
+      description: Returns list of filesystems enabled in the API that can be configured in environment files.
+      tags:
+        - Status
+      responses:
+        '200':
+          description: List of filesystems enabled on {machinename} 
+          content:
+            "application/json":
+              schema:
+                properties:
+                  description: 
+                    description: Description of the output
+                    type: string
+                    default: Filesystem information
+                  output:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/Filesystem'
   '/status/parameters':
     get:
       summary: List of API parameters
@@ -108,7 +160,6 @@ paths:
             '*/*':
               schema:
                 $ref: '#/components/schemas/Parameters'
-
   '/utilities/ls':
     parameters:
       - in: header
@@ -2822,6 +2873,43 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Service'
+
+    Filesystem:
+      properties:
+        description:
+          type: string
+          description: Description of the filesystem
+        name:
+          type: string
+          description: Internal name for the filesystem (HOME, SCRATCH, etc)
+        path:
+          type: string
+          description: Absolute path of the filesystem
+        status:
+          type: string
+          description: Description of the current status of the filesystem
+          enum:
+            - available
+            - down
+        status_code:
+          type: integer
+          description: Status code of the filesystem
+          enum:
+            - 200
+            - 400            
+
+    Filesystems:
+      type: object
+      properties:
+        system:
+          type: array
+          description: System name
+          items:
+            type: array
+            items: 
+              $ref: '#/components/schemas/Filesystem'
+
+
     Job:
       type: object
       required: