Merge pull request #141 from Cray-HPE/release/3.17.0

Release/3.17.0

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.17.0] - 2024-08-29
+### Added
+- CASMCMS-8979 - add a status endpoint for the remote build nodes.
+- CASMCMS-8977 - check that the ssh key is present each time spawning a remote job.
+- CASMINST-6602 - enable dkms by default.
+- CASMTRIAGE-7169 - job memory size was not getting picked up correctly from the ims configuration settings.
+- CASMCMS-9040 - change read/write permissions of recipe config files output in image.
+
+### Dependencies
+- CSM 1.6 moved to Kubernetes 1.24, so use client v24.x to ensure compatibility
+
 ## [3.16.2] - 2024-07-25
 ### Dependencies
 - Resolved CVE: Require `setuptools` >= 70.0

api/openapi.yaml

@@ -629,6 +629,46 @@ paths:
           $ref: '#/components/responses/NotFound'
         '500':
           $ref: '#/components/responses/InternalServerError'
+  /v3/remote-build-nodes/status/{remote_build_node_xname}:
+    parameters:
+      - $ref: '#/components/parameters/remote_build_node_xname'
+    get:
+      summary: List remote build node status objects
+      operationId: get_all_v3_remote_build_status
+      tags:
+        - remote build node status
+        - v3
+      description: Retrieve the status of all remote build nodes that are registered with IMS.
+      responses:
+        '200':
+          description: A collection of the status of each remote build node
+          content:
+            application/json:
+              schema:
+                items:
+                  $ref: '#/components/schemas/RemoteBuildNodeStatus'
+                type: array
+        '500':
+          $ref: '#/components/responses/InternalServerError'
+  /v3/remote-build-nodes/status:
+    get:
+      summary: List remote build node status objects
+      operationId: get_all_v3_remote_build_status
+      tags:
+        - remote build node status
+        - v3
+      description: Retrieve the status of all remote build nodes that are registered with IMS.
+      responses:
+        '200':
+          description: A collection of the status of each remote build node
+          content:
+            application/json:
+              schema:
+                items:
+                  $ref: '#/components/schemas/RemoteBuildNodeStatus'
+                type: array
+        '500':
+          $ref: '#/components/responses/InternalServerError'
   /v3/jobs:
     get:
       summary: Retrieve a list of JobRecords that are registered with IMS
@@ -2072,6 +2112,42 @@ components:
           example: x3000c1s10b1n0
           type: string
           minLength: 1
+    RemoteBuildNodeStatus:
+      description: A Remote Build Node Status
+      type: object
+      required:
+        - xname
+      properties:
+        xname:
+          description: Xname of the remote build node
+          example: x3000c1s10b1n0
+          type: string
+          minLength: 1
+        nodeArch:
+          description: Architecture of the remote build node
+          example: x86_64
+          type: string
+          minLength: 1
+        numCurrentJobs:
+          description: Number of current jobs running on the remote build node
+          example: 15
+          type: integer
+          minLength: 1
+        podmanStatus:
+          description: Status of the podman executable on the remote build node
+          example: Podman present at /usr/bin/podman
+          type: string
+          minLength: 1
+        sshStatus:
+          description: Status of the ssh connection to the remote build node
+          example: SSH connection established
+          type: string
+          minLength: 1
+        ableToRunJobs:
+          description: If the node is able to run new jobs
+          example: True
+          type: boolean
+          minLength: 1
     ArtifactLinkRecord:
       description: An Artifact Link Record
       type: object

constraints.txt

@@ -1,4 +1,4 @@
-# Constraints updated 5/29/2024
+# Constraints updated 8/27/2024
 aniso8601==9.0.1
 boto3==1.34.114
 botocore==1.34.114
@@ -18,8 +18,8 @@ idna==3.7
 itsdangerous==2.2.0
 Jinja2==3.1.4
 jmespath==1.0.1
-# CSM 1.6 uses Kubernetes 1.22, so use client v22.x to ensure compatability
-kubernetes==22.6.0
+# CSM 1.6 moved to Kubernetes 1.24, so use client v24.x to ensure compatability
+kubernetes>=24.2,<24.3
 MarkupSafe==2.1.5
 marshmallow==3.21.2
 oauthlib==3.2.2

kubernetes/cray-ims/values.yaml

@@ -1,7 +1,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -88,7 +88,7 @@ customer_access:
   subnet_name: "cmn"
 
 jobs:
-  enable_dkms: false
+  enable_dkms: true
   kata_runtime: "kata-qemu"
   aarch64_runtime: "kata-qemu"
 

src/server/models/jobs.py

@@ -34,6 +34,8 @@
 from marshmallow.validate import Length, OneOf, Range
 
 from src.server.helper import ARCH_ARM64, ARCH_X86_64
+from src.server.vault import test_private_key_file
+from src.server.models.remote_build_nodes import RemoteNodeStatus
 
 JOB_TYPE_CREATE = 'create'
 JOB_TYPE_CUSTOMIZE = 'customize'
@@ -83,7 +85,7 @@ def __init__(self, job_type, artifact_id, id=None, created=None, status=None,
                  kubernetes_configmap=None, enable_debug=False,
                  build_env_size=None, image_root_archive_name=None, kernel_file_name=None,
                  initrd_file_name=None, resultant_image_id=None, ssh_containers=None,
-                 kubernetes_namespace=None, kernel_parameters_file_name=None, require_dkms=False,
+                 kubernetes_namespace=None, kernel_parameters_file_name=None, require_dkms=True,
                  arch=None, job_mem_size=None, kubernetes_pvc=None, remote_build_node=""):
         # Supplied
         # v2.0
@@ -161,11 +163,11 @@ class V2JobRecordInputSchema(Schema):
     ssh_containers = fields.List(fields.Nested(SshContainerInputSchema()), allow_none=True)
 
     # v2.1
-    require_dkms = fields.Boolean(required=False, load_default=False, dump_default=False,
+    require_dkms = fields.Boolean(required=False, load_default=True, dump_default=True,
                                   metadata={"metadata": {"description": "Job requires the use of dkms"}})
 
     # v2.2
-    job_mem_size = fields.Integer(load_default=1, dump_default=1,
+    job_mem_size = fields.Integer(dump_default=1, required=False,
                                   validate=Range(min=1, error="build_env_size must be greater than or equal to 1"),
                                   metadata={"metadata": {"description": "Approximate working memory in GiB to reserve for the build job "
                                     "environment (loosely proportional to the final image size)"}})
@@ -259,13 +261,26 @@ def find_remote_node_for_job(app, job: V2JobRecordSchema) -> str:
     """
     app.logger.info(f"Checking for remote build node for job")
     best_node = ""
-    best_node_job_count = 10000
+    best_node_job_count = 10000 # seed with a really big number of jobs
+
+    # make sure the ssh key was set up correctly
+    if not test_private_key_file(app):
+        app.logger.error("Problem with ssh key - unable to create remote jobs")
+        return best_node
+
+    # Since the ssh key is good - look for a valid node
     for xname, remote_node in app.data['remote_build_nodes'].items():
-        arch, numJobs = remote_node.getStatus()
-        if arch != None and arch == job.arch:
-            app.logger.info(f"Matching remote node: {xname}, current jobs on node: {numJobs}")
-            # matching arch - can use the node, now pick the best
-            if best_node == "" or numJobs < best_node_job_count:
+        nodeStatus = remote_node.getStatus()
+        if nodeStatus.ableToRunJobs and nodeStatus.nodeArch == job.arch:
+            app.logger.info(f"Matching remote node: {xname}, current jobs on node: {nodeStatus.numCurrentJobs}")
+
+            # -1 means no job information, make sure we don't prefer those nodes
+            numNodeJobs = nodeStatus.numCurrentJobs
+            if numNodeJobs == -1:
+                numNodeJobs = 10000
+
+            # matching arch - can use the node, now pick the node with the least jobs running
+            if best_node == "" or numNodeJobs < best_node_job_count:
                 best_node = remote_node.xname
-                best_node_job_count = numJobs
+                best_node_job_count = numNodeJobs
     return best_node

src/server/models/recipes.py

@@ -1,7 +1,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2018-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2018-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -56,7 +56,7 @@ class V2RecipeRecord:
 
     # pylint: disable=W0622
     def __init__(self, name, recipe_type, linux_distribution, link=None, id=None, created=None,
-                 template_dictionary=None, require_dkms=False, arch=ARCH_X86_64):
+                 template_dictionary=None, require_dkms=True, arch=ARCH_X86_64):
         # Supplied
         self.name = name
         self.link = link
@@ -91,7 +91,7 @@ class V2RecipeRecordInputSchema(Schema):
     template_dictionary = fields.List(fields.Nested(RecipeKeyValuePair()), required=False, allow_none=True)
 
     # v2.2
-    require_dkms = fields.Boolean(load_default=False, dump_default=False,
+    require_dkms = fields.Boolean(load_default=True, dump_default=True,
                                   metadata={"metadata": {"description": "Recipe requires the use of dkms"}})
     arch = fields.Str(required=False, metadata={"metadata": {"description": "Architecture of the recipe"}},
                           validate=OneOf([ARCH_ARM64,ARCH_X86_64]), load_default=ARCH_X86_64, dump_default=ARCH_X86_64)
@@ -126,6 +126,6 @@ class V2RecipeRecordPatchSchema(Schema):
     arch = fields.Str(required=False, validate=OneOf([ARCH_ARM64,ARCH_X86_64]),
                       load_default=ARCH_X86_64, dump_default=ARCH_X86_64,
                       metadata={"metadata": {"description": "Architecture of the recipe"}})
-    require_dkms = fields.Boolean(required=False, load_default=False, dump_default=False,
+    require_dkms = fields.Boolean(required=False, load_default=True, dump_default=True,
                                   metadata={"metadata": {"description": "Recipe requires the use of dkms"}})
     template_dictionary = fields.List(fields.Nested(RecipeKeyValuePair()), required=False, allow_none=True)

src/server/models/remote_build_nodes.py

@@ -26,6 +26,7 @@
 """
 
 import socket
+import json
 from flask import current_app as app
 
 from marshmallow import Schema, fields, post_load, RAISE
@@ -38,6 +39,20 @@
 
 from src.server.helper import ARCH_ARM64, ARCH_X86_64
 
+class RemoteNodeStatus:
+    """ Object to hold the current status of a remote build node """
+
+    def __init__(self, xname: str) -> None:
+        self.xname = xname
+        self.sshStatus = "Unknown"
+        self.podmanStatus = "Unknown"
+        self.nodeArch = "Unknown"
+        self.numCurrentJobs = -1
+        self.ableToRunJobs = False
+
+    def toJson(self):
+        return self.__dict__
+
 class V3RemoteBuildNodeRecord:
     """ The RemoteBuildNodeRecord object """
 
@@ -49,21 +64,19 @@ def __init__(self, xname):
     def __repr__(self):
         return '<V3RemoteBuildNodeRecord(xname={self.xname!r})>'.format(self=self)
 
-    def getStatus(self) -> (str, int): #(arch, current jobs)
+    def getStatus(self) -> RemoteNodeStatus:
         """
         Utility function to verify that a node is set up and available for remote
         builds. If the node can not be contacted or is not set up for running IMS
         jobs, this will return (None,None)
         
         Returns:
-            Archetecture of the node if it can be determined
-            Number of jobs currently running on the node
-
+            RemoteNodeStatus object with details about the current state of the
+            remote build node.
         """
 
         # start with status Invalid
-        arch = None
-        numJobs = None
+        status = RemoteNodeStatus(self.xname)
 
         # connect to the remote node
         connect_kwargs = {"key_filename": "/app/id_ecdsa"}
@@ -75,7 +88,9 @@ def getStatus(self) -> (str, int): #(arch, current jobs)
         except (BadHostKeyException, AuthenticationException, NoValidConnectionsError,
                 SSHException, socket.error) as error:
             app.logger.error(f"Unable to connect to node: {self.xname}, Error: {error}")
-            return arch, numJobs
+            status.sshStatus = f"Unable to connect to node. Error: {error}"
+            return status
+        status.sshStatus = "SSH connection established."
 
         # make sure the above connection gets closed on exit
         try:
@@ -86,20 +101,23 @@ def getStatus(self) -> (str, int): #(arch, current jobs)
 
                 # check result
                 if result.exited != 0:
-                    app.logger.error(f"Unable to determine archecture of node: {self.xname}, Error: {result.stdout} {result.stderr}")
-                    return arch, numJobs
+                    app.logger.error(f"Unable to determine architecture of node: {self.xname}, Error: {result.stdout} {result.stderr}")
+                    status.nodeArch = f"Unable to determine architecture of node. Error: {result.stdout} {result.stderr}"
+                    return status
 
                 # see if we can pull out a known arch type
                 if "aarch64" in result.stdout:
-                    arch = ARCH_ARM64
+                    status.nodeArch = ARCH_ARM64
                 elif "x86" in result.stdout:
-                    arch = ARCH_X86_64
+                    status.nodeArch = ARCH_X86_64
                 else:
-                    app.logger.error(f"Unable to determine archecture of node: {self.xname}, Error: {result.stdout}")
-                    return arch, numJobs
+                    app.logger.error(f"Undefined architecture type for node: {self.xname}, Error: {result.stdout}")
+                    status.nodeArch = f"Undefined architecture type for node, result: {result.stdout}"
+                    return status
             except (UnexpectedExit, Failure) as error:
-                app.logger.error(f"Unable to determine archecture of node: {self.xname}, Error: {error}")
-                return arch, numJobs
+                app.logger.error(f"Unable to determine architecture of node: {self.xname}, Error: {error}")
+                status.nodeArch = f"Unable to determine architecture of node. Error: {error}"
+                return status
 
             # insure it has podman installed
             try:
@@ -109,16 +127,26 @@ def getStatus(self) -> (str, int): #(arch, current jobs)
                 # check result
                 if result.exited != 0:
                     app.logger.error(f"Unable to determine if podman is installed on node: {self.xname}, Error: {result.stdout} {result.stderr}")
-                    return None,None
+                    status.podmanStatus = f"Unable to determine if podman is installed on node. Error: {result.stdout} {result.stderr}"
+                    return status
 
                 # see if we can pull out a known arch type
                 if "/usr/bin/podman" not in result.stdout:
                     app.logger.error(f"Podman not installed on node: {self.xname}, Error: {result.stdout}")
-                    return
+                    status.podmanStatus = f"Podman not installed on node."
+                    return status
+
+                # report podman is present
+                status.podmanStatus = f"Podman present at /usr/bin/podman"
             except (UnexpectedExit, Failure) as error:
                 app.logger.error(f"Unable determine if tools are installed on node: {self.xname}, Error: {error}")
-                return None,None
+                status.podmanStatus = f"Unable determine if tools are installed on node. Error: {error}"
+                return status
 
+            # Don't fail the remote node over gathering number of current jobs - mark
+            # the node as valid now.
+            status.ableToRunJobs = True
+
             # Every running IMS job will create a working directory '/tmp/ims_(IMS_JOB_ID)'.
             # Count the number of these directories to find the number of running jobs on
             # the node - they are cleaned up when the job is complete on the node.
@@ -128,19 +156,16 @@ def getStatus(self) -> (str, int): #(arch, current jobs)
                 if result.exited != 0:
                     # let this go through and schedule a job on the node
                     app.logger.error(f"Unable to determine number of jobs on node: {self.xname}, Error: {result.stdout} {result.stderr}")
-                    numJobs = 0
                 else:
-                    numJobs = int(result.stdout)
+                    status.numCurrentJobs = int(result.stdout)
             except (UnexpectedExit, Failure) as error:
                 # Just log this, but allow the job to run
                 app.logger.error(f"Unable determine number of running jobs on node: {self.xname}, Error: {error}")
-                numJobs = 0
         finally:
             # close tha active connection
             c.close()
 
-        return arch, numJobs
-
+        return status
 
 class V3RemoteBuildNodeRecordInputSchema(Schema):
     """ A schema specifically for defining and validating user input """