From 424e7ebfc4853409652ced0b0e5ef19d9045eb1e Mon Sep 17 00:00:00 2001 From: mbenguig Date: Tue, 3 Dec 2024 16:21:18 +0100 Subject: [PATCH] Azure integration (#113) * Fix missing OS family and arch with Azure * Add constants in NodeCandidateUtils.java * Add constants in NodeCandidateUtils.java * Azure integration * Fix issue when retrieving Azure images --- .../sal/service/nc/NodeCandidateUtils.java | 1 + .../sal/service/service/CloudService.java | 10 ++- .../sal/service/service/NodeService.java | 20 ++++- .../sal/service/service/TaskBuilder.java | 37 ++++++-- .../PAResourceManagerGateway.java | 56 ------------ .../sal/service/util/ClusterUtils.java | 2 +- .../src/main/resources/Define_NS_Azure.xml | 88 +++++++++++++------ ...ript.groovy => acquire_node_script.groovy} | 0 8 files changed, 114 insertions(+), 100 deletions(-) rename sal-service/src/main/resources/{acquire_node_aws_script.groovy => acquire_node_script.groovy} (100%) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index 5f790f3..df4d984 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -297,6 +297,7 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA arch = osJSON.optBoolean("is64Bit") ? "AMD64" : "i386"; } } else if (AZURE.equals(nodeCandidateJSON.optString("cloud"))) { + image.setId(imageJSON.optString("id")); arch = osJSON.optString("arch"); } os.setOperatingSystemArchitecture(OperatingSystemArchitecture.fromValue(arch)); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java index eeb79fe..ac7cf36 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java @@ -346,10 +346,18 @@ public List getCloudImages(String sessionId, String cloudId) throws NotCo if (paCloud != null) { try { JSONArray imagesArray = connectorIaasGateway.getImages(paCloud.getDummyInfrastructureName()); + + String cloudIdOrEmpty; + if (paCloud.getCloudProviderName().equals("azure")) { + cloudIdOrEmpty = ""; + } else { + cloudIdOrEmpty = cloudId + "/"; + } List imagesIDs = IntStream.range(0, imagesArray.length()) .mapToObj(imagesArray::get) - .map(image -> cloudId + "/" + ((JSONObject) image).optString("id")) + .map(image -> cloudIdOrEmpty + ((JSONObject) image).optString("id")) .collect(Collectors.toList()); + LOGGER.debug("Filtering images related to cloud ID '{}'.", cloudId); allImages.stream().filter(image -> imagesIDs.contains(image.getId())).forEach(filteredImages::add); } catch (RuntimeException e) { diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index 1c62385..93da6c9 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -233,10 +233,22 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De break; case "azure": filename = File.separator + "Define_NS_Azure.xml"; - variables.put("azure_username", cloud.getCredentials().getUserName()); - variables.put("azure_secret", cloud.getCredentials().getPrivateKey()); - variables.put("azure_domain", cloud.getCredentials().getDomain()); - variables.put("azure_subscription_id", cloud.getCredentials().getSubscriptionId()); + variables.put("clientId", cloud.getCredentials().getUserName()); + variables.put("secret", cloud.getCredentials().getPrivateKey()); + variables.put("domain", cloud.getCredentials().getDomain()); + variables.put("subscriptionId", cloud.getCredentials().getSubscriptionId()); + variables.put("image", deployment.getNode().getNodeCandidate().getImage().getId()); + variables.put("imageOSType", + deployment.getNode() + .getNodeCandidate() + .getImage() + .getOperatingSystem() + .getOperatingSystemFamily() + .name()); + variables.put("vmSizeType", deployment.getNode().getNodeCandidate().getHardware().getProviderId()); + variables.put("vmUsername", cloud.getSshCredentials().getUsername()); + variables.put("vmPassword", cloud.getSshCredentials().getPrivateKey()); + variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); break; default: throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProviderName()); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java index 55e860c..5d9ae09 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java @@ -27,7 +27,6 @@ import com.google.common.base.Strings; import lombok.extern.log4j.Log4j2; -import net.bytebuddy.dynamic.scaffold.TypeInitializer; @Log4j2 @@ -39,9 +38,6 @@ public class TaskBuilder { private static final String SCRIPTS_SEPARATION_BASH = NEW_LINE + NEW_LINE + "# Separation script" + NEW_LINE + NEW_LINE; - private static final String SCRIPTS_SEPARATION_GROOVY = NEW_LINE + NEW_LINE + "// Separation script" + NEW_LINE + - NEW_LINE; - private static final String EMS_DEPLOY_PRE_SCRIPT = "emsdeploy_prescript.sh"; private static final String EMS_DEPLOY_PRIVATE_PRE_SCRIPT = "emsdeploy_prescript_private.sh"; @@ -58,7 +54,7 @@ public class TaskBuilder { private static final String CHECK_NODE_SOURCE_REGEXP_SCRIPT = "check_node_source_regexp.groovy"; - private static final String ACQUIRE_NODE_AWS_SCRIPT = "acquire_node_aws_script.groovy"; + private static final String ACQUIRE_NODE_SCRIPT = "acquire_node_script.groovy"; private static final String REMOVE_NODE_SCRIPT = "remove_node_script.groovy"; @@ -263,6 +259,9 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) { case "openstack": imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); break; + case "azure": + imageId = deployment.getNode().getNodeCandidate().getImage().getId(); + break; default: imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); } @@ -320,9 +319,9 @@ private Map createVariablesMapForAcquiringIAASNode(Task ta private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { LOGGER.debug("Acquiring node AWS script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString()); + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT); + taskNameSuffix, ACQUIRE_NODE_SCRIPT); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); @@ -338,9 +337,9 @@ private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, S private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { LOGGER.debug("Acquiring node OS script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString()); + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT); + taskNameSuffix, ACQUIRE_NODE_SCRIPT); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); @@ -353,12 +352,32 @@ private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, St return deployNodeTask; } + private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix, + String nodeToken) { + LOGGER.debug("Acquiring node Azure script file: " + + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); + ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() + + taskNameSuffix, ACQUIRE_NODE_SCRIPT); + + deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); + + Map variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken); + LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString()); + deployNodeTask.setVariables(variablesMap); + + addLocalDefaultNSRegexSelectionScript(deployNodeTask); + + return deployNodeTask; + } + private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { switch (deployment.getPaCloud().getCloudProviderName()) { case "aws-ec2": return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken); case "openstack": return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken); + case "azure": + return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken); default: return new ScriptTask(); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java index 6e543b7..73391e6 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java @@ -220,62 +220,6 @@ private RMStateFull getFullMonitoring() throws NotConnectedException, Permission return rmState; } - /** - * Deploy a simple AWS node source - * @param awsUsername A valid AWS user name - * @param awsKey A valid AWS secret key - * @param rmHostname The RM host name - * @param nodeSourceName The name of the node source - * @param numberVMs The number of needed VMs - * @throws NotConnectedException In case the user is not connected - * @throws PermissionRestException In case the user does not have valid permissions - */ - public void deploySimpleAWSNodeSource(String awsUsername, String awsKey, String rmHostname, String nodeSourceName, - Integer numberVMs) throws NotConnectedException, PermissionRestException { - reconnectIfDisconnected(); - // Getting NS configuration settings - String infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.AWSEC2Infrastructure"; - String[] infrastructureParameters = { awsUsername, //username - awsKey, //secret - numberVMs.toString(), //N of VMs - "1", //N VMs per node - "", //image - "", //OS - "", //awsKeyPair - "", //ram - "", //Ncore - "", //sg - "", //subnet - rmHostname, //host - "http://" + rmHostname + ":8080/connector-iaas", //connector-iaas url - "http://" + rmHostname + ":8080/rest/node.jar", //node jar url - "", "300000", //timeout - "" }; - LOGGER.debug("infrastructureParameters: " + Arrays.toString(infrastructureParameters)); - String[] infrastructureFileParameters = { "" }; - String policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.StaticPolicy"; - String[] policyParameters = { "ALL", "ME" }; - String[] policyFileParameters = {}; - String nodesRecoverable = "true"; - - LOGGER.debug("Creating NodeSource ..."); - rmRestInterface.defineNodeSource(RMConnectionHelper.getSessionId(), - nodeSourceName, - infrastructureType, - infrastructureParameters, - infrastructureFileParameters, - policyType, - policyParameters, - policyFileParameters, - nodesRecoverable); - LOGGER.info("NodeSource created."); - - LOGGER.debug("Deploying the NodeSource ..."); - rmRestInterface.deployNodeSource(RMConnectionHelper.getSessionId(), nodeSourceName); - LOGGER.info("NodeSource VMs deployed."); - - } - /** * Search the nodes with specific tags. * @param tags a list of tags which the nodes should contain. When not specified or an empty list, all the nodes known urls are returned diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java index a995d81..feabca7 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java @@ -97,7 +97,7 @@ private static Task createWorkerNodeTask(String clusterName, ClusterNodeDefiniti workerNodeTask.setName(workerNode.getNodeTaskName(clusterName)); workerNodeTask.setType(Installation.InstallationType.COMMANDS); workerNodeTask.setInstallationByType(createWorkerInstallation(envVars)); - if (cloud != null && !cloud.getSecurityGroup().isEmpty()) { + if (cloud != null && cloud.getSecurityGroup() != null && !cloud.getSecurityGroup().isEmpty()) { workerNodeTask.setSecurityGroup(cloud.getSecurityGroup()); } return workerNodeTask; diff --git a/sal-service/src/main/resources/Define_NS_Azure.xml b/sal-service/src/main/resources/Define_NS_Azure.xml index a074789..473b308 100644 --- a/sal-service/src/main/resources/Define_NS_Azure.xml +++ b/sal-service/src/main/resources/Define_NS_Azure.xml @@ -3,18 +3,34 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:proactive:jobdescriptor:3.13" xsi:schemaLocation="urn:proactive:jobdescriptor:3.13 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.13/schedulerjob.xsd" name="Define_NS_Azure" projectName="NebulOuS" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" > - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -44,26 +60,40 @@ println " ... OK !" // Getting NS configuration settings def infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.AzureInfrastructure" -def infrastructureParameters = [variables.get("azure_username"), //username - variables.get("azure_secret"), //secret - variables.get("NS_nVMs"), //N of VMs - "1", //N VMs per node - variables.get("image"), //image - variables.get("sshUsername"), //sshUsername - variables.get("sshKeyPairName"), //sshKeyPair - "", //ram - "", //Ncore - "", //VmType - variables.get("security_group"), //sg - variables.get("subnet"), //subnet - variables.get("rm_host_name"), //host - protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/connector-iaas", //connector-iaas url - protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", //node jar url +def infrastructureParameters = [variables.get("clientId"), + variables.get("secret"), + variables.get("domain"), + variables.get("subscriptionId"), + variables.get("authenticationEndpoint"), + variables.get("managementEndpoint"), + variables.get("resourceManagerEndpoint"), + variables.get("graphEndpoint"), + variables.get("rm_host_name"), + protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/connector-iaas", // connector-iaas url + variables.get("image"), + variables.get("imageOSType"), + variables.get("vmSizeType"), + variables.get("vmUsername"), + variables.get("vmPassword"), + variables.get("vmPublicKey"), + variables.get("resourceGroup"), + variables.get("region"), + variables.get("numberOfInstances"), + variables.get("numberOfNodesPerInstance"), + protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", // nodeJarURL + variables.get("privateNetworkCIDR"), + "true", // staticPublicIP "-Dproactive.useIPaddress=true", //additionalProperties" - "300000", //timeout - "mkdir -p /tmp/node && cd /tmp/node; if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u312b07-linux-x64.tar.gz; tar -xf jre-8u312b07-linux-x64.tar.gz; mv jre1.8.0_312b07/ jre; fi; wget -nv --no-check-certificate %nodeJarUrl% ; nohup jre/bin/java -Xmx600m -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &", - ""] -def infrastructureFileParameters = [variables.get("sshPrivateKey")] + variables.get("resourceUsageRefreshFreqInMin"), + variables.get("rateCardRefreshFreqInMin"), + variables.get("offerId"), + variables.get("currency"), + variables.get("locale"), + variables.get("regionInfo"), + variables.get("maxBudget"), + "mkdir -p /tmp/node && cd /tmp/node; if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u312b07-linux-x64.tar.gz; tar -xf jre-8u312b07-linux-x64.tar.gz; mv jre1.8.0_312b07/ jre; fi; wget -nv --no-check-certificate %nodeJarUrl% ; nohup jre/bin/java -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &", // linuxStartupScript + ""] // windowsStartupScript +def infrastructureFileParameters = [] def policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.EmptyPolicy" def poliyParameters = ["ALL","ME"] def policyFileParameters = [] diff --git a/sal-service/src/main/resources/acquire_node_aws_script.groovy b/sal-service/src/main/resources/acquire_node_script.groovy similarity index 100% rename from sal-service/src/main/resources/acquire_node_aws_script.groovy rename to sal-service/src/main/resources/acquire_node_script.groovy