From ae30ec2e9d7915c6e8959cc1ac4b9e6ae0612d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Tue, 5 Nov 2024 14:45:00 +0100 Subject: [PATCH 1/9] Fix missing OS family and arch with Azure --- .../org/ow2/proactive/sal/model/OperatingSystemFamily.java | 4 +++- .../org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/OperatingSystemFamily.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/OperatingSystemFamily.java index d36b611..0a6f200 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/OperatingSystemFamily.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/OperatingSystemFamily.java @@ -68,7 +68,9 @@ public enum OperatingSystemFamily { CLOUD_LINUX("CLOUD_LINUX"), - WINDOWS("WINDOWS"); + WINDOWS("WINDOWS"), + + LINUX("LINUX"); private String value; diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index be89fc6..b1139ef 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -279,7 +279,7 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA image.setProviderId(StringUtils.substringAfterLast(imageJSON.optString("id"), "/")); OperatingSystem os = new OperatingSystem(); JSONObject osJSON = imageJSON.optJSONObject("operatingSystem"); - os.setOperatingSystemFamily(OperatingSystemFamily.fromValue(osJSON.optString("family"))); + os.setOperatingSystemFamily(OperatingSystemFamily.fromValue(osJSON.optString("family").toUpperCase())); String arch = ""; if ("aws-ec2".equals(nodeCandidateJSON.optString("cloud"))) { @@ -288,6 +288,8 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA } else { arch = osJSON.optBoolean("is64Bit") ? "AMD64" : "i386"; } + } else if ("azure".equals(nodeCandidateJSON.optString("cloud"))) { + arch = osJSON.optString("arch"); } os.setOperatingSystemArchitecture(OperatingSystemArchitecture.fromValue(arch)); os.setOperatingSystemVersion(osJSON.optBigDecimal("version", BigDecimal.valueOf(0))); From c0538e46c598e3eab8203623394721d433ea2dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Tue, 5 Nov 2024 15:00:43 +0100 Subject: [PATCH 2/9] Add constants in NodeCandidateUtils.java --- .../sal/service/nc/NodeCandidateUtils.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index b1139ef..7de733c 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -38,6 +38,11 @@ @Component public class NodeCandidateUtils { + public static final String AWS_EC2 = "aws-ec2"; + public static final String AZURE = "azure"; + public static final String GCE = "gce"; + public static final String OPENSTACK = "openstack"; + @Autowired private PAConnectorIaasGateway connectorIaasGateway; @@ -224,7 +229,7 @@ private Hardware createHardware(JSONObject nodeCandidateJSON, PACloud paCloud) { hardware.setRam(Long.valueOf(minRam)); hardware.setFpga(hardwareJSON.optString("type")); - if ("aws-ec2".equals(nodeCandidateJSON.optString("cloud"))) { + if (AWS_EC2.equals(nodeCandidateJSON.optString("cloud"))) { hardware.setDisk((double) 8); } else { hardware.setDisk((double) 0); @@ -256,13 +261,13 @@ private Location createLocation(JSONObject nodeCandidateJSON, PACloud paCloud) { private GeoLocation createGeoLocation(String cloud, String region) { switch (cloud) { - case "aws-ec2": + case AWS_EC2: return new GeoLocation(geoLocationUtils.findGeoLocation("AWS", region)); - case "azure": + case AZURE: return new GeoLocation(geoLocationUtils.findGeoLocation("Azure", region)); - case "gce": + case GCE: return new GeoLocation(geoLocationUtils.findGeoLocation("GCE", region)); - case "openstack": + case OPENSTACK: return new GeoLocation(geoLocationUtils.findGeoLocation("OVH", region)); } LOGGER.warn("Cloud provider name no handled for Geo Location."); @@ -282,13 +287,13 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA os.setOperatingSystemFamily(OperatingSystemFamily.fromValue(osJSON.optString("family").toUpperCase())); String arch = ""; - if ("aws-ec2".equals(nodeCandidateJSON.optString("cloud"))) { + if (AWS_EC2.equals(nodeCandidateJSON.optString("cloud"))) { if (nodeCandidateJSON.optJSONObject("hw").optString("type").startsWith("a")) { arch = osJSON.optBoolean("is64Bit") ? "ARM64" : "ARM"; } else { arch = osJSON.optBoolean("is64Bit") ? "AMD64" : "i386"; } - } else if ("azure".equals(nodeCandidateJSON.optString("cloud"))) { + } else if (AZURE.equals(nodeCandidateJSON.optString("cloud"))) { arch = osJSON.optString("arch"); } os.setOperatingSystemArchitecture(OperatingSystemArchitecture.fromValue(arch)); @@ -376,13 +381,13 @@ public void saveNodeCandidates(List newCloudIds) { os = os.substring(0, 1).toUpperCase() + os.substring(1); String pair = os + ":" + region; switch (paCloud.getCloudProviderName()) { - case "aws-ec2": + case AWS_EC2: imageReq = "Linux"; break; - case "openstack": + case OPENSTACK: imageReq = os; break; - case "azure": + case AZURE: imageReq = os; break; default: @@ -390,7 +395,7 @@ public void saveNodeCandidates(List newCloudIds) { " is not handled yet."); } - if (paCloud.getCloudProviderName().equals("openstack")) { + if (paCloud.getCloudProviderName().equals(OPENSTACK)) { entries.add(pair); } populateNodeCandidatesFromCache(paCloud, region, imageReq, image); From 709bd1e587f0b7a34e3d81d8f059ea00dade032a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Tue, 5 Nov 2024 15:02:53 +0100 Subject: [PATCH 3/9] Add constants in NodeCandidateUtils.java --- .../org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index 7de733c..bc2e6e6 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -39,8 +39,11 @@ public class NodeCandidateUtils { public static final String AWS_EC2 = "aws-ec2"; + public static final String AZURE = "azure"; + public static final String GCE = "gce"; + public static final String OPENSTACK = "openstack"; @Autowired From ac5bc3e2ba0360842af07c7856277d32f3099f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Mon, 2 Dec 2024 12:26:39 +0100 Subject: [PATCH 4/9] Azure integration --- .../sal/service/nc/NodeCandidateUtils.java | 1 + .../sal/service/service/NodeService.java | 20 ++++- .../sal/service/service/TaskBuilder.java | 37 ++++++-- .../PAResourceManagerGateway.java | 56 ------------ .../sal/service/util/ClusterUtils.java | 2 +- .../src/main/resources/Define_NS_Azure.xml | 88 +++++++++++++------ ...ript.groovy => acquire_node_script.groovy} | 0 7 files changed, 105 insertions(+), 99 deletions(-) rename sal-service/src/main/resources/{acquire_node_aws_script.groovy => acquire_node_script.groovy} (100%) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index bc2e6e6..e38dc77 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -297,6 +297,7 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA arch = osJSON.optBoolean("is64Bit") ? "AMD64" : "i386"; } } else if (AZURE.equals(nodeCandidateJSON.optString("cloud"))) { + image.setId(imageJSON.optString("id")); arch = osJSON.optString("arch"); } os.setOperatingSystemArchitecture(OperatingSystemArchitecture.fromValue(arch)); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index 1c62385..93da6c9 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -233,10 +233,22 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De break; case "azure": filename = File.separator + "Define_NS_Azure.xml"; - variables.put("azure_username", cloud.getCredentials().getUserName()); - variables.put("azure_secret", cloud.getCredentials().getPrivateKey()); - variables.put("azure_domain", cloud.getCredentials().getDomain()); - variables.put("azure_subscription_id", cloud.getCredentials().getSubscriptionId()); + variables.put("clientId", cloud.getCredentials().getUserName()); + variables.put("secret", cloud.getCredentials().getPrivateKey()); + variables.put("domain", cloud.getCredentials().getDomain()); + variables.put("subscriptionId", cloud.getCredentials().getSubscriptionId()); + variables.put("image", deployment.getNode().getNodeCandidate().getImage().getId()); + variables.put("imageOSType", + deployment.getNode() + .getNodeCandidate() + .getImage() + .getOperatingSystem() + .getOperatingSystemFamily() + .name()); + variables.put("vmSizeType", deployment.getNode().getNodeCandidate().getHardware().getProviderId()); + variables.put("vmUsername", cloud.getSshCredentials().getUsername()); + variables.put("vmPassword", cloud.getSshCredentials().getPrivateKey()); + variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); break; default: throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProviderName()); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java index 55e860c..5d9ae09 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java @@ -27,7 +27,6 @@ import com.google.common.base.Strings; import lombok.extern.log4j.Log4j2; -import net.bytebuddy.dynamic.scaffold.TypeInitializer; @Log4j2 @@ -39,9 +38,6 @@ public class TaskBuilder { private static final String SCRIPTS_SEPARATION_BASH = NEW_LINE + NEW_LINE + "# Separation script" + NEW_LINE + NEW_LINE; - private static final String SCRIPTS_SEPARATION_GROOVY = NEW_LINE + NEW_LINE + "// Separation script" + NEW_LINE + - NEW_LINE; - private static final String EMS_DEPLOY_PRE_SCRIPT = "emsdeploy_prescript.sh"; private static final String EMS_DEPLOY_PRIVATE_PRE_SCRIPT = "emsdeploy_prescript_private.sh"; @@ -58,7 +54,7 @@ public class TaskBuilder { private static final String CHECK_NODE_SOURCE_REGEXP_SCRIPT = "check_node_source_regexp.groovy"; - private static final String ACQUIRE_NODE_AWS_SCRIPT = "acquire_node_aws_script.groovy"; + private static final String ACQUIRE_NODE_SCRIPT = "acquire_node_script.groovy"; private static final String REMOVE_NODE_SCRIPT = "remove_node_script.groovy"; @@ -263,6 +259,9 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) { case "openstack": imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); break; + case "azure": + imageId = deployment.getNode().getNodeCandidate().getImage().getId(); + break; default: imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); } @@ -320,9 +319,9 @@ private Map createVariablesMapForAcquiringIAASNode(Task ta private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { LOGGER.debug("Acquiring node AWS script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString()); + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT); + taskNameSuffix, ACQUIRE_NODE_SCRIPT); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); @@ -338,9 +337,9 @@ private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, S private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { LOGGER.debug("Acquiring node OS script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString()); + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT); + taskNameSuffix, ACQUIRE_NODE_SCRIPT); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); @@ -353,12 +352,32 @@ private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, St return deployNodeTask; } + private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix, + String nodeToken) { + LOGGER.debug("Acquiring node Azure script file: " + + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); + ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() + + taskNameSuffix, ACQUIRE_NODE_SCRIPT); + + deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); + + Map variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken); + LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString()); + deployNodeTask.setVariables(variablesMap); + + addLocalDefaultNSRegexSelectionScript(deployNodeTask); + + return deployNodeTask; + } + private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { switch (deployment.getPaCloud().getCloudProviderName()) { case "aws-ec2": return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken); case "openstack": return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken); + case "azure": + return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken); default: return new ScriptTask(); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java index 6e543b7..73391e6 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java @@ -220,62 +220,6 @@ private RMStateFull getFullMonitoring() throws NotConnectedException, Permission return rmState; } - /** - * Deploy a simple AWS node source - * @param awsUsername A valid AWS user name - * @param awsKey A valid AWS secret key - * @param rmHostname The RM host name - * @param nodeSourceName The name of the node source - * @param numberVMs The number of needed VMs - * @throws NotConnectedException In case the user is not connected - * @throws PermissionRestException In case the user does not have valid permissions - */ - public void deploySimpleAWSNodeSource(String awsUsername, String awsKey, String rmHostname, String nodeSourceName, - Integer numberVMs) throws NotConnectedException, PermissionRestException { - reconnectIfDisconnected(); - // Getting NS configuration settings - String infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.AWSEC2Infrastructure"; - String[] infrastructureParameters = { awsUsername, //username - awsKey, //secret - numberVMs.toString(), //N of VMs - "1", //N VMs per node - "", //image - "", //OS - "", //awsKeyPair - "", //ram - "", //Ncore - "", //sg - "", //subnet - rmHostname, //host - "http://" + rmHostname + ":8080/connector-iaas", //connector-iaas url - "http://" + rmHostname + ":8080/rest/node.jar", //node jar url - "", "300000", //timeout - "" }; - LOGGER.debug("infrastructureParameters: " + Arrays.toString(infrastructureParameters)); - String[] infrastructureFileParameters = { "" }; - String policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.StaticPolicy"; - String[] policyParameters = { "ALL", "ME" }; - String[] policyFileParameters = {}; - String nodesRecoverable = "true"; - - LOGGER.debug("Creating NodeSource ..."); - rmRestInterface.defineNodeSource(RMConnectionHelper.getSessionId(), - nodeSourceName, - infrastructureType, - infrastructureParameters, - infrastructureFileParameters, - policyType, - policyParameters, - policyFileParameters, - nodesRecoverable); - LOGGER.info("NodeSource created."); - - LOGGER.debug("Deploying the NodeSource ..."); - rmRestInterface.deployNodeSource(RMConnectionHelper.getSessionId(), nodeSourceName); - LOGGER.info("NodeSource VMs deployed."); - - } - /** * Search the nodes with specific tags. * @param tags a list of tags which the nodes should contain. When not specified or an empty list, all the nodes known urls are returned diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java index a995d81..feabca7 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java @@ -97,7 +97,7 @@ private static Task createWorkerNodeTask(String clusterName, ClusterNodeDefiniti workerNodeTask.setName(workerNode.getNodeTaskName(clusterName)); workerNodeTask.setType(Installation.InstallationType.COMMANDS); workerNodeTask.setInstallationByType(createWorkerInstallation(envVars)); - if (cloud != null && !cloud.getSecurityGroup().isEmpty()) { + if (cloud != null && cloud.getSecurityGroup() != null && !cloud.getSecurityGroup().isEmpty()) { workerNodeTask.setSecurityGroup(cloud.getSecurityGroup()); } return workerNodeTask; diff --git a/sal-service/src/main/resources/Define_NS_Azure.xml b/sal-service/src/main/resources/Define_NS_Azure.xml index a074789..473b308 100644 --- a/sal-service/src/main/resources/Define_NS_Azure.xml +++ b/sal-service/src/main/resources/Define_NS_Azure.xml @@ -3,18 +3,34 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:proactive:jobdescriptor:3.13" xsi:schemaLocation="urn:proactive:jobdescriptor:3.13 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.13/schedulerjob.xsd" name="Define_NS_Azure" projectName="NebulOuS" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" > - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -44,26 +60,40 @@ println " ... OK !" // Getting NS configuration settings def infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.AzureInfrastructure" -def infrastructureParameters = [variables.get("azure_username"), //username - variables.get("azure_secret"), //secret - variables.get("NS_nVMs"), //N of VMs - "1", //N VMs per node - variables.get("image"), //image - variables.get("sshUsername"), //sshUsername - variables.get("sshKeyPairName"), //sshKeyPair - "", //ram - "", //Ncore - "", //VmType - variables.get("security_group"), //sg - variables.get("subnet"), //subnet - variables.get("rm_host_name"), //host - protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/connector-iaas", //connector-iaas url - protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", //node jar url +def infrastructureParameters = [variables.get("clientId"), + variables.get("secret"), + variables.get("domain"), + variables.get("subscriptionId"), + variables.get("authenticationEndpoint"), + variables.get("managementEndpoint"), + variables.get("resourceManagerEndpoint"), + variables.get("graphEndpoint"), + variables.get("rm_host_name"), + protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/connector-iaas", // connector-iaas url + variables.get("image"), + variables.get("imageOSType"), + variables.get("vmSizeType"), + variables.get("vmUsername"), + variables.get("vmPassword"), + variables.get("vmPublicKey"), + variables.get("resourceGroup"), + variables.get("region"), + variables.get("numberOfInstances"), + variables.get("numberOfNodesPerInstance"), + protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", // nodeJarURL + variables.get("privateNetworkCIDR"), + "true", // staticPublicIP "-Dproactive.useIPaddress=true", //additionalProperties" - "300000", //timeout - "mkdir -p /tmp/node && cd /tmp/node; if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u312b07-linux-x64.tar.gz; tar -xf jre-8u312b07-linux-x64.tar.gz; mv jre1.8.0_312b07/ jre; fi; wget -nv --no-check-certificate %nodeJarUrl% ; nohup jre/bin/java -Xmx600m -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &", - ""] -def infrastructureFileParameters = [variables.get("sshPrivateKey")] + variables.get("resourceUsageRefreshFreqInMin"), + variables.get("rateCardRefreshFreqInMin"), + variables.get("offerId"), + variables.get("currency"), + variables.get("locale"), + variables.get("regionInfo"), + variables.get("maxBudget"), + "mkdir -p /tmp/node && cd /tmp/node; if ! type -p jre/bin/java; then wget -nv -N https://s3.amazonaws.com/ci-materials/Latest_jre/jre-8u312b07-linux-x64.tar.gz; tar -xf jre-8u312b07-linux-x64.tar.gz; mv jre1.8.0_312b07/ jre; fi; wget -nv --no-check-certificate %nodeJarUrl% ; nohup jre/bin/java -jar node.jar -Dproactive.communication.protocol=%protocol% -Dpython.path=%jythonPath% -Dproactive.pamr.router.address=%rmHostname% -D%instanceIdNodeProperty%=%instanceId% -r %rmUrl% -s %nodeSourceName% %nodeNamingOption% -v %credentials% -w %numberOfNodesPerInstance% %additionalProperties% &", // linuxStartupScript + ""] // windowsStartupScript +def infrastructureFileParameters = [] def policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.EmptyPolicy" def poliyParameters = ["ALL","ME"] def policyFileParameters = [] diff --git a/sal-service/src/main/resources/acquire_node_aws_script.groovy b/sal-service/src/main/resources/acquire_node_script.groovy similarity index 100% rename from sal-service/src/main/resources/acquire_node_aws_script.groovy rename to sal-service/src/main/resources/acquire_node_script.groovy From 85772e8655ce4803d6b94a039a6596080cd1e412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Mon, 2 Dec 2024 23:54:29 +0100 Subject: [PATCH 5/9] Fix issue when retrieving Azure images --- .../proactive/sal/service/service/CloudService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java index eeb79fe..ac7cf36 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java @@ -346,10 +346,18 @@ public List getCloudImages(String sessionId, String cloudId) throws NotCo if (paCloud != null) { try { JSONArray imagesArray = connectorIaasGateway.getImages(paCloud.getDummyInfrastructureName()); + + String cloudIdOrEmpty; + if (paCloud.getCloudProviderName().equals("azure")) { + cloudIdOrEmpty = ""; + } else { + cloudIdOrEmpty = cloudId + "/"; + } List imagesIDs = IntStream.range(0, imagesArray.length()) .mapToObj(imagesArray::get) - .map(image -> cloudId + "/" + ((JSONObject) image).optString("id")) + .map(image -> cloudIdOrEmpty + ((JSONObject) image).optString("id")) .collect(Collectors.toList()); + LOGGER.debug("Filtering images related to cloud ID '{}'.", cloudId); allImages.stream().filter(image -> imagesIDs.contains(image.getId())).forEach(filteredImages::add); } catch (RuntimeException e) { From c025533c02f2539efa39a2cf2fc53ef13f64e337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Thu, 12 Dec 2024 14:27:01 +0100 Subject: [PATCH 6/9] Azure integration (support ssh pub key auth) --- .../org/ow2/proactive/sal/model/SSHCredentials.java | 8 +++++++- .../proactive/sal/service/service/CloudService.java | 11 +++++++++-- .../proactive/sal/service/service/NodeService.java | 1 + sal-service/src/main/resources/Define_NS_Azure.xml | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/SSHCredentials.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/SSHCredentials.java index e3c2981..d1970fc 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/SSHCredentials.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/SSHCredentials.java @@ -33,6 +33,11 @@ public class SSHCredentials implements Serializable { @JsonProperty("keyPairName") private String keyPairName = null; + @Lob + @Column(name = "PUBLIC_KEY") + @JsonProperty("publicKey") + private String publicKey = null; + @Lob @Column(name = "PRIVATE_KEY") @JsonProperty("privateKey") @@ -49,11 +54,12 @@ public boolean equals(Object o) { SSHCredentials sshCredentials = (SSHCredentials) o; return Objects.equals(this.username, sshCredentials.username) && Objects.equals(this.keyPairName, sshCredentials.keyPairName) && + Objects.equals(this.publicKey, sshCredentials.publicKey) && Objects.equals(this.privateKey, sshCredentials.privateKey); } @Override public int hashCode() { - return Objects.hash(username, keyPairName, privateKey); + return Objects.hash(username, keyPairName, publicKey, privateKey); } } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java index ac7cf36..c222a85 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java @@ -460,8 +460,15 @@ private Credentials hideCredentials(Credentials creds) { private SSHCredentials hideSshCredentials(SSHCredentials creds) { SSHCredentials newCreds = new SSHCredentials(); if (creds != null) { - newCreds.setKeyPairName(creds.getKeyPairName()); - newCreds.setUsername(creds.getUsername()); + if (creds.getUsername() != null) { + newCreds.setUsername(creds.getUsername()); + } + if (creds.getKeyPairName() != null) { + newCreds.setKeyPairName(creds.getKeyPairName()); + } + if (creds.getPublicKey() != null) { + newCreds.setPublicKey(creds.getPublicKey()); + } if (creds.getPrivateKey() != null) { newCreds.setPrivateKey(hideString(creds.getPrivateKey(), 3)); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index 93da6c9..0d09fc8 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -248,6 +248,7 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De variables.put("vmSizeType", deployment.getNode().getNodeCandidate().getHardware().getProviderId()); variables.put("vmUsername", cloud.getSshCredentials().getUsername()); variables.put("vmPassword", cloud.getSshCredentials().getPrivateKey()); + variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); break; default: diff --git a/sal-service/src/main/resources/Define_NS_Azure.xml b/sal-service/src/main/resources/Define_NS_Azure.xml index 473b308..03ff0ba 100644 --- a/sal-service/src/main/resources/Define_NS_Azure.xml +++ b/sal-service/src/main/resources/Define_NS_Azure.xml @@ -17,7 +17,7 @@ - + From 9d06479e7f13fe10107e46e862548aa79b91616f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Thu, 12 Dec 2024 16:05:03 +0100 Subject: [PATCH 7/9] AddCloud documentation for Azure --- endpoints/2-cloud-endpoints.md | 43 +++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/endpoints/2-cloud-endpoints.md b/endpoints/2-cloud-endpoints.md index 23e52ce..ba5dd35 100644 --- a/endpoints/2-cloud-endpoints.md +++ b/endpoints/2-cloud-endpoints.md @@ -78,6 +78,40 @@ Note that cloud credentials are validated only during async process. } ] ``` + +* For Azure cloud: + +```json +[ + { + "cloudId": "{{cloud_name}}", + "cloudProviderName": "azure", + "cloudType": "PUBLIC", + "subnet": null, + "securityGroup": null, + "sshCredentials": { + "username": "ubuntu", + "keyPairName": null, + "publicKey": "{{azure-publickey}}", + "privateKey": "{{azure-password}}" + }, + "endpoint": null, + "scope": { + "prefix": null, + "value": null + }, + "identityVersion": null, + "defaultNetwork": null, + "credentials": { + "user": "{{azure-user}}", + "secret": "{{azure-secret}}", + "domain": "{{azure-domain}}", + "subscriptionId": "{{azure-subscription_id}}" + }, + "blacklist": null + } +] +``` **Reply:** Error code, 0 if no Errors - `cloudId` (string): @@ -100,7 +134,8 @@ Contains SSH access information for the cloud. The required fields are: - `username` (string): The SSH username. - `keyPairName` (string): The name of the key pair used for SSH access. - - `privateKey` (string or `null`): The private key in RSA format, with line breaks represented by `\n` for JSON compatibility. If not required, use `null`. + - `publicKey` (string or `null`): The public key in RSA format. If not required, use `null`. + - `privateKey` (string or `null`): The private key in RSA format, with line breaks represented by `\n` for JSON compatibility. If not required, use `null`. For Azure, set it to the VM ssh password. - `endpoint` (string or `null`): The authentication endpoint for the cloud provider. For OpenStack, use your specific authentication URL. AWS does not require this field, so it can be `null`. @@ -121,8 +156,10 @@ Contains authentication details for accessing the cloud. The fields are: - `user` (string): The cloud user name or access key. - `secret` (string): The cloud password or secret access key. - `domain` (string or `null`): The domain for the cloud account, required by OpenStack. For AWS, set this to `null`. - - `blacklist` (string or `null`): - Allows you to specify any blacklisted regions (e.g. locations). Use `null` if not applicable. + - `subscriptionId` (string or `null`): The subscription id for the cloud account, required by Azure. For AWS and OpenStack, set this to `null`. + +- `blacklist` (string or `null`): +Allows you to specify any blacklisted regions (e.g. locations). Use `null` if not applicable. #### 2.2- GetAllClouds endpoint: From 5e6f305f195a2b320903b3dc09969a4373fbd82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Fri, 27 Dec 2024 09:15:48 +0100 Subject: [PATCH 8/9] Fix issues in SAL Azure integration --- .../sal/service/service/TaskBuilder.java | 36 +++++++++++++++---- .../src/main/resources/Define_NS_Azure.xml | 8 ++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java index 5d9ae09..db97f1a 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java @@ -242,10 +242,12 @@ private void addLocalDefaultNSRegexSelectionScript(ScriptTask scriptTask) { } private String createIAASNodeConfigJson(Task task, Deployment deployment) { + String nodeConfigJson = "{"; ObjectMapper mapper = new ObjectMapper(); String imageId; switch (deployment.getPaCloud().getCloudProviderName()) { case "aws-ec2": + // imageId if (WhiteListedInstanceTypesUtils.isHandledHardwareInstanceType(deployment.getNode() .getNodeCandidate() .getHardware() @@ -255,19 +257,39 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) { imageId = deployment.getNode().getNodeCandidate().getLocation().getName() + "/" + deployment.getNode().getNodeCandidate().getImage().getProviderId(); } + nodeConfigJson += "\"image\": \"" + imageId + "\""; + // vmType + nodeConfigJson += ", \"vmType\": \"" + + deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; break; case "openstack": - imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); + // imageId + nodeConfigJson += "\"image\": \"" + deployment.getNode().getNodeCandidate().getImage().getProviderId() + + "\""; + // vmType + nodeConfigJson += ", \"vmType\": \"" + + deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; break; case "azure": - imageId = deployment.getNode().getNodeCandidate().getImage().getId(); + // imageId + nodeConfigJson += "\"image\": \"" + deployment.getNode().getNodeCandidate().getImage().getId() + "\""; + // vmSizeType + nodeConfigJson += ", \"vmSizeType\": \"" + + deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; break; default: - imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId(); + // imageId + nodeConfigJson += "\"image\": \"" + deployment.getNode().getNodeCandidate().getImage().getProviderId() + + "\""; + // vmType + nodeConfigJson += ", \"vmType\": \"" + + deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; } - String nodeConfigJson = "{\"image\": \"" + imageId + "\", " + "\"vmType\": \"" + - deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\", " + - "\"nodeTags\": \"" + deployment.getNodeName() + "\""; + + // nodeTags + nodeConfigJson += ", \"nodeTags\": \"" + deployment.getNodeName() + "\""; + + // portsToOpen if (task.getPortsToOpen() != null && !task.getPortsToOpen().isEmpty()) { try { nodeConfigJson += ", \"portsToOpen\": " + mapper.writeValueAsString(task.getPortsToOpen()) + "\""; @@ -275,6 +297,8 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) { LOGGER.error(Arrays.toString(e.getStackTrace())); } } + + // securityGroups if (task.getSecurityGroup() != null) { nodeConfigJson += ", \"securityGroups\": [\"" + task.getSecurityGroup() + "\"]"; } diff --git a/sal-service/src/main/resources/Define_NS_Azure.xml b/sal-service/src/main/resources/Define_NS_Azure.xml index 03ff0ba..3e28c40 100644 --- a/sal-service/src/main/resources/Define_NS_Azure.xml +++ b/sal-service/src/main/resources/Define_NS_Azure.xml @@ -21,8 +21,8 @@ - - + + @@ -78,8 +78,8 @@ def infrastructureParameters = [variables.get("clientId"), variables.get("vmPublicKey"), variables.get("resourceGroup"), variables.get("region"), - variables.get("numberOfInstances"), - variables.get("numberOfNodesPerInstance"), + variables.get("NS_nVMs"), + variables.get("NS_nNodesPerVM"), protocol + "://" + variables.get("rm_host_name") + ":"+ variables.get("pa_port") + "/rest/node.jar", // nodeJarURL variables.get("privateNetworkCIDR"), "true", // staticPublicIP From 457e4056fc9b224e1d07b8b974fc9cece155d30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Fri, 27 Dec 2024 09:35:55 +0100 Subject: [PATCH 9/9] Remove duplicated code in TaskBuilder --- .../ow2/proactive/sal/service/service/TaskBuilder.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java index db97f1a..959fdaa 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java @@ -262,14 +262,6 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) { nodeConfigJson += ", \"vmType\": \"" + deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; break; - case "openstack": - // imageId - nodeConfigJson += "\"image\": \"" + deployment.getNode().getNodeCandidate().getImage().getProviderId() + - "\""; - // vmType - nodeConfigJson += ", \"vmType\": \"" + - deployment.getNode().getNodeCandidate().getHardware().getProviderId() + "\""; - break; case "azure": // imageId nodeConfigJson += "\"image\": \"" + deployment.getNode().getNodeCandidate().getImage().getId() + "\"";