Skip to content

Commit

Permalink
Azure integration (#113)
Browse files Browse the repository at this point in the history 
* Fix missing OS family and arch with Azure

* Add constants in NodeCandidateUtils.java

* Add constants in NodeCandidateUtils.java

* Azure integration

* Fix issue when retrieving Azure images
  • Loading branch information
@mbenguig
mbenguig authored Dec 3, 2024
1 parent ea3a007 commit 424e7eb
Show file tree
Hide file tree
Showing 8 changed files with 114 additions and 100 deletions.
1 change: 1 addition & 0 deletions sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ private Image createImage(JSONObject nodeCandidateJSON, JSONObject imageJSON, PA
arch = osJSON.optBoolean("is64Bit") ? "AMD64" : "i386";
}
} else if (AZURE.equals(nodeCandidateJSON.optString("cloud"))) {
image.setId(imageJSON.optString("id"));
arch = osJSON.optString("arch");
}
os.setOperatingSystemArchitecture(OperatingSystemArchitecture.fromValue(arch));
Expand Down
10 changes: 9 additions & 1 deletion sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,10 +346,18 @@ public List<Image> getCloudImages(String sessionId, String cloudId) throws NotCo
if (paCloud != null) {
try {
JSONArray imagesArray = connectorIaasGateway.getImages(paCloud.getDummyInfrastructureName());

String cloudIdOrEmpty;
if (paCloud.getCloudProviderName().equals("azure")) {
cloudIdOrEmpty = "";
} else {
cloudIdOrEmpty = cloudId + "/";
}
List<String> imagesIDs = IntStream.range(0, imagesArray.length())
.mapToObj(imagesArray::get)
.map(image -> cloudId + "/" + ((JSONObject) image).optString("id"))
.map(image -> cloudIdOrEmpty + ((JSONObject) image).optString("id"))
.collect(Collectors.toList());

LOGGER.debug("Filtering images related to cloud ID '{}'.", cloudId);
allImages.stream().filter(image -> imagesIDs.contains(image.getId())).forEach(filteredImages::add);
} catch (RuntimeException e) {
Expand Down
20 changes: 16 additions & 4 deletions sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,10 +233,22 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De
break;
case "azure":
filename = File.separator + "Define_NS_Azure.xml";
variables.put("azure_username", cloud.getCredentials().getUserName());
variables.put("azure_secret", cloud.getCredentials().getPrivateKey());
variables.put("azure_domain", cloud.getCredentials().getDomain());
variables.put("azure_subscription_id", cloud.getCredentials().getSubscriptionId());
variables.put("clientId", cloud.getCredentials().getUserName());
variables.put("secret", cloud.getCredentials().getPrivateKey());
variables.put("domain", cloud.getCredentials().getDomain());
variables.put("subscriptionId", cloud.getCredentials().getSubscriptionId());
variables.put("image", deployment.getNode().getNodeCandidate().getImage().getId());
variables.put("imageOSType",
deployment.getNode()
.getNodeCandidate()
.getImage()
.getOperatingSystem()
.getOperatingSystemFamily()
.name());
variables.put("vmSizeType", deployment.getNode().getNodeCandidate().getHardware().getProviderId());
variables.put("vmUsername", cloud.getSshCredentials().getUsername());
variables.put("vmPassword", cloud.getSshCredentials().getPrivateKey());
variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName());
break;
default:
throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProviderName());
Expand Down
37 changes: 28 additions & 9 deletions sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import com.google.common.base.Strings;

import lombok.extern.log4j.Log4j2;
import net.bytebuddy.dynamic.scaffold.TypeInitializer;


@Log4j2
Expand All @@ -39,9 +38,6 @@ public class TaskBuilder {
private static final String SCRIPTS_SEPARATION_BASH = NEW_LINE + NEW_LINE + "# Separation script" + NEW_LINE +
NEW_LINE;

private static final String SCRIPTS_SEPARATION_GROOVY = NEW_LINE + NEW_LINE + "// Separation script" + NEW_LINE +
NEW_LINE;

private static final String EMS_DEPLOY_PRE_SCRIPT = "emsdeploy_prescript.sh";

private static final String EMS_DEPLOY_PRIVATE_PRE_SCRIPT = "emsdeploy_prescript_private.sh";
Expand All @@ -58,7 +54,7 @@ public class TaskBuilder {

private static final String CHECK_NODE_SOURCE_REGEXP_SCRIPT = "check_node_source_regexp.groovy";

private static final String ACQUIRE_NODE_AWS_SCRIPT = "acquire_node_aws_script.groovy";
private static final String ACQUIRE_NODE_SCRIPT = "acquire_node_script.groovy";

private static final String REMOVE_NODE_SCRIPT = "remove_node_script.groovy";

Expand Down Expand Up @@ -263,6 +259,9 @@ private String createIAASNodeConfigJson(Task task, Deployment deployment) {
case "openstack":
imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId();
break;
case "azure":
imageId = deployment.getNode().getNodeCandidate().getImage().getId();
break;
default:
imageId = deployment.getNode().getNodeCandidate().getImage().getProviderId();
}
Expand Down Expand Up @@ -320,9 +319,9 @@ private Map<String, TaskVariable> createVariablesMapForAcquiringIAASNode(Task ta
private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix,
String nodeToken) {
LOGGER.debug("Acquiring node AWS script file: " +
getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString());
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() +
taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT);
taskNameSuffix, ACQUIRE_NODE_SCRIPT);

deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));

Expand All @@ -338,9 +337,9 @@ private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, S
private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix,
String nodeToken) {
LOGGER.debug("Acquiring node OS script file: " +
getClass().getResource(File.separator + ACQUIRE_NODE_AWS_SCRIPT).toString());
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() +
taskNameSuffix, ACQUIRE_NODE_AWS_SCRIPT);
taskNameSuffix, ACQUIRE_NODE_SCRIPT);

deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));

Expand All @@ -353,12 +352,32 @@ private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, St
return deployNodeTask;
}

private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix,
String nodeToken) {
LOGGER.debug("Acquiring node Azure script file: " +
getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() +
taskNameSuffix, ACQUIRE_NODE_SCRIPT);

deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));

Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString());
deployNodeTask.setVariables(variablesMap);

addLocalDefaultNSRegexSelectionScript(deployNodeTask);

return deployNodeTask;
}

private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) {
switch (deployment.getPaCloud().getCloudProviderName()) {
case "aws-ec2":
return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken);
case "openstack":
return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken);
case "azure":
return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken);
default:
return new ScriptTask();
}
Expand Down
56 changes: 0 additions & 56 deletions ...n/java/org/ow2/proactive/sal/service/service/infrastructure/PAResourceManagerGateway.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,62 +220,6 @@ private RMStateFull getFullMonitoring() throws NotConnectedException, Permission
return rmState;
}

/**
* Deploy a simple AWS node source
* @param awsUsername A valid AWS user name
* @param awsKey A valid AWS secret key
* @param rmHostname The RM host name
* @param nodeSourceName The name of the node source
* @param numberVMs The number of needed VMs
* @throws NotConnectedException In case the user is not connected
* @throws PermissionRestException In case the user does not have valid permissions
*/
public void deploySimpleAWSNodeSource(String awsUsername, String awsKey, String rmHostname, String nodeSourceName,
Integer numberVMs) throws NotConnectedException, PermissionRestException {
reconnectIfDisconnected();
// Getting NS configuration settings
String infrastructureType = "org.ow2.proactive.resourcemanager.nodesource.infrastructure.AWSEC2Infrastructure";
String[] infrastructureParameters = { awsUsername, //username
awsKey, //secret
numberVMs.toString(), //N of VMs
"1", //N VMs per node
"", //image
"", //OS
"", //awsKeyPair
"", //ram
"", //Ncore
"", //sg
"", //subnet
rmHostname, //host
"http://" + rmHostname + ":8080/connector-iaas", //connector-iaas url
"http://" + rmHostname + ":8080/rest/node.jar", //node jar url
"", "300000", //timeout
"" };
LOGGER.debug("infrastructureParameters: " + Arrays.toString(infrastructureParameters));
String[] infrastructureFileParameters = { "" };
String policyType = "org.ow2.proactive.resourcemanager.nodesource.policy.StaticPolicy";
String[] policyParameters = { "ALL", "ME" };
String[] policyFileParameters = {};
String nodesRecoverable = "true";

LOGGER.debug("Creating NodeSource ...");
rmRestInterface.defineNodeSource(RMConnectionHelper.getSessionId(),
nodeSourceName,
infrastructureType,
infrastructureParameters,
infrastructureFileParameters,
policyType,
policyParameters,
policyFileParameters,
nodesRecoverable);
LOGGER.info("NodeSource created.");

LOGGER.debug("Deploying the NodeSource ...");
rmRestInterface.deployNodeSource(RMConnectionHelper.getSessionId(), nodeSourceName);
LOGGER.info("NodeSource VMs deployed.");

}

/**
* Search the nodes with specific tags.
* @param tags a list of tags which the nodes should contain. When not specified or an empty list, all the nodes known urls are returned
Expand Down
2 changes: 1 addition & 1 deletion sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ private static Task createWorkerNodeTask(String clusterName, ClusterNodeDefiniti
workerNodeTask.setName(workerNode.getNodeTaskName(clusterName));
workerNodeTask.setType(Installation.InstallationType.COMMANDS);
workerNodeTask.setInstallationByType(createWorkerInstallation(envVars));
if (cloud != null && !cloud.getSecurityGroup().isEmpty()) {
if (cloud != null && cloud.getSecurityGroup() != null && !cloud.getSecurityGroup().isEmpty()) {
workerNodeTask.setSecurityGroup(cloud.getSecurityGroup());
}
return workerNodeTask;
Expand Down
Loading

0 comments on commit 424e7eb

Please sign in to comment.