Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Kill running cluster jobs and clean database deployments #109

Merged
merged 2 commits into from
Nov 13, 2024
Merged

Kill running cluster jobs and clean database deployments #109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
15c657f
clean database deployments
ankicabarisic Nov 13, 2024
4896df2
kill job when removing a cluster if active
ankicabarisic Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions sal-service/src/main/java/org/ow2/proactive/sal/service/service/ClusterService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.ow2.proactive.sal.service.util.ByonUtils;
import org.ow2.proactive.sal.service.util.ClusterUtils;
import org.ow2.proactive.scheduler.common.exception.NotConnectedException;
import org.ow2.proactive.scheduler.common.job.JobStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

Expand All @@ -39,17 +40,17 @@ public class ClusterService {
private EdgeService edgeService;

// Define cluster state constants
private static final String STATUS_DEFINED = "defined";
private static final String STATUS_DEFINED = "Defined";

private static final String STATUS_DEPLOYED = "deployed";
private static final String STATUS_DEPLOYED = "Deployed";

private static final String STATUS_RUNNING = "running";
private static final String STATUS_FAILED = "Failed";

private static final String STATUS_FAILED = "failed";
private static final String STATUS_SUBMITTED = "Submitted"; // New status

private static final String STATUS_SUBMITTED = "submitted"; // New status
private static final String STATUS_SCALING = "Scaling";

private static final String STATUS_SCALING = "scaling";
private static final String STATUS_FINISHED = "Finished";

public boolean defineCluster(String sessionId, ClusterDefinition clusterDefinition)
throws NotConnectedException, IOException {
Expand Down Expand Up @@ -213,7 +214,8 @@ public Cluster getCluster(String sessionId, String clusterName) throws NotConnec
i += 1;
node.setNodeUrl(getNodeUrl(sessionId, clusterName, node));
}
if (states.contains("In-Error") || states.contains("Failed") || states.contains("Canceled")) {
if (states.contains(JobStatus.IN_ERROR.toString()) || states.contains(JobStatus.FAILED.toString()) ||
states.contains(JobStatus.CANCELED.toString())) {
getCluster.setStatus(STATUS_FAILED);
} else {
if (checkAllStates(states)) {
Expand Down Expand Up @@ -370,6 +372,8 @@ public boolean deleteCluster(String sessionId, String clusterName) throws NotCon
Cluster toScaleCluster = getCluster(sessionId, clusterName);
for (ClusterNodeDefinition node : toScaleCluster.getNodes()) {
if (node != null) {
//check the node job state

deleteNode(sessionId, clusterName, node, "", false);
}
}
Expand All @@ -383,7 +387,7 @@ private boolean checkAllStates(List<String> states) {
return false;
}
for (String state : states) {
if (!state.equals("Finished")) {
if (!state.equals(STATUS_FINISHED)) {
return false;
}
}
Expand Down Expand Up @@ -429,6 +433,9 @@ private Long deleteNode(String sessionId, String clusterName, ClusterNodeDefinit
// return jobId;
}
Job nodeJob = repositoryService.getJob(node.getNodeJobName(clusterName));
JobState jobState = jobService.getJobState(sessionId, nodeJob.getJobId());
if (jobState.getJobStatus().isJobAlive())
jobService.killJob(sessionId, nodeJob.getJobId());
List<Task> nodeTasks = nodeJob.getTasks();
List<Deployment> nodeDeployments = new ArrayList<>();
for (Task task : nodeTasks) {
Expand Down
1 change: 1 addition & 0 deletions sal-service/src/main/java/org/ow2/proactive/sal/service/service/RepositoryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -948,6 +948,7 @@ public boolean cleanAll(String sessionId) {
success &= performCleanup("Vault Keys", vaultKeyRepository::deleteAll);
success &= performCleanup("ClusterNodeDefinitions", clusterNodeDefRepository::deleteAll);
success &= performCleanup("Clusters", clusterRepository::deleteAll);
success &= performCleanup("Deployments", deploymentRepository::deleteAll);

LOGGER.info("Database cleanup completed with success status: {}", success);
} catch (Exception e) {
Expand Down
Loading