Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle not found exception in task status check #5648

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Handle not found exception in task status check #5648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 25 additions & 11 deletions plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,21 +456,35 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
// if there are no tasks checks the job status
return checkJobStatus()
}

final elapsed = System.currentTimeMillis() - submissionTime
if (elapsed > executor.config.maxStatusDuration.toMillis()) {
throw new ProcessUnrecoverableException("[GOOGLE BATCH] Task status check timed out after ${executor.config.maxStatusDuration} - job=$jobId task=$taskId not found")
}

final now = System.currentTimeMillis()
final delta = now - timestamp;
if( !taskState || delta >= 1_000) {
final status = client.getTaskStatus(jobId, taskId)
final newState = status?.state as String
if( newState ) {
log.trace "[GOOGLE BATCH] Get job=$jobId task=$taskId state=$newState"
taskState = newState
timestamp = now
try {
final status = client.getTaskStatus(jobId, taskId)
final newState = status?.state as String
if( newState ) {
log.trace "[GOOGLE BATCH] Get job=$jobId task=$taskId state=$newState"
taskState = newState
timestamp = now
}
if( newState == 'PENDING' ) {
final eventsCount = status.getStatusEventsCount()
final lastEvent = eventsCount > 0 ? status.getStatusEvents(eventsCount - 1) : null
if( lastEvent?.getDescription()?.contains('CODE_GCE_QUOTA_EXCEEDED') )
log.warn1 "Batch job cannot be run: ${lastEvent.getDescription()}"
}
}
if( newState == 'PENDING' ) {
final eventsCount = status.getStatusEventsCount()
final lastEvent = eventsCount > 0 ? status.getStatusEvents(eventsCount - 1) : null
if( lastEvent?.getDescription()?.contains('CODE_GCE_QUOTA_EXCEEDED') )
log.warn1 "Batch job cannot be run: ${lastEvent.getDescription()}"
catch (NotFoundException e) {
log.trace "[GOOGLE BATCH] Task status not found yet for job=$jobId task=$taskId - returning PENDING state"
taskState = 'PENDING'
timestamp = now
return taskState
}
}
return taskState
Expand Down
4 changes: 3 additions & 1 deletion plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ class BatchConfig {
String getServiceAccountEmail() { serviceAccountEmail }
BatchRetryConfig getRetryConfig() { retryConfig }
List<Integer> getAutoRetryExitCodes() { autoRetryExitCodes }

Duration getMaxStatusDuration() { maxStatusDuration }

static BatchConfig create(Session session) {
final result = new BatchConfig()
result.googleOpts = GoogleOpts.create(session)
Expand All @@ -87,6 +88,7 @@ class BatchConfig {
result.serviceAccountEmail = session.config.navigate('google.batch.serviceAccountEmail')
result.retryConfig = new BatchRetryConfig( session.config.navigate('google.batch.retryPolicy') as Map ?: Map.of() )
result.autoRetryExitCodes = session.config.navigate('google.batch.autoRetryExitCodes', DEFAULT_RETRY_LIST) as List<Integer>
result.maxStatusDuration = session.config.navigate('google.batch.maxStatusDuration', Duration.ofMinutes(5)) as Duration
return result
}

Expand Down