From d50e5b7a22e3ca045b33af113477e133e8140771 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Tue, 9 Jul 2024 11:17:00 +0200 Subject: [PATCH 01/50] Remove buildlogs aws config to not initialize AwsS3Operations (#558) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- src/main/resources/application-buildlogs-aws-test.yml | 6 ------ src/main/resources/application-buildlogs-aws.yml | 7 ------- src/main/resources/application-buildlogs-local.yml | 9 --------- 3 files changed, 22 deletions(-) delete mode 100644 src/main/resources/application-buildlogs-aws.yml diff --git a/src/main/resources/application-buildlogs-aws-test.yml b/src/main/resources/application-buildlogs-aws-test.yml index e93358e3d..b147c2ea1 100644 --- a/src/main/resources/application-buildlogs-aws-test.yml +++ b/src/main/resources/application-buildlogs-aws-test.yml @@ -1,10 +1,4 @@ --- -micronaut: - object-storage: - aws: - build-logs: - bucket: "${wave.build.logs.bucket}" ---- wave: build: logs: diff --git a/src/main/resources/application-buildlogs-aws.yml b/src/main/resources/application-buildlogs-aws.yml deleted file mode 100644 index 767e1cddc..000000000 --- a/src/main/resources/application-buildlogs-aws.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -micronaut: - object-storage: - aws: - build-logs: - bucket: "${wave.build.logs.bucket}" -... diff --git a/src/main/resources/application-buildlogs-local.yml b/src/main/resources/application-buildlogs-local.yml index f6d7a6e2f..53ac81a8c 100644 --- a/src/main/resources/application-buildlogs-local.yml +++ b/src/main/resources/application-buildlogs-local.yml @@ -4,13 +4,4 @@ wave: logs: bucket: "$PWD/build-workspace" prefix: 'wave-build/logs' ---- -# unfortunately "local" object storage requires min Java 17 -# keeping this only for reference -micronaut: - object-storage: - local: - build-logs: - enabled: true - path: "${wave.build.logs.bucket}" ... From c08e4044fed16b3ca57e59cdd9a8b36073fe4b09 Mon Sep 17 00:00:00 2001 From: Justine Geffen Date: Tue, 9 Jul 2024 12:03:36 +0200 Subject: [PATCH 02/50] Update Wave docs to include licensing note (#557) --- docs/index.mdx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/index.mdx b/docs/index.mdx index 0b9fa81cb..25d7045ac 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -10,6 +10,10 @@ It allows for the on-demand assembly, augmentation, and deployment of containeri The Wave container service itself is not a container registry. All containers builds are stored in a Seqera-hosted image registry for a limited time or frozen to a user-specified container registry. +:::note +Wave is available for free as part of Seqera Cloud. As it is open source software, no support is provided by Seqera. For a supported, self-hosted, solution please [contact us](https://seqera.io/contact-us/). +::: + ## Features ### Private container registries @@ -26,7 +30,7 @@ Wave offers a flexible approach to container image management. It allows you to #### An example of Wave augmentation -Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your docker client and the registry. When you request an augmented image, Wave intercepts the process. +Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your Docker client and the registry. When you request an augmented image, Wave intercepts the process. 1. Base image layers download: The Docker client downloads the standard Ubuntu layers from the registry. 2. Custom layer injection: Wave injects your custom layer, denoted by "ω", which could represent application code, libraries, configurations etc. @@ -39,7 +43,7 @@ Imagine you have a base Ubuntu image in a container registry. Wave acts as a pro 1. Streamlined workflows: Wave simplifies your workflow by eliminating the need to manually build and manage custom images. 2. Flexibility: You can easily modify the custom layer for different use cases, allowing for greater adaptability. -### Conda based containers +### Conda-based containers Package management systems such as Conda and Bioconda simplify the installation of scientific software. However, there’s considerable friction when it comes to using those tools to deploy pipelines in cloud environments. Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just declare the Conda packages in your Nextflow pipeline and Wave will assemble the required container. From ec8952227d4ba926096eeb213b9c3012f083f58d Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Tue, 9 Jul 2024 20:22:29 +0200 Subject: [PATCH 03/50] Enable ECR authentication via AWS compute env credentials (#303) This commit implements the ability to authenticate container hosted in AWS ECR registry by using the credentials associated with AWS Batch compute environment. Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/service/ContainerRegistryKeys.groovy | 31 +++++- .../wave/service/CredentialServiceImpl.groovy | 21 +++- .../io/seqera/wave/tower/PlatformId.groovy | 7 +- .../wave/tower/client/TowerClient.groovy | 12 +++ .../wave/tower/compute/ComputeEnv.groovy | 34 +++++++ .../DescribeWorkflowLaunchResponse.groovy | 37 +++++++ .../compute/WorkflowLaunchResponse.groovy | 31 ++++++ .../RegistryCredentialsProviderTest.groovy | 1 + .../service/CredentialsServiceTest.groovy | 97 +++++++++++++++++-- 9 files changed, 259 insertions(+), 12 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy create mode 100644 src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy create mode 100644 src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunchResponse.groovy diff --git a/src/main/groovy/io/seqera/wave/service/ContainerRegistryKeys.groovy b/src/main/groovy/io/seqera/wave/service/ContainerRegistryKeys.groovy index 7e4429bab..b87da03df 100644 --- a/src/main/groovy/io/seqera/wave/service/ContainerRegistryKeys.groovy +++ b/src/main/groovy/io/seqera/wave/service/ContainerRegistryKeys.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service import groovy.json.JsonSlurper import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import io.seqera.wave.util.StringUtils /** @@ -27,19 +28,45 @@ import io.seqera.wave.util.StringUtils * * @author Paolo Di Tommaso */ +@Slf4j @CompileStatic class ContainerRegistryKeys { + /** + * The registry user name + */ String userName + + /** + * The registry secret + */ String password + + /** + * The registry target host - NOTE: this can be null when the keys where obtained by AWS credentials record + */ String registry static ContainerRegistryKeys fromJson(String json) { final root = (Map) new JsonSlurper().parseText(json) - return new ContainerRegistryKeys(userName: root.userName, password: root.password, registry: root.registry) + // parse container registry credentials + if( root.discriminator == 'container-reg' ) { + return new ContainerRegistryKeys(userName: root.userName, password: root.password, registry: root.registry) + } + // Map AWS keys to registry username and password + if( root.discriminator == 'aws' ) { + // AWS keys can have also the `assumeRoleArn`, not clear yet how to handle it + // https://github.com/seqeralabs/platform/blob/64d12c6f3f399f26422a746c0d97cea6d8ddebbb/tower-enterprise/src/main/groovy/io/seqera/tower/domain/aws/AwsSecurityKeys.groovy#L39-L39 + if( root.assumeRoleArn ) { + log.warn "The use of AWS assumeRoleArn for container credentials is not supported - accessKey=${root.accessKey}; assumeRoleArn=${root.assumeRoleArn}" + return null + } + return new ContainerRegistryKeys(userName: root.accessKey, password: root.secretKey) + } + throw new IllegalArgumentException("Unsupported credentials key discriminator type: ${root.discriminator}") } @Override String toString() { - return "ContainerRegistryKeys[registry=$registry; userName=$userName; password=${StringUtils.redact(password)})]" + return "ContainerRegistryKeys[registry=${registry}; userName=${userName}; password=${StringUtils.redact(password)})]" } } diff --git a/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy index d0e0b9e7b..4fcef1eae 100644 --- a/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy @@ -22,9 +22,11 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.tower.crypto.AsymmetricCipher import io.seqera.tower.crypto.EncryptedPacket +import io.seqera.wave.service.aws.AwsEcrService import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.auth.JwtAuth +import io.seqera.wave.tower.client.CredentialsDescription import io.seqera.wave.tower.client.TowerClient import jakarta.inject.Inject import jakarta.inject.Singleton @@ -76,11 +78,14 @@ class CredentialServiceImpl implements CredentialsService { // This cannot be implemented at the moment since, in tower, container registry // credentials are associated to the whole registry final matchingRegistryName = registryName ?: DOCKER_IO - final creds = all.find { + def creds = all.find { it.provider == 'container-reg' && (it.registry ?: DOCKER_IO) == matchingRegistryName } + if (!creds && identity.workflowId && AwsEcrService.isEcrHost(registryName) ) { + creds = findComputeCreds(identity) + } if (!creds) { - log.debug "No credentials matching criteria registryName=$registryName; userId=$identity.userId; workspaceId=$identity.workspaceId; endpoint=$identity.towerEndpoint" + log.debug "No credentials matching criteria registryName=$registryName; userId=$identity.userId; workspaceId=$identity.workspaceId; workflowId=${identity.workflowId}; endpoint=$identity.towerEndpoint" return null } @@ -93,6 +98,18 @@ class CredentialServiceImpl implements CredentialsService { return parsePayload(credentials) } + CredentialsDescription findComputeCreds(PlatformId identity) { + final response = towerClient.describeWorkflowLaunch(identity.towerEndpoint, JwtAuth.of(identity), identity.workflowId) + if( !response ) + return null + final computeEnv = response.get()?.launch?.computeEnv + if( !computeEnv ) + return null + if( computeEnv.platform != 'aws-batch' ) + return null + return new CredentialsDescription(id: computeEnv.credentialsId, provider: 'aws') + } + protected String decryptCredentials(byte[] encodedKey, String payload) { final packet = EncryptedPacket.decode(payload) final cipher = AsymmetricCipher.getInstance() diff --git a/src/main/groovy/io/seqera/wave/tower/PlatformId.groovy b/src/main/groovy/io/seqera/wave/tower/PlatformId.groovy index e5812f500..79af8ea13 100644 --- a/src/main/groovy/io/seqera/wave/tower/PlatformId.groovy +++ b/src/main/groovy/io/seqera/wave/tower/PlatformId.groovy @@ -39,9 +39,10 @@ class PlatformId { final Long workspaceId final String accessToken final String towerEndpoint + final String workflowId boolean asBoolean() { - user!=null || workspaceId!=null || accessToken || towerEndpoint + user!=null || workspaceId!=null || accessToken || towerEndpoint || workflowId } Long getUserId() { @@ -53,7 +54,8 @@ class PlatformId { user, request.towerWorkspaceId, request.towerAccessToken, - request.towerEndpoint ) + request.towerEndpoint, + request.workflowId) } static PlatformId of(User user, ContainerInspectRequest request) { @@ -71,6 +73,7 @@ class PlatformId { ", workspaceId=" + workspaceId + ", accessToken=" + StringUtils.trunc(accessToken,25) + ", towerEndpoint=" + towerEndpoint + + ", workflowId=" + workflowId + ')'; } } diff --git a/src/main/groovy/io/seqera/wave/tower/client/TowerClient.groovy b/src/main/groovy/io/seqera/wave/tower/client/TowerClient.groovy index 354115424..f0aa79654 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/TowerClient.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/TowerClient.groovy @@ -25,9 +25,11 @@ import io.micronaut.cache.annotation.Cacheable import io.micronaut.core.annotation.Nullable import io.seqera.wave.tower.auth.JwtAuth import io.seqera.wave.tower.client.connector.TowerConnector +import io.seqera.wave.tower.compute.DescribeWorkflowLaunchResponse import jakarta.inject.Inject import jakarta.inject.Singleton import org.apache.commons.lang3.StringUtils + /** * Implement a client to interact with Tower services * @@ -112,4 +114,14 @@ class TowerClient { StringUtils.removeEnd(endpoint, "/") } + @Cacheable(value = 'cache-tower-client', atomic = true) + CompletableFuture describeWorkflowLaunch(String towerEndpoint, JwtAuth authorization, String workflowId) { + final uri = workflowLaunchEndpoint(towerEndpoint,workflowId) + return getAsync(uri, towerEndpoint, authorization, DescribeWorkflowLaunchResponse.class) + } + + protected static URI workflowLaunchEndpoint(String towerEndpoint, String workflowId) { + return URI.create("${checkEndpoint(towerEndpoint)}/workflow/${workflowId}/launch") + } + } diff --git a/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy b/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy new file mode 100644 index 000000000..dc2c81d7a --- /dev/null +++ b/src/main/groovy/io/seqera/wave/tower/compute/ComputeEnv.groovy @@ -0,0 +1,34 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.tower.compute + +import groovy.transform.CompileStatic + +/** + * Model the response of compute environment from seqera platform + * + * @author Munish Chouhan + */ +@CompileStatic +class ComputeEnv { + String id + String platform + String credentialsId +} + diff --git a/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy b/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy new file mode 100644 index 000000000..1859796ad --- /dev/null +++ b/src/main/groovy/io/seqera/wave/tower/compute/DescribeWorkflowLaunchResponse.groovy @@ -0,0 +1,37 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.tower.compute + +import groovy.transform.CompileStatic +/** + * Model the response of workflow launch describe request + * + * @author Munish Chouhan + */ +@CompileStatic +class DescribeWorkflowLaunchResponse { + + WorkflowLaunchResponse launch + + DescribeWorkflowLaunchResponse() {} + + DescribeWorkflowLaunchResponse(WorkflowLaunchResponse launch) { + this.launch = launch + } +} diff --git a/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunchResponse.groovy b/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunchResponse.groovy new file mode 100644 index 000000000..a271b0b89 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/tower/compute/WorkflowLaunchResponse.groovy @@ -0,0 +1,31 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.tower.compute + +import groovy.transform.CompileStatic + +/** + * Model the response of workflow launch response from seqera platform + * + * @author Munish Chouhan + */ +@CompileStatic +class WorkflowLaunchResponse { + ComputeEnv computeEnv +} diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryCredentialsProviderTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryCredentialsProviderTest.groovy index ca38eff6c..2c34a9ce8 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryCredentialsProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryCredentialsProviderTest.groovy @@ -106,6 +106,7 @@ class RegistryCredentialsProviderTest extends Specification { def WORKSPACE_ID = 200 def TOWER_TOKEN = "token" def TOWER_ENDPOINT = "localhost:8080" + def WORKFLOW_ID = "id123" and: def credentialService = Mock(CredentialsService) def credentialsFactory = new RegistryCredentialsFactoryImpl(awsEcrService: Mock(AwsEcrService)) diff --git a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy index e55df4766..a2babb87d 100644 --- a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service - import spock.lang.Specification import java.security.PublicKey @@ -38,6 +37,9 @@ import io.seqera.wave.tower.client.CredentialsDescription import io.seqera.wave.tower.client.GetCredentialsKeysResponse import io.seqera.wave.tower.client.ListCredentialsResponse import io.seqera.wave.tower.client.TowerClient +import io.seqera.wave.tower.compute.ComputeEnv +import io.seqera.wave.tower.compute.DescribeWorkflowLaunchResponse +import io.seqera.wave.tower.compute.WorkflowLaunchResponse import jakarta.inject.Inject /** @@ -79,7 +81,7 @@ class CredentialsServiceTest extends Specification { and: 'registry credentials to access a registry stored in tower' def credentialsId = 'credentialsId' - def registryCredentials = '{"userName":"me", "password": "you", "registry": "quay.io"}' + def registryCredentials = '{"userName":"me", "password": "you", "registry": "quay.io", "discriminator":"container-reg"}' def credentialsDescription = new CredentialsDescription( id: credentialsId, provider: 'container-reg', @@ -167,7 +169,7 @@ class CredentialsServiceTest extends Specification { registry: 'docker.io' ) and: - def identity = new PlatformId(new User(id:10), 10,"token",'tower.io') + def identity = new PlatformId(new User(id:10), 10,"token",'tower.io', '101') def auth = JwtAuth.of(identity) when: @@ -187,26 +189,109 @@ class CredentialsServiceTest extends Specification { credentials: [nonContainerRegistryCredentials,otherRegistryCredentials] )) + and:'no compute credentials' + 0 * towerClient.describeWorkflowLaunch('tower.io',auth,'101') >> null + then: credentials == null } - def 'should parse credentials payload' () { given: def svc = new CredentialServiceImpl() when: - def keys = svc.parsePayload('{"registry":"foo.io", "userName":"me", "password": "you"}') + def keys = svc.parsePayload('{"registry":"foo.io", "userName":"me", "password": "you", "discriminator":"container-reg"}') then: keys.registry == 'foo.io' keys.userName == 'me' keys.password == 'you' } + def 'should parse aws keys payload' () { + given: + def svc = new CredentialServiceImpl() + + when: + def keys = svc.parsePayload('{"accessKey":"12345", "secretKey": "67890","discriminator":"aws"}') + then: + keys.userName == '12345' + keys.password == '67890' + keys.registry == null + } + + def 'should get registry creds from compute creds when not found in tower credentials'() { + given: 'a tower user in a workspace on a specific instance with a valid token' + def userId = 10 + def workspaceId = 10 + def token = "valid-token" + def towerEndpoint = "http://tower.io:9090" + def workflowId = "id123" + def registryName = '1000000.dkr.ecr.eu-west-1.amazonaws.com' + + and: 'a previously registered key' + def keypair = TEST_CIPHER.generateKeyPair() + def keyId = 'generated-key-id' + def keyRecord = new PairingRecord( + service: PairingService.TOWER_SERVICE, + endpoint: towerEndpoint, + pairingId: keyId, + privateKey: keypair.getPrivate().getEncoded(), + expiration: (Instant.now() + Duration.ofSeconds(10)) ) + + + and: 'registry credentials to access a registry stored in tower' + def credentialsId = 'credentialsId' + and: 'other credentials registered by the user' + def nonContainerRegistryCredentials = new CredentialsDescription( + id: 'alt-creds', + provider: 'azure', + registry: null ) + and: 'workflow launch info' + def computeEnv = new ComputeEnv( + id: 'computeId', + credentialsId: credentialsId, + platform: 'aws-batch' + ) + def launch = new WorkflowLaunchResponse( + computeEnv: computeEnv + ) + def describeWorkflowLaunchResponse = new DescribeWorkflowLaunchResponse( + launch: launch + ) + and: 'compute credentials' + def computeCredentials = '{"accessKey":"me", "secretKey": "you", "discriminator":"aws"}' + and: + def identity = new PlatformId(new User(id:userId), workspaceId,token,towerEndpoint,workflowId) + def auth = JwtAuth.of(identity) + + when: 'look those registry credentials from tower' + def containerCredentials = credentialsService.findRegistryCreds(registryName,identity) + + then: 'the registered key is fetched correctly from the security service' + 1 * securityService.getPairingRecord(PairingService.TOWER_SERVICE, towerEndpoint) >> keyRecord + + and: 'credentials are listed once and return a potential match' + 1 * towerClient.listCredentials(towerEndpoint,auth,workspaceId) >> CompletableFuture.completedFuture(new ListCredentialsResponse( + credentials: [nonContainerRegistryCredentials])) + + and:'fetched compute credentials' + 1*towerClient.describeWorkflowLaunch(towerEndpoint, auth, workflowId) >> CompletableFuture.completedFuture(describeWorkflowLaunchResponse) + + and: 'they match and the encrypted credentials are fetched' + 1 * towerClient.fetchEncryptedCredentials(towerEndpoint, auth, credentialsId, keyId, workspaceId) >> CompletableFuture.completedFuture( + encryptedCredentialsFromTower(keypair.getPublic(), computeCredentials)) + + and: + containerCredentials.userName == 'me' + containerCredentials.password == "you" + noExceptionThrown() + } private static GetCredentialsKeysResponse encryptedCredentialsFromTower(PublicKey key, String credentials) { - return new GetCredentialsKeysResponse(keys: TEST_CIPHER.encrypt(key,credentials.getBytes()).encode()) + if( credentials ) + return new GetCredentialsKeysResponse(keys: TEST_CIPHER.encrypt(key,credentials.getBytes()).encode()) + return null } } From 0d5413bf5a360ce5486304f12eb1af2fec4eaccc Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 10 Jul 2024 23:26:49 +0200 Subject: [PATCH 04/50] Fix Blob cache config for local dev Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/service/aws/S3ClientFactory.groovy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/aws/S3ClientFactory.groovy b/src/main/groovy/io/seqera/wave/service/aws/S3ClientFactory.groovy index 8b404737a..5327cb7d1 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/S3ClientFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/S3ClientFactory.groovy @@ -24,7 +24,6 @@ import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value import io.seqera.wave.configuration.BlobCacheConfig -import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton import software.amazon.awssdk.auth.credentials.AwsBasicCredentials @@ -42,16 +41,13 @@ import software.amazon.awssdk.services.s3.S3Client @Slf4j class S3ClientFactory { - @Inject - private BlobCacheConfig blobConfig - @Value('${aws.region}') private String awsRegion; @Singleton @Requires(property = 'wave.blobCache.enabled', value = 'true') @Named('BlobS3Client') - S3Client cloudflareS3Client() { + S3Client cloudflareS3Client(BlobCacheConfig blobConfig) { final creds = AwsBasicCredentials.create(blobConfig.storageAccessKey, blobConfig.storageSecretKey) final builder = S3Client.builder() .region(Region.of(blobConfig.storageRegion)) From 0f17ca80baa45aed0317b05925b07e5f88c3e735 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 10 Jul 2024 23:37:35 +0200 Subject: [PATCH 05/50] [release] bump version 1.9.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a7ee35a3e..f8e233b27 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.3 +1.9.0 diff --git a/changelog.txt b/changelog.txt index 6b0429062..2a5e5cf94 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,23 @@ # Wave changelog +1.9.0 - 11 Jul 2024 +- Add Typespec API definitions (#537) [32f7dd16] +- Add cache record-stats (#534) [229926e2] +- Add http 429 error to auth service retry condition [8282a492] +- Check and delete corrupted blobs cache uploads (#533) [b0c775a3] +- Deprecate the support for Spack and remove the support for it in the codebase (#550) [85a05196] +- Enable ECR authentication via AWS compute env credentials (#303) [ec895222] +- Fix Blob cache config for local dev [0d5413bf] +- Fix client cache deadlock (#547) [cc6012ff] +- Fix multiple s3clients in wave (#554) [45689500] +- Minor change in mail notification [0aba6997] +- Refactored metrics service (#549) [5e0d32ac] +- Remove buildlogs aws config to not initialize AwsS3Operations (#558) [d50e5b7a] +- Update metrics response (#536) [b6b36a97] +- Bump buildkit 0.14.0 (#528) [c54172d1] +- Bump buildkit 0.14.1 (#548) [37afb782] +- Bump trivy 0.53.0 [f27ff527] +- Bump version 1.8.3 [55b473e1] + 1.8.3 - 29 Jun 2024 - Fix client cache deadlock (#547) [cc6012ff] - Fix failing test [263b44d3] From b2790ee8bc60e1338fb73d6825770f3168dbdc1c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 12 Jul 2024 19:13:05 +0200 Subject: [PATCH 06/50] Simplify credentials lookup Signed-off-by: Paolo Di Tommaso --- .../auth/RegistryCredentialsProvider.groovy | 26 +++++++++++++++++++ .../wave/core/RegistryProxyService.groovy | 4 +-- .../ContainerInspectServiceImpl.groovy | 12 +++------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProvider.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProvider.groovy index 325c9aa04..1bc6134e5 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProvider.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProvider.groovy @@ -40,6 +40,15 @@ interface RegistryCredentialsProvider { */ RegistryCredentials getDefaultCredentials(String registry) + /** + * Provides the default credentials for the specified container + * + * @param container + * A container name e.g. docker.io/library/ubuntu. + * @return + * A {@link RegistryCredentials} object holding the credentials for the specified container or {@code null} + * if not credentials can be found + */ RegistryCredentials getDefaultCredentials(ContainerPath container) /** @@ -56,4 +65,21 @@ interface RegistryCredentialsProvider { */ RegistryCredentials getUserCredentials(ContainerPath container, PlatformId identity) + /** + * Provides the credentials for the specified container. When the platform identity is provider + * this is equivalent to #getUserCredentials. + * + * @param container + * A container name e.g. docker.io/library/ubuntu. + * @param identity + * The platform identity of the user submitting the request + * @return + * A {@link RegistryCredentials} object holding the credentials for the specified container or {@code null} + * if not credentials can be found + */ + default RegistryCredentials getCredentials(ContainerPath container, PlatformId identity) { + return !identity + ? getDefaultCredentials(container) + : getUserCredentials(container, identity) + } } diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index da1f75db3..3ecd517c1 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -107,9 +107,7 @@ class RegistryProxyService { } protected RegistryCredentials getCredentials(RoutePath route) { - final result = !route.identity - ? credentialsProvider.getDefaultCredentials(route) - : credentialsProvider.getUserCredentials(route, route.identity) + final result = credentialsProvider.getCredentials(route, route.identity) log.debug "Credentials for route path=${route.targetContainer}; identity=${route.identity} => ${result}" return result } diff --git a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy index 862b3bc30..ddf868bcb 100644 --- a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy @@ -105,9 +105,7 @@ class ContainerInspectServiceImpl implements ContainerInspectService { // skip this index host because it has already be added to the list continue } - final creds = !identity - ? credentialsProvider.getDefaultCredentials(path) - : credentialsProvider.getUserCredentials(path, identity) + final creds = credentialsProvider.getCredentials(path, identity) log.debug "Build credentials for repository: $repo => $creds" if( !creds ) { // skip this host because there are no credentials @@ -177,9 +175,7 @@ class ContainerInspectServiceImpl implements ContainerInspectService { else if( item instanceof InspectRepository ) { final path = ContainerCoordinates.parse(item.getImage()) - final creds = !identity - ? credentialsProvider.getDefaultCredentials(path) - : credentialsProvider.getUserCredentials(path, identity) + final creds = credentialsProvider.getCredentials(path, identity) log.debug "Config credentials for repository: ${item.getImage()} => $creds" final entry = fetchConfig0(path, creds).config?.entrypoint @@ -219,9 +215,7 @@ class ContainerInspectServiceImpl implements ContainerInspectService { ContainerSpec containerSpec(String containerImage, PlatformId identity) { final path = ContainerCoordinates.parse(containerImage) - final creds = !identity - ? credentialsProvider.getDefaultCredentials(path) - : credentialsProvider.getUserCredentials(path, identity) + final creds = credentialsProvider.getCredentials(path, identity) log.debug "Inspect credentials for repository: ${containerImage} => $creds" final client = client0(path, creds) From 600fce7a2e1df3f9413d58ff73b9aa61406d0c24 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 16 Jul 2024 09:18:59 +0200 Subject: [PATCH 07/50] Prevent hard error when launch credentials cannot be accessed Signed-off-by: Paolo Di Tommaso --- .../seqera/wave/service/CredentialServiceImpl.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy index 4fcef1eae..8af268b0a 100644 --- a/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/CredentialServiceImpl.groovy @@ -99,6 +99,16 @@ class CredentialServiceImpl implements CredentialsService { } CredentialsDescription findComputeCreds(PlatformId identity) { + try { + return findComputeCreds0(identity) + } + catch (Exception e) { + log.error("Unable to retrieve Platform launch credentials for $identity - cause ${e.message}") + return null + } + } + + protected CredentialsDescription findComputeCreds0(PlatformId identity) { final response = towerClient.describeWorkflowLaunch(identity.towerEndpoint, JwtAuth.of(identity), identity.workflowId) if( !response ) return null From 1b8038759f683c38da5155b7bc9b28056070603b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 16 Jul 2024 10:44:13 +0200 Subject: [PATCH 08/50] Update changelog Signed-off-by: Paolo Di Tommaso --- changelog.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.txt b/changelog.txt index 2a5e5cf94..65c3d6b10 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.9.1 - 11 Jul 2024 +- Prevent hard error when launch credentials cannot be accessed [a318a855] + 1.9.0 - 11 Jul 2024 - Add Typespec API definitions (#537) [32f7dd16] - Add cache record-stats (#534) [229926e2] From 5f045e14ba72f45245e0d9ccf28cfe75e47efe89 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 17 Jul 2024 16:29:30 +0200 Subject: [PATCH 09/50] Enable Cleanup for blob transfer pods (#568) * Added cleanup for transfer pods Signed-off-by: munishchouhan * minor change [ci skip] Signed-off-by: munishchouhan * removed unwanted imports [ci skip] Signed-off-by: munishchouhan * refactored Signed-off-by: munishchouhan * fixed indentation Signed-off-by: munishchouhan * added deletePodWhenReachStatus Signed-off-by: munishchouhan * added tests Signed-off-by: munishchouhan * refactored Signed-off-by: munishchouhan * Update src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy [ci skip] * Update src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy Co-authored-by: Paolo Di Tommaso * refactored Signed-off-by: munishchouhan * Minor change Signed-off-by: Paolo Di Tommaso --------- Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/configuration/BlobCacheConfig.groovy | 3 + .../blob/impl/KubeTransferStrategy.groovy | 31 ++++++- .../seqera/wave/service/k8s/K8sService.groovy | 2 + .../wave/service/k8s/K8sServiceImpl.groovy | 20 +++++ .../blob/impl/KubeTransferStrategyTest.groovy | 82 +++++++++++++++++++ .../service/k8s/K8sServiceImplTest.groovy | 57 +++++++++++++ 6 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index fe33b40d6..efca71c62 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -87,6 +87,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.url-signature-duration:30m}') Duration urlSignatureDuration + @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}') + Duration podDeleteTimeout + Map getEnvironment() { final result = new HashMap(10) if( storageRegion ) { diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy index 6204595e9..ee09c9be0 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy @@ -18,16 +18,22 @@ package io.seqera.wave.service.blob.impl +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService + import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Replaces import io.micronaut.context.annotation.Requires +import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.service.blob.BlobCacheInfo import io.seqera.wave.service.blob.TransferStrategy +import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.k8s.K8sService import jakarta.inject.Inject +import jakarta.inject.Named /** * Implements {@link TransferStrategy} that runs s5cmd using a * Kubernetes job @@ -46,15 +52,24 @@ class KubeTransferStrategy implements TransferStrategy { @Inject private K8sService k8sService + @Inject + private CleanupStrategy cleanup + + @Inject + @Named(TaskExecutors.IO) + private ExecutorService executor + @Override BlobCacheInfo transfer(BlobCacheInfo info, List command) { final podName = podName(info) final pod = k8sService.transferContainer(podName, blobConfig.s5Image, command, blobConfig) final terminated = k8sService.waitPod(pod, blobConfig.transferTimeout.toMillis()) final stdout = k8sService.logsPod(podName) - return terminated + final result = terminated ? info.completed(terminated.exitCode, stdout) : info.failed(stdout) + cleanupPod(podName, terminated.exitCode) + return result } protected String podName(BlobCacheInfo info) { @@ -65,4 +80,18 @@ class KubeTransferStrategy implements TransferStrategy { .putUnencodedChars(info.creationTime.toString()) .hash() } + + private void cleanupPod(String podName, int exitCode) { + if( !cleanup.shouldCleanup(exitCode) ) { + return + } + + CompletableFuture.supplyAsync (() -> + k8sService.deletePodWhenReachStatus( + podName, + 'Succeeded', + blobConfig.podDeleteTimeout.toMillis()), + executor) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index cbea2605f..8398a6089 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -54,4 +54,6 @@ interface K8sService { V1Pod transferContainer(String name, String containerImage, List args, BlobCacheConfig blobConfig) V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) + + void deletePodWhenReachStatus(String podName, String statusName, long timeout) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 01d03dd98..dec8434a7 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -487,6 +487,26 @@ class K8sServiceImpl implements K8sService { .deleteNamespacedPod(name, namespace, (String)null, (String)null, (Integer)null, (Boolean)null, (String)null, (V1DeleteOptions)null) } + /** + * Delete a pod where the status is reached + * + * @param name The name of the pod to be deleted + * @param statusName The status to be reached + * @param timeout The max wait time in milliseconds + */ + @Override + void deletePodWhenReachStatus(String podName, String statusName, long timeout){ + final pod = getPod(podName) + final start = System.currentTimeMillis() + while( (System.currentTimeMillis() - start) < timeout ) { + if( pod?.status?.phase == statusName ) { + deletePod(podName) + return + } + sleep 5_000 + } + } + @Override V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy new file mode 100644 index 000000000..d546baa72 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -0,0 +1,82 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.impl + +import spock.lang.Specification + +import java.time.Duration +import java.util.concurrent.Executors + +import io.kubernetes.client.openapi.models.V1ContainerStateTerminated +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodStatus +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.cleanup.CleanupStrategy +import io.seqera.wave.service.k8s.K8sService +/** + * + * @author Munish Chouhan + */ +class KubeTransferStrategyTest extends Specification { + + K8sService k8sService = Mock(K8sService) + BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10)) + CleanupStrategy cleanup = new CleanupStrategy(buildConfig: new BuildConfig(cleanup: "OnSuccess")) + KubeTransferStrategy strategy = new KubeTransferStrategy(k8sService: k8sService, blobConfig: blobConfig, cleanup: cleanup, executor: Executors.newSingleThreadExecutor()) + + def "transfer should complete successfully with valid inputs"() { + given: + def uri = "s3://bucket/file.txt" + def info = BlobCacheInfo.create(uri, null, null) + def command = ["s5cmd", "cp", uri, "/local/path"] + k8sService.transferContainer(_, blobConfig.s5Image, command, blobConfig) >> new V1Pod(status: new V1PodStatus(phase: 'Succeeded')) + k8sService.getPod(_) >> new V1Pod(status: new V1PodStatus(phase: 'Succeeded')) + k8sService.waitPod(_, _) >> new V1ContainerStateTerminated(exitCode: 0) + k8sService.logsPod(_) >> "Transfer completed" + + when: + def result = strategy.transfer(info, command) + + then: + result.succeeded() + result.exitStatus == 0 + result.logs == "Transfer completed" + result.done() + } + + def "transfer should fail when pod execution exceeds timeout"() { + given: + def uri = "s3://bucket/file.txt" + def info = BlobCacheInfo.create(uri, null, null) + def command = ["s5cmd", "cp", uri, "/local/path"] + k8sService.transferContainer(_, blobConfig.s5Image, command, blobConfig) >> new V1Pod(status: new V1PodStatus(phase: 'Running')) + k8sService.waitPod(_, blobConfig.transferTimeout.toMillis()) >> new V1ContainerStateTerminated(exitCode: 1) + k8sService.logsPod(_) >> "Transfer timeout" + + when: + def result = strategy.transfer(info, command) + + then: + result.failed("Transfer timeout") + result.logs == "Transfer timeout" + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 39e2f585c..f7c064b5c 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -24,7 +24,11 @@ import java.nio.file.Path import java.time.Duration import io.kubernetes.client.custom.Quantity +import io.kubernetes.client.openapi.ApiClient +import io.kubernetes.client.openapi.apis.CoreV1Api import io.kubernetes.client.openapi.models.V1EnvVar +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodStatus import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig @@ -570,4 +574,57 @@ class K8sServiceImplTest extends Specification { cleanup: ctx.close() } + + def "deletePodWhenReachStatus should delete pod when status is reached within timeout"() { + given: + def podName = "test-pod" + def statusName = "Succeeded" + def timeout = 5000 + def api = Mock(CoreV1Api) + api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: statusName)) + def k8sClient = new K8sClient() { + @Override + ApiClient apiClient() { + return null + } + CoreV1Api coreV1Api() { + return api + } + } + + def k8sService = new K8sServiceImpl(k8sClient: k8sClient) + + when: + k8sService.deletePodWhenReachStatus(podName, statusName, timeout) + + then: + 1 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) + } + + def "deletePodWhenReachStatus should not delete pod if status is not reached within timeout"() { + given: + def podName = "test-pod" + def statusName = "Succeeded" + def timeout = 5000 + def api = Mock(CoreV1Api) + api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: "Running")) + def k8sClient = new K8sClient() { + @Override + ApiClient apiClient() { + return null + } + CoreV1Api coreV1Api() { + return api + } + } + + def k8sService = new K8sServiceImpl(k8sClient: k8sClient) + + when: + k8sService.deletePodWhenReachStatus(podName, statusName, timeout) + + then: + 0 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) + } + } From c118f0de2f29fd34c333a8e0891084077ab6f3c3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 17 Jul 2024 18:52:43 +0200 Subject: [PATCH 10/50] Bump wave-utils@0.13.1 Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4d3be2206..622c954b8 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") api 'io.seqera:lib-mail:1.0.0' api 'io.seqera:wave-api:0.10.0' - api 'io.seqera:wave-utils:0.12.0' + api 'io.seqera:wave-utils:0.13.1' implementation("io.micronaut:micronaut-http-client") implementation("io.micronaut:micronaut-jackson-databind") From 760f2173989208831e18ef9ea29a06ecb8868bd7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 22 Jul 2024 12:41:11 +0200 Subject: [PATCH 11/50] Throw an error on missing credentials for custom build repo (#564) Signed-off-by: Paolo Di Tommaso --- .../ContainerInspectServiceImpl.groovy | 12 +++- .../ContainerControllerHttpTest.groovy | 58 ++----------------- .../builder/ContainerBuildServiceTest.groovy | 11 +++- 3 files changed, 26 insertions(+), 55 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy index ddf868bcb..be6b738e3 100644 --- a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy @@ -33,6 +33,7 @@ import io.seqera.wave.core.ContainerPath import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.core.spec.ConfigSpec import io.seqera.wave.core.spec.ContainerSpec +import io.seqera.wave.exception.BadRequestException import io.seqera.wave.http.HttpClientFactory import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.proxy.ProxyClient @@ -91,7 +92,16 @@ class ContainerInspectServiceImpl implements ContainerInspectService { repos.add(buildRepo) if( cacheRepo ) repos.add(cacheRepo) - return credsJson(repos, identity) + final result = credsJson(repos, identity) + if( buildRepo && !result.contains(host0(buildRepo)) ) + throw new BadRequestException("Missing credentials for target build repository: $buildRepo") + if( cacheRepo && !result.contains(host0(cacheRepo)) ) + throw new BadRequestException("Missing credentials for target cache repository: $buildRepo") + return result + } + + static protected String host0(String repo) { + repo.tokenize('/')[0] } protected String credsJson(Set repositories, PlatformId identity) { diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy index 411101c33..85bb245db 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy @@ -42,7 +42,6 @@ import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.PairingServiceImpl import io.seqera.wave.tower.User import io.seqera.wave.tower.auth.JwtAuth -import io.seqera.wave.tower.client.ListCredentialsResponse import io.seqera.wave.tower.client.TowerClient import io.seqera.wave.tower.client.UserInfoResponse import jakarta.inject.Inject @@ -307,29 +306,14 @@ class ContainerControllerHttpTest extends Specification { } def 'should get the correct image name with imageSuffix name strategy'(){ - given: - def endpoint = 'http://cloud.seqera.io' - def token = 'foo' - def refresh = 'foo2' - def auth = JwtAuth.of(endpoint, token, refresh) - and: - pairingService.getPairingRecord(TOWER_SERVICE, endpoint) >> { new PairingRecord('tower', endpoint) } - towerClient.userInfo(endpoint, auth) >> CompletableFuture.completedFuture(new UserInfoResponse(user:new User(id:1))) - towerClient.listCredentials(_,_,_) >> CompletableFuture.completedFuture(new ListCredentialsResponse(credentials:[])) - when: - def cfg = new ContainerConfig(workingDir: '/foo') def packages = new PackagesSpec(channels: ['conda-forge', 'bioconda'], entries: ['salmon'], type: 'CONDA') SubmitContainerTokenRequest request = new SubmitContainerTokenRequest( - towerAccessToken: token, - towerRefreshToken: refresh, - towerEndpoint: endpoint, - towerWorkspaceId: 10, nameStrategy: "imageSuffix", packages: packages, freeze: true, - buildRepository: "registry/repository") + buildRepository: "docker.io/foo/test") and: def response = httpClient .toBlocking() @@ -337,33 +321,18 @@ class ContainerControllerHttpTest extends Specification { .body() then: - response.targetImage.startsWith("registry/repository/salmon") + response.targetImage.startsWith("docker.io/foo/test/salmon") } def 'should get the correct image name with tagPrefix name strategy'(){ - given: - def endpoint = 'http://cloud.seqera.io' - def token = 'foo' - def refresh = 'foo2' - def auth = JwtAuth.of(endpoint, token, refresh) - and: - pairingService.getPairingRecord(TOWER_SERVICE, endpoint) >> { new PairingRecord('tower', endpoint) } - towerClient.userInfo(endpoint, auth) >> CompletableFuture.completedFuture(new UserInfoResponse(user:new User(id:1))) - towerClient.listCredentials(_,_,_) >> CompletableFuture.completedFuture(new ListCredentialsResponse(credentials:[])) - when: - def cfg = new ContainerConfig(workingDir: '/foo') def packages = new PackagesSpec(channels: ['conda-forge', 'bioconda'], entries: ['salmon'], type: 'CONDA') SubmitContainerTokenRequest request = new SubmitContainerTokenRequest( - towerAccessToken: token, - towerRefreshToken: refresh, - towerEndpoint: endpoint, - towerWorkspaceId: 10, nameStrategy: "tagPrefix", packages: packages, freeze: true, - buildRepository: "registry/repository") + buildRepository: "docker.io/foo/test") and: def response = httpClient .toBlocking() @@ -371,32 +340,17 @@ class ContainerControllerHttpTest extends Specification { .body() then: - response.targetImage.startsWith("registry/repository:salmon") + response.targetImage.startsWith("docker.io/foo/test:salmon") } def 'should get the correct image name with default name strategy'(){ - given: - def endpoint = 'http://cloud.seqera.io' - def token = 'foo' - def refresh = 'foo2' - def auth = JwtAuth.of(endpoint, token, refresh) - and: - pairingService.getPairingRecord(TOWER_SERVICE, endpoint) >> { new PairingRecord('tower', endpoint) } - towerClient.userInfo(endpoint, auth) >> CompletableFuture.completedFuture(new UserInfoResponse(user:new User(id:1))) - towerClient.listCredentials(_,_,_) >> CompletableFuture.completedFuture(new ListCredentialsResponse(credentials:[])) - when: - def cfg = new ContainerConfig(workingDir: '/foo') def packages = new PackagesSpec(channels: ['conda-forge', 'bioconda'], entries: ['salmon'], type: 'CONDA') SubmitContainerTokenRequest request = new SubmitContainerTokenRequest( - towerAccessToken: token, - towerRefreshToken: refresh, - towerEndpoint: endpoint, - towerWorkspaceId: 10, packages: packages, freeze: true, - buildRepository: "registry/repository") + buildRepository: "docker.io/foo/test") and: def response = httpClient .toBlocking() @@ -404,6 +358,6 @@ class ContainerControllerHttpTest extends Specification { .body() then: - response.targetImage.startsWith("registry/repository:salmon") + response.targetImage.startsWith("docker.io/foo/test:salmon") } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 72cd7abdc..8002de2b3 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -93,8 +93,6 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai when: def result = service.launch(req) - and: - println result.logs then: result.id result.startTime @@ -589,4 +587,13 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai record2.buildId == request.buildId record2.digest == 'abc123' } + + def 'should return only the host name' () { + expect: + ContainerInspectServiceImpl.host0(CONTAINER) == EXPECTED + where: + CONTAINER | EXPECTED + 'docker.io' | 'docker.io' + 'docker.io/foo/'| 'docker.io' + } } From 273f4b21cee86a51630a57562782c6e1956cdbe5 Mon Sep 17 00:00:00 2001 From: Justine Geffen Date: Wed, 24 Jul 2024 10:29:19 +0200 Subject: [PATCH 12/50] Updated Wave license note (#570) Co-authored-by: Munish Chouhan Co-authored-by: Paolo Di Tommaso --- docs/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.mdx b/docs/index.mdx index 25d7045ac..8695b60f1 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -11,7 +11,7 @@ It allows for the on-demand assembly, augmentation, and deployment of containeri The Wave container service itself is not a container registry. All containers builds are stored in a Seqera-hosted image registry for a limited time or frozen to a user-specified container registry. :::note -Wave is available for free as part of Seqera Cloud. As it is open source software, no support is provided by Seqera. For a supported, self-hosted, solution please [contact us](https://seqera.io/contact-us/). +Wave is also available as hosted service on [Seqera Platform](https://cloud.seqera.io/). For Seqera Enterprise customers, a licensed self-hosted Wave solution is also available. Contact us [contact us](https://seqera.io/contact-us/) for more information. ::: ## Features From 44819b48c5f6a5efd0649076c4e6582c5d82c282 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 24 Jul 2024 15:59:46 +0200 Subject: [PATCH 13/50] Use k8s job for blob cache transfer (#479) This commit implements the use of K8s job to carry out the run of container image blob cache uploads. Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/configuration/BlobCacheConfig.groovy | 3 + .../blob/impl/BlobCacheServiceImpl.groovy | 6 +- .../blob/impl/KubeTransferStrategy.groovy | 44 ++++-- .../service/builder/KubeBuildStrategy.groovy | 12 +- .../seqera/wave/service/k8s/K8sService.groovy | 13 +- .../wave/service/k8s/K8sServiceImpl.groovy | 131 +++++++++++++----- .../wave/service/scan/KubeScanStrategy.groovy | 8 +- .../io/seqera/wave/util/K8sHelper.groovy | 12 ++ .../impl/DockerTransferStrategyTest.groovy | 2 +- .../blob/impl/KubeTransferStrategyTest.groovy | 58 +++++--- .../service/k8s/K8sServiceImplTest.groovy | 61 ++++---- .../io/seqera/wave/util/K8sHelperTest.groovy | 29 ++++ 12 files changed, 265 insertions(+), 114 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index efca71c62..97748a23c 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -87,6 +87,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.url-signature-duration:30m}') Duration urlSignatureDuration + @Value('${wave.blobCache.retryAttempts:3}') + Integer retryAttempts + @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}') Duration podDeleteTimeout diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 6f8f91b05..75161bd99 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -295,8 +295,10 @@ class BlobCacheServiceImpl implements BlobCacheService { static BlobCacheInfo awaitCompletion(BlobStore store, String key, BlobCacheInfo current) { final beg = System.currentTimeMillis() - // add 10% delay gap to prevent race condition with timeout expiration - final max = (store.timeout.toMillis() * 1.10) as long + // set the await timeout nealy double as the blob transfer timeout, this because the + // transfer pod can spend `timeout` time in pending status awaiting to be scheduled + // and the same `timeout` time amount carrying out the transfer (upload) operation + final max = (store.timeout.toMillis() * 2.10) as long while( true ) { if( current==null ) { return BlobCacheInfo.unknown() diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy index ee09c9be0..69afe08dd 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy @@ -32,6 +32,7 @@ import io.seqera.wave.service.blob.BlobCacheInfo import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.k8s.K8sService +import io.seqera.wave.util.K8sHelper import jakarta.inject.Inject import jakarta.inject.Named /** @@ -61,18 +62,43 @@ class KubeTransferStrategy implements TransferStrategy { @Override BlobCacheInfo transfer(BlobCacheInfo info, List command) { - final podName = podName(info) - final pod = k8sService.transferContainer(podName, blobConfig.s5Image, command, blobConfig) - final terminated = k8sService.waitPod(pod, blobConfig.transferTimeout.toMillis()) - final stdout = k8sService.logsPod(podName) - final result = terminated - ? info.completed(terminated.exitCode, stdout) - : info.failed(stdout) - cleanupPod(podName, terminated.exitCode) + final jobName = getJobName(info) + // run the transfer job + final result = transfer0(info, command, jobName) + // delete job + cleanupJob(jobName, result.exitStatus) return result } - protected String podName(BlobCacheInfo info) { + protected BlobCacheInfo transfer0(BlobCacheInfo info, List command, String jobName) { + final job = k8sService.transferJob(jobName, blobConfig.s5Image, command, blobConfig) + final timeout = Math.round(blobConfig.transferTimeout.toMillis() *1.1f) + final podList = k8sService.waitJob(job, timeout) + final size = podList?.items?.size() ?: 0 + // verify the upload pod has been created + if( size < 1 ) { + log.error "== Blob cache transfer failed - unable to schedule upload job: $info" + return info.failed("Unable to scheduler transfer job") + } + // Find the latest created pod among the pods associated with the job + final latestPod = K8sHelper.findLatestPod(podList) + + final pod = k8sService.getPod(latestPod.metadata.name) + final exitCode = k8sService.waitPodCompletion(pod, timeout) + final stdout = k8sService.logsPod(pod) + + return exitCode!=null + ? info.completed(exitCode, stdout) + : info.failed(stdout) + } + + protected void cleanupJob(String jobName, Integer exitCode) { + if( cleanup.shouldCleanup(exitCode) ) { + CompletableFuture.supplyAsync (() -> k8sService.deleteJob(jobName), executor) + } + } + + protected static String getJobName(BlobCacheInfo info) { return 'transfer-' + Hashing .sipHash24() .newHasher() diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index a5bf97c7d..6f79aa5aa 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -44,7 +44,7 @@ import static java.nio.file.StandardOpenOption.WRITE import static java.nio.file.attribute.PosixFilePermission.OWNER_READ import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE /** - * Build a container image using running a K8s job + * Build a container image using running a K8s pod * * @author Paolo Di Tommaso */ @@ -100,11 +100,11 @@ class KubeBuildStrategy extends BuildStrategy { final selector= getSelectorLabel(req.platform, nodeSelectorMap) final spackCfg0 = req.isSpackBuild ? spackConfig : null final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, spackCfg0, selector) - final terminated = k8sService.waitPod(pod, buildConfig.buildTimeout.toMillis()) - final stdout = k8sService.logsPod(name) - if( terminated ) { - final digest = terminated.exitCode==0 ? proxyService.getImageDigest(req, true) : null - return BuildResult.completed(req.buildId, terminated.exitCode, stdout, req.startTime, digest) + final exitCode = k8sService.waitPodCompletion(pod, buildConfig.buildTimeout.toMillis()) + final stdout = k8sService.logsPod(pod) + if( exitCode!=null ) { + final digest = exitCode==0 ? proxyService.getImageDigest(req, true) : null + return BuildResult.completed(req.buildId, exitCode, stdout, req.startTime, digest) } else { return BuildResult.failed(req.buildId, stdout, req.startTime) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 8398a6089..915f5f313 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -20,9 +20,9 @@ package io.seqera.wave.service.k8s import java.nio.file.Path -import io.kubernetes.client.openapi.models.V1ContainerStateTerminated import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.SpackConfig @@ -43,7 +43,7 @@ interface K8sService { V1Pod getPod(String name) - String logsPod(String name) + String logsPod(V1Pod pod) void deletePod(String name) @@ -51,9 +51,14 @@ interface K8sService { V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) - V1Pod transferContainer(String name, String containerImage, List args, BlobCacheConfig blobConfig) + Integer waitPodCompletion(V1Pod pod, long timeout) - V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) + void deleteJob(String name) + + V1Job transferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) + + V1PodList waitJob(V1Job job, Long timeout) void deletePodWhenReachStatus(String podName, String statusName, long timeout) + } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index dec8434a7..d4dc5d130 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.custom.Quantity import io.kubernetes.client.openapi.models.V1ContainerBuilder -import io.kubernetes.client.openapi.models.V1ContainerStateTerminated import io.kubernetes.client.openapi.models.V1DeleteOptions import io.kubernetes.client.openapi.models.V1EnvVar import io.kubernetes.client.openapi.models.V1HostPathVolumeSource @@ -35,6 +34,7 @@ import io.kubernetes.client.openapi.models.V1JobBuilder import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder +import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1ResourceRequirements import io.kubernetes.client.openapi.models.V1Volume import io.kubernetes.client.openapi.models.V1VolumeMount @@ -419,31 +419,32 @@ class K8sServiceImpl implements K8sService { } /** - * Wait for a pod a completion + * Wait for a pod a completion. + * + * NOTE: this method assumes the pod is running exactly *one* container. * * @param pod * The pod name * @param timeout * Max wait time in milliseconds * @return - * An instance of {@link V1ContainerStateTerminated} representing the termination state - * or {@code null} if the state cannot be determined or timeout was reached, + * An Integer value representing the container exit code or {@code null} if the state cannot be determined + * or timeout was reached. */ @Override - V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) { - final name = pod.metadata.name + Integer waitPodCompletion(V1Pod pod, long timeout) { final start = System.currentTimeMillis() // wait for termination while( true ) { final phase = pod.status?.phase if( phase && phase != 'Pending' ) { - final status = pod.status.containerStatuses.find( it -> it.name==name ) + final status = pod.status.containerStatuses.first() if( !status ) return null if( !status.state ) return null if( status.state.terminated ) { - return status.state.terminated + return status.state.terminated.exitCode } } @@ -453,24 +454,26 @@ class K8sServiceImpl implements K8sService { if( delta > timeout ) return null sleep 5_000 - pod = getPod(name) + pod = getPod(pod.metadata.name) } } /** - * Fetch the logs of a pod + * Fetch the logs of a pod. * - * @param name The pod name + * NOTE: this method assume the pod runs exactly *one* container. + * + * @param name The {@link V1Pod} object representing the pod from where retrieve the logs * @return The logs as a string or when logs are not available or cannot be accessed */ @Override - String logsPod(String name) { + String logsPod(V1Pod pod) { try { final logs = k8sClient.podLogs() - logs.streamNamespacedPodLog(namespace, name, name).getText() + logs.streamNamespacedPodLog(namespace, pod.metadata.name, pod.spec.containers.first().name).getText() } catch (Exception e) { - log.error "Unable to fetch logs for pod: $name", e + log.error "Unable to fetch logs for pod: ${pod.metadata.name}", e return null } } @@ -566,49 +569,67 @@ class K8sServiceImpl implements K8sService { builder.build() } + /** + * Create a Job for blob transfer + * + * @param name + * The name of job and container + * @param containerImage + * The container image to be used + * @param args + * The transfer command to be performed + * @param blobConfig + * The config to be used for transfer + * @return + * The {@link V1Job} description the submitted job + */ @Override - V1Pod transferContainer(String name, String containerImage, List args, BlobCacheConfig blobConfig) { - final spec = transferSpec(name, containerImage, args, blobConfig) + V1Job transferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) { + final spec = createTransferJobSpec(name, containerImage, args, blobConfig) + return k8sClient - .coreV1Api() - .createNamespacedPod(namespace, spec, null, null, null,null) + .batchV1Api() + .createNamespacedJob(namespace, spec, null, null, null,null) } - V1Pod transferSpec(String name, String containerImage, List args, BlobCacheConfig blobConfig) { + V1Job createTransferJobSpec(String name, String containerImage, List args, BlobCacheConfig blobConfig) { - V1PodBuilder builder = new V1PodBuilder() + V1JobBuilder builder = new V1JobBuilder() //metadata section builder.withNewMetadata() .withNamespace(namespace) .withName(name) - .addToLabels(labels) + .withLabels(labels) .endMetadata() - //spec section - def spec = builder - .withNewSpec() - .withServiceAccount(serviceAccount) - .withActiveDeadlineSeconds( blobConfig.transferTimeout.toSeconds() ) - .withRestartPolicy("Never") - final requests = new V1ResourceRequirements() if( blobConfig.requestsCpu ) requests.putRequestsItem('cpu', new Quantity(blobConfig.requestsCpu)) if( blobConfig.requestsMemory ) requests.putRequestsItem('memory', new Quantity(blobConfig.requestsMemory)) + //spec section + def spec = builder.withNewSpec() + .withBackoffLimit(blobConfig.retryAttempts) + .withNewTemplate() + .editOrNewSpec() + .withServiceAccount(serviceAccount) + .withActiveDeadlineSeconds(blobConfig.transferTimeout.toSeconds()) + .withRestartPolicy("Never") //container section - spec.addNewContainer() - .withName(name) - .withImage(containerImage) - .withEnv(toEnvList(blobConfig.getEnvironment())) - .withArgs(args) - .withResources(requests) - .endContainer() + .addNewContainer() + .withName(name) + .withImage(containerImage) + .withArgs(args) + .withResources(requests) + .withEnv(toEnvList(blobConfig.getEnvironment())) + .endContainer() + .endSpec() + .endTemplate() .endSpec() - builder.build() + return spec.build() } protected List toEnvList(Map env) { @@ -618,4 +639,42 @@ class K8sServiceImpl implements K8sService { return result } + /** + * Wait for a job to complete + * + * @param k8s job + * @param timeout + * Max wait time in milliseconds + * @return list of pods created by the job + */ + @Override + V1PodList waitJob(V1Job job, Long timeout) { + sleep 5_000 + final startTime = System.currentTimeMillis() + // wait for termination + while (System.currentTimeMillis() - startTime < timeout) { + final name = job.metadata.name + final status = getJobStatus(name) + if (status != JobStatus.Pending) { + return k8sClient + .coreV1Api() + .listNamespacedPod(namespace, null, null, null, null, "job-name=$name", null, null, null, null, null, null) + } + job = getJob(name) + } + return null + } + + /** + * Delete a job + * + * @param name, name of the job to be deleted + */ + @Override + void deleteJob(String name) { + k8sClient + .batchV1Api() + .deleteNamespacedJob(name, namespace, null, null, null, null,"Background", null) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index c17b9bb85..2490f14ed 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -28,7 +28,6 @@ import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.openapi.ApiException -import io.kubernetes.client.openapi.models.V1Job import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires @@ -89,17 +88,16 @@ class KubeScanStrategy extends ScanStrategy { final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) - V1Job job final trivyCommand = scanCommand(req.targetImage, reportFile, scanConfig) final selector= getSelectorLabel(req.platform, nodeSelectorMap) final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, configFile, scanConfig, selector) - final terminated = k8sService.waitPod(pod, scanConfig.timeout.toMillis()) - if( terminated ) { + final exitCode = k8sService.waitPodCompletion(pod, scanConfig.timeout.toMillis()) + if( exitCode==0 ) { log.info("Container scan completed for id: ${req.id}") return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) } else{ - final stdout = k8sService.logsPod(podName) + final stdout = k8sService.logsPod(pod) log.info("Container scan failed for scan id: ${req.id} - stdout: $stdout") return ScanResult.failure(req, startTime) } diff --git a/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy b/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy index 7a2f82a73..336890e79 100644 --- a/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy @@ -19,6 +19,8 @@ package io.seqera.wave.util import groovy.transform.CompileStatic +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException @@ -73,4 +75,14 @@ class K8sHelper { return Map.of(parts[0], parts[1]) } + static V1Pod findLatestPod(V1PodList allPods) { + // Find the latest created pod among the pods associated with the job + def latest = allPods.getItems().get(0) + for (def pod : allPods.items) { + if (pod.metadata?.creationTimestamp?.isAfter(latest.metadata.creationTimestamp)) { + latest = pod + } + } + return latest + } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy index 501b1e792..adf5582f7 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy @@ -39,10 +39,10 @@ class DockerTransferStrategyTest extends Specification { s5Image: 'cr.seqera.io/public/s5cmd:latest' ) def strategy = new DockerTransferStrategy(blobConfig: config) - and: when: def result = strategy.createProcess(['s5cmd', 'run', '--this']) + then: result.command() == [ 'docker', diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy index d546baa72..33f8320d9 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -21,10 +21,12 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification import java.time.Duration +import java.time.OffsetDateTime import java.util.concurrent.Executors -import io.kubernetes.client.openapi.models.V1ContainerStateTerminated +import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1PodStatus import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig @@ -38,45 +40,57 @@ import io.seqera.wave.service.k8s.K8sService class KubeTransferStrategyTest extends Specification { K8sService k8sService = Mock(K8sService) - BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10)) + BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10), retryAttempts: 3) CleanupStrategy cleanup = new CleanupStrategy(buildConfig: new BuildConfig(cleanup: "OnSuccess")) KubeTransferStrategy strategy = new KubeTransferStrategy(k8sService: k8sService, blobConfig: blobConfig, cleanup: cleanup, executor: Executors.newSingleThreadExecutor()) - def "transfer should complete successfully with valid inputs"() { + def "transfer should return completed info when job is successful"() { given: - def uri = "s3://bucket/file.txt" - def info = BlobCacheInfo.create(uri, null, null) - def command = ["s5cmd", "cp", uri, "/local/path"] - k8sService.transferContainer(_, blobConfig.s5Image, command, blobConfig) >> new V1Pod(status: new V1PodStatus(phase: 'Succeeded')) - k8sService.getPod(_) >> new V1Pod(status: new V1PodStatus(phase: 'Succeeded')) - k8sService.waitPod(_, _) >> new V1ContainerStateTerminated(exitCode: 0) - k8sService.logsPod(_) >> "Transfer completed" + def info = BlobCacheInfo.create("https://test.com/blobs", null, null) + def command = ["transfer", "blob"] + final jobName = "job-123" + def podName = "$jobName-abc" + def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) + pod.status = new V1PodStatus(phase: "Succeeded") + def podList = new V1PodList(items: [pod]) + k8sService.transferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) + k8sService.waitJob(_, _) >> podList + k8sService.getPod(_) >> pod + k8sService.waitPodCompletion(_, _) >> 0 + k8sService.logsPod(_) >> "transfer successful" when: def result = strategy.transfer(info, command) then: - result.succeeded() result.exitStatus == 0 - result.logs == "Transfer completed" + result.logs == "transfer successful" result.done() + result.succeeded() } - def "transfer should fail when pod execution exceeds timeout"() { + def "transfer should return failed info when job is failed"() { given: - def uri = "s3://bucket/file.txt" - def info = BlobCacheInfo.create(uri, null, null) - def command = ["s5cmd", "cp", uri, "/local/path"] - k8sService.transferContainer(_, blobConfig.s5Image, command, blobConfig) >> new V1Pod(status: new V1PodStatus(phase: 'Running')) - k8sService.waitPod(_, blobConfig.transferTimeout.toMillis()) >> new V1ContainerStateTerminated(exitCode: 1) - k8sService.logsPod(_) >> "Transfer timeout" + def info = BlobCacheInfo.create("https://test.com/blobs", null, null) + def command = ["transfer", "blob"] + final jobName = "job-123" + def podName = "$jobName-abc" + def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) + pod.status = new V1PodStatus(phase: "Succeeded") + def podList = new V1PodList(items: [pod]) + k8sService.transferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) + k8sService.waitJob(_, _) >> podList + k8sService.getPod(_) >> pod + k8sService.waitPodCompletion(pod, _) >> 1 + k8sService.logsPod(pod) >> "transfer failed" when: def result = strategy.transfer(info, command) then: - result.failed("Transfer timeout") - result.logs == "Transfer timeout" + result.exitStatus == 1 + result.logs == "transfer failed" + result.done() + !result.succeeded() } - } diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index f7c064b5c..d0752f39f 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -498,7 +498,7 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def 'should create transfer spec with defaults' () { + def 'should create transfer job spec with defaults' () { given: def PROPS = [ 'wave.build.workspace': '/build/work', @@ -510,30 +510,32 @@ class K8sServiceImplTest extends Specification { def config = Mock(BlobCacheConfig) { getTransferTimeout() >> Duration.ofSeconds(20) getEnvironment() >> [:] + getRetryAttempts() >> 5 } when: - def result = k8sService.transferSpec('foo', 'my-image:latest', ['this','that'], config) + def result = k8sService.createTransferJobSpec('foo', 'my-image:latest', ['this','that'], config) + result then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' and: - result.spec.activeDeadlineSeconds == 20 - result.spec.serviceAccount == null - and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - and: - !result.spec.containers.get(0).getEnv() - !result.spec.containers.get(0).getResources().limits - !result.spec.containers.get(0).getResources().requests - - cleanup: + result.spec.backoffLimit == 5 + and: + verifyAll(result.spec.template.spec) { + activeDeadlineSeconds == 20 + serviceAccount == null + containers.get(0).name == 'foo' + containers.get(0).image == 'my-image:latest' + containers.get(0).args == ['this','that'] + !containers.get(0).getEnv() + !containers.get(0).getResources().limits + !containers.get(0).getResources().requests + } ctx.close() } - def 'should create transfer spec with custom settings' () { + def 'should create transfer job spec with custom settings' () { given: def PROPS = [ 'wave.build.workspace': '/build/work', @@ -548,28 +550,29 @@ class K8sServiceImplTest extends Specification { getEnvironment() >> ['FOO':'one', 'BAR':'two'] getRequestsCpu() >> '2' getRequestsMemory() >> '8Gi' + getRetryAttempts() >> 3 } when: - def result = k8sService.transferSpec('foo', 'my-image:latest', ['this','that'], config) + def result = k8sService.createTransferJobSpec('foo', 'my-image:latest', ['this','that'], config) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' and: - result.spec.activeDeadlineSeconds == 20 - result.spec.serviceAccount == 'foo-sa' - and: - verifyAll(result.spec.containers.get(0)) { - name == 'foo' - image == 'my-image:latest' - args == ['this', 'that'] - getEnv().get(0) == new V1EnvVar().name('FOO').value('one') - getEnv().get(1) == new V1EnvVar().name('BAR').value('two') - getResources().requests.get('cpu') == new Quantity('2') - getResources().requests.get('memory') == new Quantity('8Gi') + result.spec.backoffLimit == 3 + and: + verifyAll(result.spec.template.spec) { + activeDeadlineSeconds == 20 + serviceAccount == 'foo-sa' + containers.get(0).name == 'foo' + containers.get(0).image == 'my-image:latest' + containers.get(0).args == ['this','that'] + containers.get(0).getEnv().get(0) == new V1EnvVar().name('FOO').value('one') + containers.get(0).getEnv().get(1) == new V1EnvVar().name('BAR').value('two') + containers.get(0).getResources().requests.get('cpu') == new Quantity('2') + containers.get(0).getResources().requests.get('memory') == new Quantity('8Gi') + !containers.get(0).getResources().limits } - and: - !result.spec.containers.get(0).getResources().limits cleanup: ctx.close() diff --git a/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy index a0fe8d0f8..4c606d0d9 100644 --- a/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy @@ -20,6 +20,11 @@ package io.seqera.wave.util import spock.lang.Specification +import java.time.OffsetDateTime + +import io.kubernetes.client.openapi.models.V1ObjectMeta +import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException @@ -55,5 +60,29 @@ class K8sHelperTest extends Specification { err.message == "Unsupported container platform 'linux/amd64'" } + def "should return the latest pod when multiple pods are present"() { + given: + def pod1 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now().minusDays(1))) + def pod2 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now())) + def allPods = new V1PodList().items(Arrays.asList(pod1, pod2)) + + when: + def latestPod = K8sHelper.findLatestPod(allPods) + + then: + latestPod == pod2 + } + + def "should return the only pod when one pod is present"() { + given: + def pod = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now())) + def allPods = new V1PodList().items(Collections.singletonList(pod)) + + when: + def latestPod = K8sHelper.findLatestPod(allPods) + + then: + latestPod == pod + } } From 7cf0b756b6aecdbca0cdc3946ee4fc438f613780 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 24 Jul 2024 19:41:05 +0200 Subject: [PATCH 14/50] Extend build timeout (#574) Signed-off-by: Paolo Di Tommaso Co-authored-by: Munish Chouhan --- .../wave/service/builder/BuildCacheStore.groovy | 11 +++++++---- .../controller/RegistryControllerRedisTest.groovy | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy index 58de37e6a..c6890036f 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy @@ -89,9 +89,10 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt @Override boolean storeIfAbsent(String imageName, BuildResult build) { - // store up 1.5 time the build timeout to prevent a missed cache + // store up 2.5 time the build timeout to prevent a missed cache // update on job termination remains too long in the store - final ttl = Duration.ofMillis(Math.round(getTimeout().toMillis() * 1.5f)) + // note: this should be longer than the max await time used in the Waiter#awaitCompletion method + final ttl = Duration.ofMillis(Math.round(getTimeout().toMillis() * 2.5f)) return putIfAbsent(imageName, build, ttl) } @@ -115,8 +116,10 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt static BuildResult awaitCompletion(BuildStore store, String imageName, BuildResult current) { final beg = System.currentTimeMillis() - // add 10% delay gap to prevent race condition with timeout expiration - final max = (store.timeout.toMillis() * 1.10) as long + // await nearly double of the build timeout time because the build job + // can require additional time, other than the build time, to be scheduled + // note: see also #storeIfAbsent method + final max = (store.timeout.toMillis() * 2.10) as long while( true ) { if( current==null ) { return BuildResult.unknown() diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy index 69ba8d4e3..089c494a0 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy @@ -55,7 +55,7 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr embeddedServer = ApplicationContext.run(EmbeddedServer, [ REDIS_HOST : redisHostName, REDIS_PORT : redisPort, - 'wave.build.timeout':'3s', + 'wave.build.timeout':'2s', 'micronaut.server.port': port, 'micronaut.http.services.default.url' : "http://localhost:$port".toString(), ], 'test', 'h2', 'redis') @@ -107,7 +107,7 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr response.getContentLength() == 10242 } - @Timeout(10) + @Timeout(15) void 'should render a timeout when build failed'() { given: HttpClient client = applicationContext.createBean(HttpClient) From 2ef034c05d1f064ed8400440e342d7e367597b70 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 25 Jul 2024 11:21:14 +0200 Subject: [PATCH 15/50] Add delete after finish to blob cache jobs (#576) Signed-off-by: Paolo Di Tommaso Signed-off-by: munishchouhan Co-authored-by: munishchouhan --- .../io/seqera/wave/configuration/BlobCacheConfig.groovy | 3 +++ .../seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy | 2 +- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 1 + .../io/seqera/wave/service/k8s/K8sServiceImplTest.groovy | 4 ++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index 97748a23c..c7c526e55 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -90,6 +90,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.retryAttempts:3}') Integer retryAttempts + @Value('${wave.blobCache.deleteAfterFinished:7d}') + Duration deleteAfterFinished + @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}') Duration podDeleteTimeout diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 75161bd99..1f5bba682 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -295,7 +295,7 @@ class BlobCacheServiceImpl implements BlobCacheService { static BlobCacheInfo awaitCompletion(BlobStore store, String key, BlobCacheInfo current) { final beg = System.currentTimeMillis() - // set the await timeout nealy double as the blob transfer timeout, this because the + // set the await timeout nearly double as the blob transfer timeout, this because the // transfer pod can spend `timeout` time in pending status awaiting to be scheduled // and the same `timeout` time amount carrying out the transfer (upload) operation final max = (store.timeout.toMillis() * 2.10) as long diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index d4dc5d130..d80880b0f 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -612,6 +612,7 @@ class K8sServiceImpl implements K8sService { //spec section def spec = builder.withNewSpec() .withBackoffLimit(blobConfig.retryAttempts) + .withTtlSecondsAfterFinished(blobConfig.deleteAfterFinished.toSeconds() as Integer) .withNewTemplate() .editOrNewSpec() .withServiceAccount(serviceAccount) diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index d0752f39f..c715a1cf2 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -511,6 +511,7 @@ class K8sServiceImplTest extends Specification { getTransferTimeout() >> Duration.ofSeconds(20) getEnvironment() >> [:] getRetryAttempts() >> 5 + getDeleteAfterFinished() >> Duration.ofDays(10) } when: @@ -521,6 +522,7 @@ class K8sServiceImplTest extends Specification { result.metadata.namespace == 'my-ns' and: result.spec.backoffLimit == 5 + result.spec.ttlSecondsAfterFinished == Duration.ofDays(10).seconds as Integer and: verifyAll(result.spec.template.spec) { activeDeadlineSeconds == 20 @@ -551,6 +553,7 @@ class K8sServiceImplTest extends Specification { getRequestsCpu() >> '2' getRequestsMemory() >> '8Gi' getRetryAttempts() >> 3 + getDeleteAfterFinished() >> Duration.ofDays(1) } when: @@ -560,6 +563,7 @@ class K8sServiceImplTest extends Specification { result.metadata.namespace == 'my-ns' and: result.spec.backoffLimit == 3 + result.spec.ttlSecondsAfterFinished == Duration.ofDays(1).seconds as Integer and: verifyAll(result.spec.template.spec) { activeDeadlineSeconds == 20 From 383c4b486b02d77d8ef07fb575180f696d68a94a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 25 Jul 2024 13:53:51 +0200 Subject: [PATCH 16/50] [release] bump version 1.10.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f8e233b27..81c871de4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.0 +1.10.0 diff --git a/changelog.txt b/changelog.txt index 65c3d6b10..b58320ecd 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,15 @@ # Wave changelog +1.10.0 - 25 Jul 2024 +- Add delete after finish to blob cache jobs (#576) [2ef034c0] +- Use k8s job for blob cache transfer (#479) [44819b48] +- Updated Wave license note (#570) [273f4b21] +- Throw an error on missing credentials for custom build repo (#564) [760f2173] +- Enable Cleanup for blob transfer pods (#568) [5f045e14] +- Prevent hard error when launch credentials cannot be accessed [600fce7a] +- Simplify credentials lookup [b2790ee8] +- Extend build timeout (#574) [7cf0b756] +- Bump wave-utils@0.13.1 [c118f0de] + 1.9.1 - 11 Jul 2024 - Prevent hard error when launch credentials cannot be accessed [a318a855] From 5a9120b856e53d385d46004b8d54a5be7146b5e4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 25 Jul 2024 18:19:40 +0200 Subject: [PATCH 17/50] Improve error report when platform cannot be found (#579) Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/core/ContainerAugmenter.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy index 243585147..112286465 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy @@ -32,6 +32,7 @@ import io.seqera.wave.api.ContainerLayer import io.seqera.wave.core.spec.ConfigSpec import io.seqera.wave.core.spec.ContainerSpec import io.seqera.wave.core.spec.ManifestSpec +import io.seqera.wave.exception.BadRequestException import io.seqera.wave.exception.DockerRegistryException import io.seqera.wave.proxy.ProxyClient import io.seqera.wave.storage.Storage @@ -397,7 +398,11 @@ class ContainerAugmenter { protected String findTargetDigest(Map json, boolean oci) { final record = (Map)json.manifests.find(oci ? this.&matchesOciManifest : this.&matchesDockerManifest) + if( !record ) + throw new BadRequestException("Cannot find platform '${platform}' in the manifest: ${JsonOutput.toJson(json)}") final result = record.get('digest') + if( !result ) + throw new BadRequestException("Cannot find digest entry for platform '${platform}' in the manifest: ${JsonOutput.toJson(json)}") log.trace "Find target digest platform: $platform ==> digest: $result" return result } From 702528df27c2a7c81a4d332b2330a92e47ded7fc Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 26 Jul 2024 14:25:42 +0200 Subject: [PATCH 18/50] Add second level cache for Registry auth record (#582) Signed-off-by: Paolo Di Tommaso --- .../wave/auth/RegistryAuthCacheStore.groovy | 62 +++++++++++++++++++ .../auth/RegistryLookupServiceImpl.groovy | 14 ++++- .../wave/encoder/MoshiEncodeStrategy.groovy | 1 + .../io/seqera/wave/encoder/UriAdapter.groovy | 41 ++++++++++++ .../encoder/MoshiEncodingStrategyTest.groovy | 21 ++++++- 5 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/auth/RegistryAuthCacheStore.groovy create mode 100644 src/main/groovy/io/seqera/wave/encoder/UriAdapter.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthCacheStore.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthCacheStore.groovy new file mode 100644 index 000000000..bd132aeb3 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthCacheStore.groovy @@ -0,0 +1,62 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth + +import java.time.Duration + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.service.cache.AbstractCacheStore +import io.seqera.wave.service.cache.impl.CacheProvider +import jakarta.inject.Singleton + +/** + * Implement a cache store for {@link RegistryAuth} object that + * can be distributed across wave replicas + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class RegistryAuthCacheStore extends AbstractCacheStore { + + private Duration duration + + RegistryAuthCacheStore( + CacheProvider provider, + @Value('${wave.registry-auth.cache.duration:`3h`}') Duration duration) + { + super(provider, new MoshiEncodeStrategy() {}) + this.duration = duration + log.info "Creating Registry Auth cache store ― duration=$duration" + } + + @Override + protected String getPrefix() { + return 'registry-auth/v1:' + } + + @Override + protected Duration getDuration() { + return duration + } +} diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index db65227a1..02781c8ca 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -50,12 +50,23 @@ class RegistryLookupServiceImpl implements RegistryLookupService { @Inject private HttpClientConfig httpConfig + @Inject + private RegistryAuthCacheStore store private CacheLoader loader = new CacheLoader() { @Override RegistryAuth load(URI endpoint) throws Exception { - final result = lookup0(endpoint) + // check if there's a record in the store cache (redis) + def result = store.get(endpoint.toString()) + if( result ) { + log.debug "Authority lookup for endpoint: '$endpoint' => $result [from store]" + return result + } + // look-up using the corresponding API endpoint + result = lookup0(endpoint) log.debug "Authority lookup for endpoint: '$endpoint' => $result" + // save it in the store cache (redis) + store.put(endpoint.toString(), result) return result } } @@ -66,7 +77,6 @@ class RegistryLookupServiceImpl implements RegistryLookupService { .expireAfterAccess(1, TimeUnit.HOURS) .build(loader) - protected RegistryAuth lookup0(URI endpoint) { final httpClient = HttpClientFactory.followRedirectsHttpClient() final request = HttpRequest.newBuilder() .uri(endpoint) .GET() .build() diff --git a/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy b/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy index 44960fc9c..da21ef99f 100644 --- a/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy @@ -69,6 +69,7 @@ abstract class MoshiEncodeStrategy implements EncodingStrategy { .add(new ByteArrayAdapter()) .add(new DateTimeAdapter()) .add(new PathAdapter()) + .add(new UriAdapter()) .add(PolymorphicJsonAdapterFactory.of(DigestStore.class, "@type") .withSubtype(LazyDigestStore, LazyDigestStore.simpleName) .withSubtype(ZippedDigestStore, ZippedDigestStore.simpleName) diff --git a/src/main/groovy/io/seqera/wave/encoder/UriAdapter.groovy b/src/main/groovy/io/seqera/wave/encoder/UriAdapter.groovy new file mode 100644 index 000000000..125a467df --- /dev/null +++ b/src/main/groovy/io/seqera/wave/encoder/UriAdapter.groovy @@ -0,0 +1,41 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.encoder + + +import com.squareup.moshi.FromJson +import com.squareup.moshi.ToJson +/** + * Mosh adapter for {@link URI} class + * + * @author Paolo Di Tommaso + */ +class UriAdapter { + + @ToJson + String serialize(URI uri) { + return uri != null ? uri.toString() : null + } + + @FromJson + URI deserialize(String data) { + return data != null ? URI.create(data) : null + } + +} diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index f9074a75e..8275e1513 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -26,6 +26,7 @@ import java.time.Instant import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.auth.RegistryAuth import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildEvent @@ -38,14 +39,13 @@ import io.seqera.wave.service.pairing.socket.msg.ProxyHttpResponse import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.DockerDigestStore -import io.seqera.wave.storage.LazyDigestStore import io.seqera.wave.storage.HttpDigestStore +import io.seqera.wave.storage.LazyDigestStore import io.seqera.wave.storage.ZippedDigestStore import io.seqera.wave.storage.reader.DataContentReader import io.seqera.wave.storage.reader.GzipContentReader import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User - /** * * @author Paolo Di Tommaso @@ -405,4 +405,21 @@ class MoshiEncodingStrategyTest extends Specification { and: copy == record1 } + + def 'should encode and decode registry info' () { + given: + def encoder = new MoshiEncodeStrategy() { } + and: + def auth = RegistryAuth.parse('Bearer realm="https://auth.docker.io/token",service="registry.docker.io"') + + when: + def json = encoder.encode(auth) + and: + def copy = encoder.decode(json) + then: + copy.getClass() == auth.getClass() + and: + copy == auth + + } } From 9e36cf76122ec887ba30f91a0de00f347b195ae9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 26 Jul 2024 14:32:19 +0200 Subject: [PATCH 19/50] [release] bump version 1.10.1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 81c871de4..4dae2985b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.0 +1.10.1 diff --git a/changelog.txt b/changelog.txt index b58320ecd..849fc0691 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,8 @@ # Wave changelog +1.10.1 - 26 Jul 2024 +- Add second level cache for Registry auth record (#582) [702528df] +- Improve error report when platform cannot be found (#579) [5a9120b8] + 1.10.0 - 25 Jul 2024 - Add delete after finish to blob cache jobs (#576) [2ef034c0] - Use k8s job for blob cache transfer (#479) [44819b48] From fc436f7c8f080817e4b24440df691fee4408e197 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 28 Jul 2024 15:52:02 +0200 Subject: [PATCH 20/50] Minor logging improvement Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/controller/RegistryProxyController.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index a687da7cd..479161d59 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -173,15 +173,16 @@ class RegistryProxyController { final entry = storage.getBlob(route.getTargetPath()).orElse(null) String location if( location=dockerRedirection(entry) ) { - log.debug "Blob found in the cache: $route.path ==> mapping to: ${location}" + log.debug "Blob found in the cache [docker]: ${route.path} ==> mapping to: ${location}" final target = RoutePath.parse(location, route.identity) return handleDelegate0(target, httpRequest) } else if ( location=httpRedirect(entry) ) { - log.debug "Blob found in the cache: $route.path ==> mapping to: $location" + log.debug "Blob found in the cache [http]: ${route.path} ==> mapping to: ${location}" return fromCacheRedirect(location) } else if( entry ) { + log.trace "Blob found in the cache [digest]: ${route.path} ==> entry: ${entry}" return fromCacheDigest(entry) } } From 51703862fac7def3e52dab3365760873b2151806 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 30 Jul 2024 10:51:12 +0200 Subject: [PATCH 21/50] Bump Surrealdb 1.5.4 for testing Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy index abfb70c49..c13cf2ebb 100644 --- a/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/SurrealDBTestContainer.groovy @@ -34,7 +34,7 @@ trait SurrealDBTestContainer { private static final def LOGGER = LoggerFactory.getLogger(SurrealDBTestContainer.class); @Shared - static GenericContainer surrealContainer = new GenericContainer(DockerImageName.parse("surrealdb/surrealdb:v1.4.2")) + static GenericContainer surrealContainer = new GenericContainer(DockerImageName.parse("surrealdb/surrealdb:v1.5.4")) .withExposedPorts(8000) .withCommand("start","--user", "root", "--pass", "root", '--log', 'debug') .waitingFor( From f05623a2aafbc090c807c2616e8a279160dc5e41 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Thu, 1 Aug 2024 17:53:22 +0200 Subject: [PATCH 22/50] Remove contentreader and contentreaderfactory classes (#584) Signed-off-by: munishchouhan --- .../controller/RegistryProxyController.groovy | 3 - .../wave/encoder/MoshiEncodeStrategy.groovy | 10 --- .../seqera/wave/storage/LazyDigestStore.java | 83 ------------------- .../wave/storage/ManifestCacheStore.groovy | 10 --- .../io/seqera/wave/storage/Storage.java | 4 - .../wave/storage/reader/ContentReader.java | 37 --------- .../storage/reader/ContentReaderFactory.java | 44 ---------- .../storage/reader/DataContentReader.java | 51 ------------ .../storage/reader/GzipContentReader.java | 64 -------------- .../storage/reader/HttpContentReader.java | 58 ------------- .../encoder/MoshiEncodingStrategyTest.groovy | 49 ----------- .../builder/ContainerBuildServiceTest.groovy | 3 +- .../wave/storage/LazyDigestStoreTest.groovy | 45 ---------- .../reader/DataContentReaderTest.groovy | 49 ----------- .../reader/GzipContentReaderTest.groovy | 47 ----------- 15 files changed, 1 insertion(+), 556 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/storage/LazyDigestStore.java delete mode 100644 src/main/groovy/io/seqera/wave/storage/reader/ContentReader.java delete mode 100644 src/main/groovy/io/seqera/wave/storage/reader/ContentReaderFactory.java delete mode 100644 src/main/groovy/io/seqera/wave/storage/reader/DataContentReader.java delete mode 100644 src/main/groovy/io/seqera/wave/storage/reader/GzipContentReader.java delete mode 100644 src/main/groovy/io/seqera/wave/storage/reader/HttpContentReader.java delete mode 100644 src/test/groovy/io/seqera/wave/storage/LazyDigestStoreTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/storage/reader/DataContentReaderTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/storage/reader/GzipContentReaderTest.groovy diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index 479161d59..c112dd0bd 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -53,7 +53,6 @@ import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.DockerDigestStore import io.seqera.wave.storage.HttpDigestStore -import io.seqera.wave.storage.LazyDigestStore import io.seqera.wave.storage.Storage import io.seqera.wave.util.Retryable import jakarta.inject.Inject @@ -198,8 +197,6 @@ class RegistryProxyController { private String httpRedirect(DigestStore entry) { if( entry instanceof HttpDigestStore ) return (entry as HttpDigestStore).location - if( entry instanceof LazyDigestStore ) - return (entry as LazyDigestStore).location return null } diff --git a/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy b/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy index da21ef99f..6d0c1f1b5 100644 --- a/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/encoder/MoshiEncodeStrategy.groovy @@ -32,12 +32,7 @@ import io.seqera.wave.service.pairing.socket.msg.ProxyHttpResponse import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.DockerDigestStore import io.seqera.wave.storage.HttpDigestStore -import io.seqera.wave.storage.LazyDigestStore import io.seqera.wave.storage.ZippedDigestStore -import io.seqera.wave.storage.reader.ContentReader -import io.seqera.wave.storage.reader.DataContentReader -import io.seqera.wave.storage.reader.GzipContentReader -import io.seqera.wave.storage.reader.HttpContentReader import io.seqera.wave.util.TypeHelper /** * Implements a JSON {@link EncodingStrategy} based on Mosh JSON serializer @@ -71,14 +66,9 @@ abstract class MoshiEncodeStrategy implements EncodingStrategy { .add(new PathAdapter()) .add(new UriAdapter()) .add(PolymorphicJsonAdapterFactory.of(DigestStore.class, "@type") - .withSubtype(LazyDigestStore, LazyDigestStore.simpleName) .withSubtype(ZippedDigestStore, ZippedDigestStore.simpleName) .withSubtype(HttpDigestStore, HttpDigestStore.simpleName) .withSubtype(DockerDigestStore, DockerDigestStore.simpleName) ) - .add(PolymorphicJsonAdapterFactory.of(ContentReader.class, "@type") - .withSubtype(DataContentReader.class, DataContentReader.simpleName) - .withSubtype(GzipContentReader.class, GzipContentReader.simpleName) - .withSubtype(HttpContentReader.class, HttpContentReader.simpleName)) .add(PolymorphicJsonAdapterFactory.of(PairingMessage.class, "@type") .withSubtype(ProxyHttpRequest.class, ProxyHttpRequest.simpleName) .withSubtype(ProxyHttpResponse.class, ProxyHttpResponse.simpleName) diff --git a/src/main/groovy/io/seqera/wave/storage/LazyDigestStore.java b/src/main/groovy/io/seqera/wave/storage/LazyDigestStore.java deleted file mode 100644 index c22e99b21..000000000 --- a/src/main/groovy/io/seqera/wave/storage/LazyDigestStore.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage; - -import java.io.IOException; - -import io.seqera.wave.storage.reader.ContentReader; -import io.seqera.wave.storage.reader.HttpContentReader; - -/** - * Implements a digest store that loads the binary content on-demand - * - * @author Paolo Di Tommaso - */ -@Deprecated -public class LazyDigestStore implements DigestStore { - - final private String mediaType; - final private String digest; - final private ContentReader contentReader; - final private Integer size; - - public LazyDigestStore(ContentReader content, String mediaType, String digest, int size) { - this.contentReader = content; - this.mediaType = mediaType; - this.digest = digest; - this.size = size; - } - - @Override - public byte[] getBytes() throws InterruptedException { - try { - return contentReader !=null ? contentReader.readAllBytes() : null; - } - catch (IOException e) { - throw new IllegalStateException("Unable to load digest content at path: "+ contentReader, e); - } - } - - @Override - public String getMediaType() { - return mediaType; - } - - @Override - public String getDigest() { - return digest; - } - - public Integer getSize() { - return size; - } - - public String toString() { - return String.format("LazyDigestStore(mediaType=%s; digest=%s; size=%d; reader=%s)", mediaType, digest, size, contentReader.toString()); - } - - @Override - public String toLogString() { - return String.format("LazyDigestStore(digest=%s; size=%d; mediaType=%s; reader=%s)", digest, size, mediaType, contentReader.toLogString()); - } - - public String getLocation() { - return contentReader instanceof HttpContentReader ? ((HttpContentReader) contentReader).getUrl() : null; - } - -} diff --git a/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy b/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy index 135290633..1c6751590 100644 --- a/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy @@ -27,7 +27,6 @@ import io.seqera.wave.api.ContainerLayer import io.seqera.wave.encoder.MoshiEncodeStrategy import io.seqera.wave.service.cache.AbstractCacheStore import io.seqera.wave.service.cache.impl.CacheProvider -import io.seqera.wave.storage.reader.ContentReader import jakarta.inject.Singleton /** * Implements manifest cache for {@link DigestStore} @@ -97,15 +96,6 @@ class ManifestCacheStore extends AbstractCacheStore implements Stor return result } - @Override - @Deprecated - DigestStore saveBlob(String path, ContentReader content, String type, String digest, int size) { - log.trace "Save Blob ==> $path" - final result = new LazyDigestStore(content, type, digest, size) - this.put(path, result) - return result - } - @Override DigestStore saveBlob(String path, ContainerLayer layer) { log.trace "Save Blob ==> $path; layer=${layer}" diff --git a/src/main/groovy/io/seqera/wave/storage/Storage.java b/src/main/groovy/io/seqera/wave/storage/Storage.java index ba8b53059..c6df330d4 100644 --- a/src/main/groovy/io/seqera/wave/storage/Storage.java +++ b/src/main/groovy/io/seqera/wave/storage/Storage.java @@ -22,7 +22,6 @@ import java.util.Optional; import io.seqera.wave.api.ContainerLayer; -import io.seqera.wave.storage.reader.ContentReader; /** * @author : jorge @@ -39,8 +38,5 @@ public interface Storage { DigestStore saveBlob(String path, byte[] content, String type, String digest); - @Deprecated - DigestStore saveBlob(String path, ContentReader content, String type, String digest, int size); - DigestStore saveBlob(String path, ContainerLayer layer); } diff --git a/src/main/groovy/io/seqera/wave/storage/reader/ContentReader.java b/src/main/groovy/io/seqera/wave/storage/reader/ContentReader.java deleted file mode 100644 index 7679effae..000000000 --- a/src/main/groovy/io/seqera/wave/storage/reader/ContentReader.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader; - -import java.io.IOException; -import java.io.Serializable; - -/** - * Generic interface to read layer content - * - * @author Paolo Di Tommaso - */ -@Deprecated -public interface ContentReader extends Serializable { - - byte[] readAllBytes() throws IOException, InterruptedException; - - default String toLogString() { - return toString(); - } -} diff --git a/src/main/groovy/io/seqera/wave/storage/reader/ContentReaderFactory.java b/src/main/groovy/io/seqera/wave/storage/reader/ContentReaderFactory.java deleted file mode 100644 index ad04916e9..000000000 --- a/src/main/groovy/io/seqera/wave/storage/reader/ContentReaderFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader; - -/** - * Creates a concrete instance of {@link ContentReader} - * for the given location string - * - * @author Paolo Di Tommaso - */ -@Deprecated -public class ContentReaderFactory { - - public static ContentReader of(String location) { - if( location==null ) - throw new IllegalArgumentException("Missing content location"); - if( location.startsWith("http://") || location.startsWith("https://") ) - return new HttpContentReader(location); - if( location.startsWith("data:") ) { - return new DataContentReader(location.substring(5)); - } - if( location.startsWith("gzip:") ) { - return GzipContentReader.fromBase64EncodedString(location.substring(5)); - } - throw new IllegalArgumentException("Unsupported content location: " + location); - } - -} diff --git a/src/main/groovy/io/seqera/wave/storage/reader/DataContentReader.java b/src/main/groovy/io/seqera/wave/storage/reader/DataContentReader.java deleted file mode 100644 index a45795d2e..000000000 --- a/src/main/groovy/io/seqera/wave/storage/reader/DataContentReader.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader; - -import java.util.Base64; - -/** - * Read a layer content from the given http(s) url - * - * @author Paolo Di Tommaso - */ -@Deprecated -public class DataContentReader implements ContentReader{ - - final private String data; - - public DataContentReader(String data) { - this.data = data; - } - - @Override - public byte[] readAllBytes() { - return Base64.getDecoder().decode(data); - } - - @Override - public String toString() { - return String.format("DataContentReader(%s)",data); - } - - @Override - public String toLogString() { - return "data=base64+encoded+string"; - } -} diff --git a/src/main/groovy/io/seqera/wave/storage/reader/GzipContentReader.java b/src/main/groovy/io/seqera/wave/storage/reader/GzipContentReader.java deleted file mode 100644 index 3dfc1b7f6..000000000 --- a/src/main/groovy/io/seqera/wave/storage/reader/GzipContentReader.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader; - -import java.io.IOException; -import java.util.Base64; - -import io.seqera.wave.util.ZipUtils; - -/** - * Implements a {@link ContentReader} that hold data - * - * @author Paolo Di Tommaso - */ -@Deprecated -public class GzipContentReader implements ContentReader { - - final private byte[] data; - - private GzipContentReader(byte[] data) { - this.data = data; - } - - @Override - public byte[] readAllBytes() { - return ZipUtils.decompressAsBytes(data); - } - - public static GzipContentReader fromPlainString(String value) throws IOException { - final byte[] compressed = ZipUtils.compress(value); - return new GzipContentReader(compressed); - } - - public static GzipContentReader fromBase64EncodedString(String value) { - final byte[] decoded = Base64.getDecoder().decode(value); - return new GzipContentReader(decoded); - } - - @Override - public String toString() { - return String.format("GzipContentReader(%s)",new String(Base64.getEncoder().encode(data))); - } - - @Override - public String toLogString() { - return "gzip=base64+encoded+string"; - } -} diff --git a/src/main/groovy/io/seqera/wave/storage/reader/HttpContentReader.java b/src/main/groovy/io/seqera/wave/storage/reader/HttpContentReader.java deleted file mode 100644 index 67a16a70c..000000000 --- a/src/main/groovy/io/seqera/wave/storage/reader/HttpContentReader.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader; - -import java.io.IOException; - -/** - * Read a layer content from the given http(s) url - * - * @author Paolo Di Tommaso - */ -@Deprecated -public class HttpContentReader implements ContentReader { - - final private String url; - - public HttpContentReader(String url) { - this.url = url; - } - - @Override - public byte[] readAllBytes() throws IOException, InterruptedException { - throw new UnsupportedOperationException("HttpContentReader does not support 'readAllBytes' operation"); - } - - @Deprecated - public String getUrl() { return url; } - - public String getLocation() { - return url; - } - - @Override - public String toString() { - return String.format("HttpContentReader(%s)",url); - } - - @Override - public String toLogString() { - return String.format("location=%s", url); - } -} diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index 8275e1513..cc608a0d6 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -40,10 +40,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.DockerDigestStore import io.seqera.wave.storage.HttpDigestStore -import io.seqera.wave.storage.LazyDigestStore import io.seqera.wave.storage.ZippedDigestStore -import io.seqera.wave.storage.reader.DataContentReader -import io.seqera.wave.storage.reader.GzipContentReader import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User /** @@ -154,52 +151,6 @@ class MoshiEncodingStrategyTest extends Specification { result.freeze } - def 'should encode and decode lazy digest store' () { - given: - def encoder = new MoshiEncodeStrategy() { } - and: - def data = new LazyDigestStore(new DataContentReader('FOO'.bytes.encodeBase64().toString()), 'media', '12345', 1000) - - when: - def json = encoder.encode(data) - println json - - and: - def copy = encoder.decode(json) - then: - copy.getClass() == data.getClass() - and: - copy.bytes == data.bytes - copy.digest == data.digest - copy.mediaType == data.mediaType - copy.size == 1000 - } - - def 'should encode and decode gzip content reader' () { - given: - def encoder = new MoshiEncodeStrategy() { } - and: - def data = new LazyDigestStore( - GzipContentReader.fromPlainString('Hello world'), - 'text/json', - '12345', - 2000 ) - - when: - def json = encoder.encode(data) - println json - - and: - def copy = encoder.decode(json) - then: - copy.getClass() == data.getClass() - and: - copy.bytes == data.bytes - copy.digest == data.digest - copy.mediaType == data.mediaType - copy.size == data.size - } - def 'should encode and decode zipped digest store' () { given: def DATA = 'Hello wold!' diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 8002de2b3..0f846366a 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -45,7 +45,6 @@ import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.inspect.ContainerInspectServiceImpl import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord -import io.seqera.wave.storage.reader.ContentReaderFactory import io.seqera.wave.test.RedisTestContainer import io.seqera.wave.test.SurrealDBTestContainer import io.seqera.wave.tower.PlatformId @@ -450,7 +449,7 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai and: HttpHandler handler = { HttpExchange exchange -> - def body = ContentReaderFactory.of(cl.location).readAllBytes() + def body = cl.location.bytes exchange.getResponseHeaders().add("Content-Type", "application/tar+gzip") exchange.sendResponseHeaders(200, body.size()) exchange.getResponseBody() << body diff --git a/src/test/groovy/io/seqera/wave/storage/LazyDigestStoreTest.groovy b/src/test/groovy/io/seqera/wave/storage/LazyDigestStoreTest.groovy deleted file mode 100644 index 9e24bc4ce..000000000 --- a/src/test/groovy/io/seqera/wave/storage/LazyDigestStoreTest.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage - -import spock.lang.Specification - -import io.seqera.wave.storage.reader.DataContentReader -/** - * - * @author Paolo Di Tommaso - */ -class LazyDigestStoreTest extends Specification { - - def 'should load a lazy digest' () { - given: - def CONTENT = 'Hello world!' - and: - def data = new DataContentReader(CONTENT.bytes.encodeBase64().toString()) - - when: - def digest = new LazyDigestStore(data, 'text', 'sha256:122345567890', 1000) - then: - digest.bytes == CONTENT.bytes - digest.digest == 'sha256:122345567890' - digest.mediaType == 'text' - digest.size == 1000 - } - -} diff --git a/src/test/groovy/io/seqera/wave/storage/reader/DataContentReaderTest.groovy b/src/test/groovy/io/seqera/wave/storage/reader/DataContentReaderTest.groovy deleted file mode 100644 index 32a4be106..000000000 --- a/src/test/groovy/io/seqera/wave/storage/reader/DataContentReaderTest.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader - -import spock.lang.Specification -/** - * - * @author Paolo Di Tommaso - */ -class DataContentReaderTest extends Specification { - - def 'should decode data' () { - given: - def encoded = 'Hello world'.bytes.encodeBase64().toString() - def reader = new DataContentReader(encoded) - - when: - def decoded = reader.readAllBytes() - then: - new String(decoded) == 'Hello world' - } - - def 'should decode data string' () { - given: - def encoded = 'Hello world'.bytes.encodeBase64().toString() - def reader = ContentReaderFactory.of("data:$encoded") - - when: - def decoded = reader.readAllBytes() - then: - new String(decoded) == 'Hello world' - } -} diff --git a/src/test/groovy/io/seqera/wave/storage/reader/GzipContentReaderTest.groovy b/src/test/groovy/io/seqera/wave/storage/reader/GzipContentReaderTest.groovy deleted file mode 100644 index 4635e7b9a..000000000 --- a/src/test/groovy/io/seqera/wave/storage/reader/GzipContentReaderTest.groovy +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.storage.reader - -import spock.lang.Specification - -import io.seqera.wave.util.ZipUtils - -/** - * - * @author Paolo Di Tommaso - */ -class GzipContentReaderTest extends Specification { - - def 'should decode data' () { - given: - def DATA = 'Hola mundo!' - - when: - def reader1 = GzipContentReader.fromPlainString(DATA) - then: - new String(reader1.readAllBytes()) == DATA - - when: - final compressed = ZipUtils.compress(DATA); - and: - def reader2 = GzipContentReader.fromBase64EncodedString(compressed.encodeBase64().toString()) - then: - new String(reader2.readAllBytes()) == DATA - } -} From 96fadbf5cb04ecf9a01e2c642f1edae47559eccd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 2 Aug 2024 13:52:08 +0200 Subject: [PATCH 23/50] Minor change dumpThreads method Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/cron/ThreadMonitorCron.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/cron/ThreadMonitorCron.groovy b/src/main/groovy/io/seqera/wave/cron/ThreadMonitorCron.groovy index 4dbc8cdeb..28d5e1760 100644 --- a/src/main/groovy/io/seqera/wave/cron/ThreadMonitorCron.groovy +++ b/src/main/groovy/io/seqera/wave/cron/ThreadMonitorCron.groovy @@ -96,16 +96,16 @@ class ThreadMonitorCron { } private String dumpThreads() { - def buffer = new StringBuffer() + final result = new StringBuilder() Map m = Thread.getAllStackTraces(); for(Map.Entry e : m.entrySet()) { - buffer.append('\n').append(e.getKey().toString()).append('\n') + result.append('\n').append(e.getKey().toString()).append('\n') for (StackTraceElement s : e.getValue()) { - buffer.append(" " + s).append('\n') + result.append(" " + s).append('\n') } } - return buffer.toString() + return result.toString() } static protected String getDumpFile(String file) { From e3b3d0278f603dcf8cc3ca4fea72baad803de57e Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Sat, 3 Aug 2024 12:31:17 +0200 Subject: [PATCH 24/50] Add second level cache in RegistryAuthService (#588) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/auth/RegistryAuthServiceImpl.groovy | 33 +++++++++++- .../wave/auth/RegistryTokenCacheStore.groovy | 54 +++++++++++++++++++ .../wave/auth/RegistryAuthServiceTest.groovy | 43 +++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/auth/RegistryTokenCacheStore.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index ec6c4cb28..5d1c9723b 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import java.util.concurrent.ExecutionException import java.util.concurrent.TimeUnit @@ -30,10 +31,12 @@ import com.google.common.util.concurrent.UncheckedExecutionException import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.http.HttpClientFactory +import io.seqera.wave.util.RegHelper import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils import jakarta.inject.Inject @@ -52,28 +55,54 @@ import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS @CompileStatic class RegistryAuthServiceImpl implements RegistryAuthService { + @PackageScope static final Duration _1_HOUR = Duration.ofHours(1) + @Inject private HttpClientConfig httpConfig + @Inject + private RegistryTokenCacheStore tokenStore + @Canonical @ToString(includePackage = false, includeNames = true) static private class CacheKey { final String image final RegistryAuth auth final RegistryCredentials creds + + String stableKey() { + return RegHelper.sipHash(['content': toString()]) + } } private CacheLoader loader = new CacheLoader() { @Override String load(CacheKey key) throws Exception { - return getToken0(key) + return getToken(key) } } + protected String getToken(CacheKey key){ + // check if there's a record in the store cache (redis) + // since the key is shared across replicas, it should be stable (java hashCode is not good) + final stableKey = "key-" + key.stableKey() + def result = tokenStore.get(stableKey) + if( result ) { + log.debug "Registry auth token for cachekey: '$key' [$stableKey] => $result [from store]" + return result + } + // look-up using the corresponding API endpoint + result = getToken0(key) + log.debug "Registry auth token for cachekey: '$key' [$stableKey] => $result" + // save it in the store cache (redis) + tokenStore.put(stableKey, result) + return result + } + private LoadingCache cacheTokens = CacheBuilder .newBuilder() .maximumSize(10_000) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) .build(loader) @Inject diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryTokenCacheStore.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryTokenCacheStore.groovy new file mode 100644 index 000000000..2ec5ae1c7 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/auth/RegistryTokenCacheStore.groovy @@ -0,0 +1,54 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth + +import java.time.Duration + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.service.cache.AbstractCacheStore +import io.seqera.wave.service.cache.impl.CacheProvider +import jakarta.inject.Singleton +/** + * Implement a cache store for {@link io.seqera.wave.auth.RegistryAuthServiceImpl.CacheKey} object and token that + * can be distributed across wave replicas + * + * @author Munish Chouhan + */ +@Slf4j +@Singleton +@CompileStatic +class RegistryTokenCacheStore extends AbstractCacheStore { + + RegistryTokenCacheStore(CacheProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + log.info "Creating Registry Auth token cache store" + } + + @Override + protected String getPrefix() { + return 'registry-token/v1:' + } + + @Override + protected Duration getDuration() { + return RegistryAuthServiceImpl._1_HOUR + } +} diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy index cd33bbe4d..b56fb35b8 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy @@ -36,6 +36,9 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis @Shared ApplicationContext applicationContext + @Inject + private RegistryTokenCacheStore tokenStore + @Shared @Value('${wave.registries.docker.io.username}') String dockerUsername @@ -164,4 +167,44 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis 'localhost' | 'test' | null | "localhost?scope=repository:test:pull" } + void "getToken should return token from store cache if present"() { + given: + RegistryAuthServiceImpl impl = loginService as RegistryAuthServiceImpl + def key = Mock(RegistryAuthServiceImpl.CacheKey) + def expectedToken = "cachedToken" + and: + tokenStore.put("key-" + key.stableKey(), expectedToken) + + when: + def result = impl.getToken(key) + + then: + result == expectedToken + } + + def 'check stableKey identity' () { + given: + def a1 = RegistryAuth.parse('Bearer realm="https://quay.io/v2/auth",service="quay.io"') + def a2 = RegistryAuth.parse('Bearer realm="https://quay.io/v2/auth",service="quay.io"') + def k1 = [username: 'foo', password: 'bar'] as RegistryCredentials + def k2 = [username: 'foo', password: 'bar'] as RegistryCredentials + def k3 = [username: 'foo', password: 'xyz'] as RegistryCredentials + def i1 = "ubuntu:latest" + def i2 = "ubuntu:24.04" + and: + def c1 = new RegistryAuthServiceImpl.CacheKey(i1, a1, k1) + def c2 = new RegistryAuthServiceImpl.CacheKey(i1, a1, k2) + def c3 = new RegistryAuthServiceImpl.CacheKey(i1, a2, k1) + def c4 = new RegistryAuthServiceImpl.CacheKey(i2, a1, k1) + def c5 = new RegistryAuthServiceImpl.CacheKey(i1, a1, k3) + + expect: + c1.stableKey() == '23476a51c7b6216a' + c1.stableKey() == c2.stableKey() + c1.stableKey() == c3.stableKey() + and: + c1.stableKey() != c4.stableKey() + c1.stableKey() != c5.stableKey() + } + } From c34a46292a29dc57e0bf31a2c53028c071626431 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Mon, 5 Aug 2024 14:02:01 +0200 Subject: [PATCH 25/50] Added remove from redis in invalidateAuthorization (#591) Signed-off-by: munishchouhan Co-authored-by: Paolo Di Tommaso --- .../wave/auth/RegistryAuthServiceImpl.groovy | 10 ++++++++-- .../wave/auth/RegistryAuthServiceTest.groovy | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 5d1c9723b..bdc7fe6da 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -85,7 +85,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getToken(CacheKey key){ // check if there's a record in the store cache (redis) // since the key is shared across replicas, it should be stable (java hashCode is not good) - final stableKey = "key-" + key.stableKey() + final stableKey = getStableKey(key) def result = tokenStore.get(stableKey) if( result ) { log.debug "Registry auth token for cachekey: '$key' [$stableKey] => $result [from store]" @@ -282,7 +282,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService { void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) cacheTokens.invalidate(key) + tokenStore.remove(getStableKey(key)) } - + /** + * Invalidate all cached authorization tokens + */ + private static String getStableKey(CacheKey key) { + return "key-" + key.stableKey() + } } diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy index b56fb35b8..7cbb097e7 100644 --- a/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthServiceTest.groovy @@ -207,4 +207,18 @@ class RegistryAuthServiceTest extends Specification implements SecureDockerRegis c1.stableKey() != c5.stableKey() } + void 'invalidateAuthorization should remove token from cache'() { + given: + RegistryAuthServiceImpl impl = loginService as RegistryAuthServiceImpl + def key = new RegistryAuthServiceImpl.CacheKey("image", Mock(RegistryAuth), Mock(RegistryCredentials)) + def stableKey = "key-" + key.stableKey() + tokenStore.put(stableKey, "token") + + when: + impl.invalidateAuthorization("image", key.auth, key.creds) + + then: + !tokenStore.get(stableKey) + } + } From 12627cf1a889c146c8197fdd6a222ab61d9887d6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 3 Aug 2024 12:34:49 +0200 Subject: [PATCH 26/50] [release] bump version 1.10.2 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4dae2985b..5ad2491cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.1 +1.10.2 diff --git a/changelog.txt b/changelog.txt index 849fc0691..44a5a22e2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.10.2 - 3 Aug 2024 +- Add second level cache in RegistryAuthService (#588) [4ce329a7] + 1.10.1 - 26 Jul 2024 - Add second level cache for Registry auth record (#582) [702528df] - Improve error report when platform cannot be found (#579) [5a9120b8] From 23f2f6fc4ae4efcbcef0500f655c5ebaa08d1e0b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 6 Aug 2024 09:43:42 +0200 Subject: [PATCH 27/50] Bump Java 21 as base image Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 622c954b8..5e840af70 100644 --- a/build.gradle +++ b/build.gradle @@ -116,7 +116,7 @@ micronaut { // jib { from { - image = 'cr.seqera.io/public/nf-jdk:corretto-17.0.10-al2023-jemalloc' + image = 'cr.seqera.io/public/nf-jdk:corretto-21-al2023-jemalloc' platforms { platform { architecture = 'amd64'; os = 'linux' } } From 9a49aa73e7507fdf76d040e2ead459d4d2b37336 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 6 Aug 2024 09:44:56 +0200 Subject: [PATCH 28/50] Update changelog Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5ad2491cf..587c5f0c7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.2 +1.10.3 diff --git a/changelog.txt b/changelog.txt index 44a5a22e2..2dcdabdb1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.10.3 - 5 Aug 2024 +- Fix from invalidateAuthorization from redis cache store (#591) [8f293d7c] + 1.10.2 - 3 Aug 2024 - Add second level cache in RegistryAuthService (#588) [4ce329a7] From 14f0476dc08efcc770ec8e0b4d5a56bacd4aa4e7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 6 Aug 2024 11:32:43 +0200 Subject: [PATCH 29/50] Prevent NPE when creds are empty Signed-off-by: Paolo Di Tommaso --- .../wave/service/inspect/ContainerInspectServiceImpl.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy index be6b738e3..03b4e5093 100644 --- a/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImpl.groovy @@ -93,9 +93,9 @@ class ContainerInspectServiceImpl implements ContainerInspectService { if( cacheRepo ) repos.add(cacheRepo) final result = credsJson(repos, identity) - if( buildRepo && !result.contains(host0(buildRepo)) ) + if( buildRepo && result && !result.contains(host0(buildRepo)) ) throw new BadRequestException("Missing credentials for target build repository: $buildRepo") - if( cacheRepo && !result.contains(host0(cacheRepo)) ) + if( cacheRepo && result && !result.contains(host0(cacheRepo)) ) throw new BadRequestException("Missing credentials for target cache repository: $buildRepo") return result } From b1ae9564041b588319589bff404574234f7108a7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 6 Aug 2024 11:42:22 +0200 Subject: [PATCH 30/50] [release] bump version 1.10.4 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 587c5f0c7..18b311420 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.3 +1.10.4 diff --git a/changelog.txt b/changelog.txt index 2dcdabdb1..07c8a2cc6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,12 @@ # Wave changelog +1.10.4 - 6 Aug 2024 +- Fix Prevent NPE when creds are empty [14f0476d] +- Remove ContentReader and ContentReaderFactory classes (#584) [f05623a2] +- Bump Surrealdb 1.5.4 for testing [51703862] +- Minor change dumpThreads method [96fadbf5] +- Minor logging improvement [fc436f7c] +- Bump Java 21 as base image [23f2f6fc] + 1.10.3 - 5 Aug 2024 - Fix from invalidateAuthorization from redis cache store (#591) [8f293d7c] From b5baea039532b476ba80f4e4ab0aeba2d9d92153 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 7 Aug 2024 11:19:40 +0200 Subject: [PATCH 31/50] Change k8s Job deletion to foreground pods propagation (#595) Signed-off-by: munishchouhan --- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index d80880b0f..b7d96dafc 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -675,7 +675,7 @@ class K8sServiceImpl implements K8sService { void deleteJob(String name) { k8sClient .batchV1Api() - .deleteNamespacedJob(name, namespace, null, null, null, null,"Background", null) + .deleteNamespacedJob(name, namespace, null, null, null, null,"Foreground", null) } } From cf4b75882c23faf0ea2fc03eae20134e5d11c20c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 8 Aug 2024 10:15:59 +0200 Subject: [PATCH 32/50] Increase blob cache timeout to 10m and decrese status to 1h Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/configuration/BlobCacheConfig.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index c7c526e55..16bcabe8f 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -44,10 +44,10 @@ class BlobCacheConfig { @Value('${wave.blobCache.failure.duration:4s}') Duration failureDuration - @Value('${wave.blobCache.timeout:5m}') + @Value('${wave.blobCache.timeout:10m}') Duration transferTimeout - @Value('${wave.blobCache.status.duration:5d}') + @Value('${wave.blobCache.status.duration:1h}') Duration statusDuration @Value('${wave.blobCache.storage.bucket}') From e4c756713fd3de0334cb6237939892e8db9f3c9d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 8 Aug 2024 14:43:57 +0200 Subject: [PATCH 33/50] Improve blob cache logging Signed-off-by: Paolo Di Tommaso --- .../wave/service/blob/BlobCacheInfo.groovy | 7 +++++ .../blob/impl/BlobCacheServiceImpl.groovy | 2 +- .../service/blob/BlobCacheInfoTest.groovy | 28 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy index a3d5eaf45..eab2733f6 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy @@ -17,6 +17,7 @@ */ package io.seqera.wave.service.blob +import java.time.Duration import java.time.Instant import groovy.transform.Canonical @@ -91,6 +92,12 @@ class BlobCacheInfo { locationUri && completionTime!=null } + Duration duration() { + creationTime && completionTime + ? Duration.between(creationTime, completionTime) + : null + } + static BlobCacheInfo create(String locationUrl, Map> request, Map> response) { final headers0 = new LinkedHashMap() for( Map.Entry> it : request ) diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 1f5bba682..3dbb5211a 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -231,7 +231,7 @@ class BlobCacheServiceImpl implements BlobCacheService { // the transfer command to be executed final cli = transferCommand(route, info) final result = transferStrategy.transfer(info, cli) - log.debug "== Blob cache completed for object '${target}'; status=$result.exitStatus" + log.debug "== Blob cache completed for object '${target}'; status=$result.exitStatus; duration: ${result.duration()}" return result } catch (Throwable t) { diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy index 8b97bdc07..15984ee6b 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy @@ -18,8 +18,10 @@ package io.seqera.wave.service.blob +import spock.lang.Shared import spock.lang.Specification +import java.time.Duration import java.time.Instant /** @@ -208,4 +210,30 @@ class BlobCacheInfoTest extends Specification { result2.contentLength == 100L result2.cacheControl == '12345' } + + @Shared + Instant now = Instant.now() + + def 'should validate duration' () { + given: + def info = new BlobCacheInfo( + null, + null, + null, + null, + null, + CREATE, + COMPLETE ) + + expect: + info.duration() == EXPECTED + + where: + CREATE | COMPLETE | EXPECTED + null | null | null + now | null | null + null | now | null + now | now.plusSeconds(10) | Duration.ofSeconds(10) + now | now.plusSeconds(60) | Duration.ofSeconds(60) + } } From 861d0580e75e341ba8ad808b9c9396479a96a085 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 11 Aug 2024 13:53:18 +0200 Subject: [PATCH 34/50] Message queue name refactoring Rename MessageBroker to MessageQueue and method `take` to `poll` for conistency with implemented semantic. Signed-off-by: Paolo Di Tommaso --- .../data/queue/AbstractMessageQueue.groovy | 6 +- ...ssageBroker.groovy => MessageQueue.groovy} | 21 ++++- ...Broker.groovy => LocalMessageQueue.groovy} | 15 +++- ...Broker.groovy => RedisMessageQueue.groovy} | 15 +++- .../socket/PairingOutboundQueue.groovy | 4 +- .../AbstractMessageQueueLocalTest.groovy | 4 +- .../AbstractMessageQueueRedisTest.groovy | 10 +-- .../data/queue/LocalMessageQueueTest.groovy | 65 +++++++++++++++ .../data/queue/RedisMessageQueueTest.groovy | 79 +++++++++++++++++++ 9 files changed, 197 insertions(+), 22 deletions(-) rename src/main/groovy/io/seqera/wave/service/data/queue/{MessageBroker.groovy => MessageQueue.groovy} (74%) rename src/main/groovy/io/seqera/wave/service/data/queue/impl/{LocalQueueBroker.groovy => LocalMessageQueue.groovy} (77%) rename src/main/groovy/io/seqera/wave/service/data/queue/impl/{RedisQueueBroker.groovy => RedisMessageQueue.groovy} (79%) create mode 100644 src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy index d3a2eda64..0f6ddc9a0 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy @@ -48,7 +48,7 @@ abstract class AbstractMessageQueue implements Runnable { final private static AtomicInteger count = new AtomicInteger() - final private MessageBroker broker + final private MessageQueue broker final private EncodingStrategy encoder @@ -65,7 +65,7 @@ abstract class AbstractMessageQueue implements Runnable { .expireAfterWrite(10, TimeUnit.MINUTES) .build() - AbstractMessageQueue(MessageBroker broker) { + AbstractMessageQueue(MessageQueue broker) { final type = TypeHelper.getGenericType(this, 0) this.encoder = new MoshiEncodeStrategy(type) {} this.broker = broker @@ -160,7 +160,7 @@ abstract class AbstractMessageQueue implements Runnable { // infer the target queue from the client key final target = targetFromClientKey(entry.key) // poll for a message from the queue - final value = broker.take(target) + final value = broker.poll(target) // if there's a message try to send it if( value != null ) { try { diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/MessageBroker.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/MessageQueue.groovy similarity index 74% rename from src/main/groovy/io/seqera/wave/service/data/queue/MessageBroker.groovy rename to src/main/groovy/io/seqera/wave/service/data/queue/MessageQueue.groovy index 6e9979d99..ef55dee0d 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/MessageBroker.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/MessageQueue.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.service.data.queue +import java.time.Duration + import groovy.transform.CompileStatic /** * Interface for a message broker modelled as a blocking queue. @@ -27,7 +29,7 @@ import groovy.transform.CompileStatic * @param The type of message that can be sent through the broker. */ @CompileStatic -interface MessageBroker { +interface MessageQueue { /** * Inserts the specified element at the tail of the specified queue. @@ -40,8 +42,19 @@ interface MessageBroker { void offer(String target, M value) /** - * Retrieves and removes the head of this queue, waiting up to the specified wait time if necessary - * for an element to become available. + * Retrieves and removes the head of this queue, or returns null if this queue is empty. + * + * @param target + * The queue unique identifier + * @param timeout + * How long to wait before giving up, in units of unit unit – a TimeUnit determining how to interpret the timeout parameter + * @return + * The head of this queue, or null if queue is empty + */ + M poll(String target) + + /** + * Retrieves and removes the head of this queue, waiting up to the specified wait time if necessary for an element to become available. * * @param target * The queue unique identifier @@ -50,7 +63,7 @@ interface MessageBroker { * @return * The head of this queue, or null if the specified waiting time elapses before an element is available */ - M take(String target) + M poll(String target, Duration timeout) } diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalQueueBroker.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalMessageQueue.groovy similarity index 77% rename from src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalQueueBroker.groovy rename to src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalMessageQueue.groovy index b5e0b0211..d58905224 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalQueueBroker.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/impl/LocalMessageQueue.groovy @@ -18,13 +18,15 @@ package io.seqera.wave.service.data.queue.impl +import java.time.Duration import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.seqera.wave.service.data.queue.MessageBroker +import io.seqera.wave.service.data.queue.MessageQueue import jakarta.inject.Singleton /** * Implement a message broker based on a simple blocking queue. @@ -36,7 +38,7 @@ import jakarta.inject.Singleton @Requires(notEnv = 'redis') @Singleton @CompileStatic -class LocalQueueBroker implements MessageBroker { +class LocalMessageQueue implements MessageQueue { private ConcurrentHashMap> store = new ConcurrentHashMap<>() @@ -48,10 +50,17 @@ class LocalQueueBroker implements MessageBroker { } @Override - String take(String target) { + String poll(String target) { store .computeIfAbsent(target, (it)->new LinkedBlockingQueue()) .poll() } + String poll(String target, Duration timeout) { + final q = store .computeIfAbsent(target, (it)->new LinkedBlockingQueue()) + final millis = timeout.toMillis() + return millis>0 + ? q.poll(millis, TimeUnit.MILLISECONDS) + : q.take() + } } diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisQueueBroker.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy similarity index 79% rename from src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisQueueBroker.groovy rename to src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy index 5af0aae9b..7eaaeb5b1 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisQueueBroker.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy @@ -18,11 +18,12 @@ package io.seqera.wave.service.data.queue.impl +import java.time.Duration import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.seqera.wave.service.data.queue.MessageBroker +import io.seqera.wave.service.data.queue.MessageQueue import jakarta.inject.Inject import jakarta.inject.Singleton import redis.clients.jedis.Jedis @@ -36,7 +37,7 @@ import redis.clients.jedis.JedisPool @Requires(env = 'redis') @Singleton @CompileStatic -class RedisQueueBroker implements MessageBroker { +class RedisMessageQueue implements MessageQueue { @Inject private JedisPool pool @@ -49,10 +50,18 @@ class RedisQueueBroker implements MessageBroker { } @Override - String take(String target) { + String poll(String target) { try (Jedis conn = pool.getResource()) { return conn.rpop(target) } } + + @Override + String poll(String target, Duration duration) { + try (Jedis conn = pool.getResource()) { + double d = duration.toMillis() / 1000.0 + return conn.brpop(d, target) + } + } } diff --git a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingOutboundQueue.groovy b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingOutboundQueue.groovy index eec48ef1b..889af4871 100644 --- a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingOutboundQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingOutboundQueue.groovy @@ -24,7 +24,7 @@ import javax.annotation.PreDestroy import groovy.transform.CompileStatic import io.micronaut.context.annotation.Value import io.seqera.wave.service.data.queue.AbstractMessageQueue -import io.seqera.wave.service.data.queue.MessageBroker +import io.seqera.wave.service.data.queue.MessageQueue import io.seqera.wave.service.pairing.socket.msg.PairingMessage import jakarta.inject.Singleton /** @@ -39,7 +39,7 @@ class PairingOutboundQueue extends AbstractMessageQueue { final private Duration pollInterval PairingOutboundQueue( - MessageBroker broker, + MessageQueue broker, @Value('${wave.pairing.channel.awaitTimeout:100ms}') Duration pollInterval ) { super(broker) diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy index 457efe00c..480c9013d 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueLocalTest.groovy @@ -30,7 +30,7 @@ import io.seqera.wave.service.pairing.socket.msg.PairingHeartbeat import io.seqera.wave.service.pairing.socket.msg.PairingMessage import jakarta.inject.Inject /** - * Test class {@link AbstractMessageQueue} using a {@link io.seqera.wave.service.data.queue.impl.LocalQueueBroker} + * Test class {@link AbstractMessageQueue} using a {@link io.seqera.wave.service.data.queue.impl.LocalMessageQueue} * * @author Jordi Deu-Pons */ @@ -38,7 +38,7 @@ import jakarta.inject.Inject class AbstractMessageQueueLocalTest extends Specification { @Inject - private MessageBroker broker + private MessageQueue broker def 'should send and consume a request'() { diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy index 98d7be26b..4bfc76ca5 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueueRedisTest.groovy @@ -26,13 +26,13 @@ import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeUnit import io.micronaut.context.ApplicationContext -import io.seqera.wave.service.data.queue.impl.RedisQueueBroker +import io.seqera.wave.service.data.queue.impl.RedisMessageQueue import io.seqera.wave.service.pairing.socket.PairingOutboundQueue import io.seqera.wave.service.pairing.socket.msg.PairingHeartbeat import io.seqera.wave.service.pairing.socket.msg.PairingMessage import io.seqera.wave.test.RedisTestContainer /** - * Test class {@link AbstractMessageQueue} using a {@link RedisQueueBroker} + * Test class {@link AbstractMessageQueue} using a {@link RedisMessageQueue} * * @author Jordi Deu-Pons */ @@ -52,7 +52,7 @@ class AbstractMessageQueueRedisTest extends Specification implements RedisTestCo def 'should send and consume a request'() { given: - def broker = applicationContext.getBean(RedisQueueBroker) + def broker = applicationContext.getBean(RedisMessageQueue) def queue = new PairingOutboundQueue(broker, Duration.ofMillis(100)) and: def result = new CompletableFuture() @@ -70,10 +70,10 @@ class AbstractMessageQueueRedisTest extends Specification implements RedisTestCo def 'should send and consume a request across instances'() { given: - def broker1 = applicationContext.getBean(RedisQueueBroker) + def broker1 = applicationContext.getBean(RedisMessageQueue) def queue1 = new PairingOutboundQueue(broker1, Duration.ofMillis(100)) and: - def broker2 = applicationContext.getBean(RedisQueueBroker) + def broker2 = applicationContext.getBean(RedisMessageQueue) def queue2 = new PairingOutboundQueue(broker2, Duration.ofMillis(100)) and: def result = new CompletableFuture() diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy new file mode 100644 index 000000000..6361907ce --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy @@ -0,0 +1,65 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.queue + +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.data.queue.impl.LocalMessageQueue +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class LocalMessageQueueTest extends Specification { + + @Inject + private LocalMessageQueue broker + + def 'should return null if empty' () { + expect: + broker.poll('foo') == null + + when: + def start = System.currentTimeMillis() + and: + broker.poll('foo', Duration.ofMillis(500)) == null + and: + def delta = System.currentTimeMillis()-start + then: + assert delta>500 + assert delta<1000 + } + + def 'should offer and poll a value' () { + given: + broker.offer('bar', 'alpha') + broker.offer('bar', 'beta') + + expect: + broker.poll('foo') == null + broker.poll('bar') == 'alpha' + broker.poll('bar') == 'beta' + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy new file mode 100644 index 000000000..87b347146 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy @@ -0,0 +1,79 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.queue + +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.context.ApplicationContext +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.data.queue.impl.RedisMessageQueue +import io.seqera.wave.test.RedisTestContainer +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class RedisMessageQueueTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext context + + def setup() { + context = ApplicationContext.run([ + REDIS_HOST: redisHostName, + REDIS_PORT: redisPort + ], 'test', 'redis') + + } + + def 'should return null if empty' () { + given: + def broker = context.getBean(RedisMessageQueue) + + expect: + broker.poll('foo') == null + + when: + def start = System.currentTimeMillis() + and: + broker.poll('foo', Duration.ofMillis(500)) == null + and: + def delta = System.currentTimeMillis()-start + then: + assert delta>500 + assert delta<1000 + } + + def 'should offer and poll a value' () { + given: + def broker = context.getBean(RedisMessageQueue) + and: + broker.offer('bar', 'alpha') + broker.offer('bar', 'beta') + + expect: + broker.poll('foo') == null + broker.poll('bar') == 'alpha' + broker.poll('bar') == 'beta' + } + +} From dfb64badfe96fc800967fe4dedc8f14e3b766c23 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 16 Aug 2024 21:47:40 +0200 Subject: [PATCH 35/50] Improve blob cache reliability (#596) Signed-off-by: Paolo Di Tommaso Signed-off-by: munishchouhan Co-authored-by: munishchouhan --- .../wave/configuration/BlobCacheConfig.groovy | 6 + .../controller/RegistryProxyController.groovy | 3 +- .../wave/service/blob/BlobCacheInfo.groovy | 54 ++++-- .../blob/impl/BlobCacheServiceImpl.groovy | 167 ++++-------------- .../blob/impl/DockerTransferStrategy.groovy | 109 ++++++++++-- .../blob/impl/KubeTransferStrategy.groovy | 100 +++++------ .../blob/impl/SimpleTransferStrategy.groovy | 58 ------ .../service/blob/transfer/Transfer.groovy | 72 ++++++++ .../blob/transfer/TransferManager.groovy | 163 +++++++++++++++++ .../blob/transfer/TransferQueue.groovy | 48 +++++ .../{ => transfer}/TransferStrategy.groovy | 11 +- .../TransferTimeoutException.groovy | 2 +- .../data/queue/impl/RedisMessageQueue.groovy | 3 +- .../seqera/wave/service/k8s/K8sService.groovy | 18 +- .../wave/service/k8s/K8sServiceImpl.groovy | 21 ++- .../service/stream/StreamServiceImpl.groovy | 4 +- .../io/seqera/wave/util/K8sHelper.groovy | 13 -- .../service/blob/BlobCacheInfoTest.groovy | 64 +++++-- .../blob/impl/BlobCacheServiceImplTest.groovy | 114 +----------- .../impl/DockerTransferStrategyTest.groovy | 22 ++- .../blob/impl/KubeTransferStrategyTest.groovy | 100 ++++++++--- .../impl/SimpleTransferStrategyTest.groovy | 54 ------ .../blob/transfer/TransferManagerTest.groovy | 143 +++++++++++++++ .../service/blob/transfer/TransferTest.groovy | 61 +++++++ .../data/queue/LocalMessageQueueTest.groovy | 2 +- .../data/queue/RedisMessageQueueTest.groovy | 13 ++ .../service/k8s/K8sServiceImplTest.groovy | 56 ++++++ .../io/seqera/wave/util/K8sHelperTest.groovy | 31 ---- 28 files changed, 973 insertions(+), 539 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategy.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy rename src/main/groovy/io/seqera/wave/service/blob/{ => transfer}/TransferStrategy.groovy (80%) rename src/main/groovy/io/seqera/wave/service/blob/{ => transfer}/TransferTimeoutException.groovy (96%) delete mode 100644 src/test/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategyTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index 16bcabe8f..8640206d6 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -44,6 +44,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.failure.duration:4s}') Duration failureDuration + @Value('${wave.blobCache.grace.duration:20s}') + Duration graceDuration + @Value('${wave.blobCache.timeout:10m}') Duration transferTimeout @@ -96,6 +99,9 @@ class BlobCacheConfig { @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}') Duration podDeleteTimeout + @Value('${wave.blobCache.transfer.executor-shutdown-timeout:20s}') + Duration transferExecutorShutdownTimeout + Map getEnvironment() { final result = new HashMap(10) if( storageRegion ) { diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index c112dd0bd..f187078cb 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -223,7 +223,7 @@ class RegistryProxyController { return fromDownloadResponse(resp, route, headers) } else { - log.debug "Pulling stream from repository: '${route.getTargetContainer()}'" + log.warn "Pulling stream from repository: '${route.getTargetContainer()}' - Streaming is not scalable. Blob cache service should be configured instead." return fromStreamResponse(resp, route, headers) } } @@ -315,7 +315,6 @@ class RegistryProxyController { } MutableHttpResponse fromDownloadResponse(final DelegateResponse resp, RoutePath route, Map> headers) { - log.debug "== Blob cache upstream $resp" final blobCache = blobCacheService .retrieveBlobCache(route, headers, resp.headers) log.debug "== Blob cache response [succeeded=${blobCache.succeeded()}] $blobCache" if( !blobCache.succeeded() ) { diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy index eab2733f6..29757a856 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy @@ -20,12 +20,11 @@ package io.seqera.wave.service.blob import java.time.Duration import java.time.Instant +import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.transform.ToString import groovy.util.logging.Slf4j - /** * Model a blob cache metadata entry * @@ -42,6 +41,16 @@ class BlobCacheInfo { */ final String locationUri + /** + * The object storage path URI e.g. s3://bucket-name/some/path + */ + final String objectUri + + /** + * it is the name of k8s job or docker container depends on the transfer strategy + */ + final String jobName + /** * The request http headers */ @@ -84,6 +93,10 @@ class BlobCacheInfo { */ final String logs + String id() { + return objectUri + } + boolean succeeded() { locationUri && exitStatus==0 } @@ -98,14 +111,15 @@ class BlobCacheInfo { : null } - static BlobCacheInfo create(String locationUrl, Map> request, Map> response) { + static BlobCacheInfo create(String locationUri, String objectUri, Map> request, Map> response) { final headers0 = new LinkedHashMap() for( Map.Entry> it : request ) headers0.put( it.key, it.value.join(',') ) final length = headerLong0(response, 'Content-Length') final type = headerString0(response, 'Content-Type') final cache = headerString0(response, 'Cache-Control') - new BlobCacheInfo(locationUrl, headers0, length, type, cache, Instant.now(), null, null, null) + final creationTime = Instant.now() + return new BlobCacheInfo(locationUri, objectUri, generateJobName(locationUri, creationTime), headers0, length, type, cache, creationTime, null, null, null) } static String headerString0(Map> headers, String name) { @@ -125,18 +139,24 @@ class BlobCacheInfo { BlobCacheInfo cached() { new BlobCacheInfo( locationUri, + objectUri, + jobName, headers, contentLength, contentType, cacheControl, creationTime, creationTime, - 0) + 0, + null + ) } BlobCacheInfo completed(int status, String logs) { new BlobCacheInfo( locationUri, + objectUri, + jobName, headers, contentLength, contentType, @@ -144,12 +164,15 @@ class BlobCacheInfo { creationTime, Instant.now(), status, - logs) + logs + ) } BlobCacheInfo failed(String logs) { new BlobCacheInfo( locationUri, + objectUri, + jobName, headers, contentLength, contentType, @@ -161,9 +184,11 @@ class BlobCacheInfo { ) } - BlobCacheInfo withLocation(String uri) { + BlobCacheInfo withLocation(String location) { new BlobCacheInfo( - uri, + location, + objectUri, + jobName, headers, contentLength, contentType, @@ -175,9 +200,8 @@ class BlobCacheInfo { ) } - @Memoized - static BlobCacheInfo unknown() { - new BlobCacheInfo(null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null) { + static BlobCacheInfo unknown(String logs) { + new BlobCacheInfo(null, null, null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null, logs) { @Override BlobCacheInfo withLocation(String uri) { // prevent the change of location for unknown status @@ -186,4 +210,12 @@ class BlobCacheInfo { } } + static private String generateJobName(String locationUri, Instant creationTime) { + return 'transfer-' + Hashing + .sipHash24() + .newHasher() + .putUnencodedChars(locationUri) + .putUnencodedChars(creationTime.toString()) + .hash() + } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 3dbb5211a..4dc8d8978 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -20,14 +20,11 @@ package io.seqera.wave.service.blob.impl import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value -import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.HttpClientConfig import io.seqera.wave.core.RegistryProxyService @@ -37,19 +34,14 @@ import io.seqera.wave.service.blob.BlobCacheInfo import io.seqera.wave.service.blob.BlobCacheService import io.seqera.wave.service.blob.BlobSigningService import io.seqera.wave.service.blob.BlobStore -import io.seqera.wave.service.blob.TransferStrategy -import io.seqera.wave.service.blob.TransferTimeoutException -import io.seqera.wave.util.BucketTokenizer +import io.seqera.wave.service.blob.transfer.TransferQueue +import io.seqera.wave.service.blob.transfer.TransferStrategy import io.seqera.wave.util.Escape import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils import jakarta.annotation.PostConstruct import jakarta.inject.Inject -import jakarta.inject.Named import jakarta.inject.Singleton -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest -import software.amazon.awssdk.services.s3.model.HeadObjectRequest import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS /** * Implements cache for container image layer blobs @@ -75,11 +67,10 @@ class BlobCacheServiceImpl implements BlobCacheService { private RegistryProxyService proxyService @Inject - @Named(TaskExecutors.IO) - private ExecutorService executor + private TransferStrategy transferStrategy @Inject - private TransferStrategy transferStrategy + private TransferQueue transferQueue @Inject private BlobSigningService signingService @@ -87,10 +78,6 @@ class BlobCacheServiceImpl implements BlobCacheService { @Inject private HttpClientConfig httpConfig - @Inject - @Named('BlobS3Client') - private S3Client s3Client - private HttpClient httpClient @PostConstruct @@ -101,20 +88,28 @@ class BlobCacheServiceImpl implements BlobCacheService { @Override BlobCacheInfo retrieveBlobCache(RoutePath route, Map> requestHeaders, Map> responseHeaders) { - final uri = blobDownloadUri(route) - log.trace "Container blob download uri: $uri" + final locationUri = blobDownloadUri(route) + final objectUri = blobStorePath(route) + log.trace "Container blob download uri: $locationUri; target object: $objectUri" + + final info = BlobCacheInfo.create(locationUri, objectUri, requestHeaders, responseHeaders) + // both S3 and R2 are strongly consistent + // therefore it's safe to check and return directly + // if it exists (no risk of returning a partial upload) + // https://developers.cloudflare.com/r2/reference/consistency/ + if( blobExists(info.locationUri) && !debug ) { + log.debug "== Blob cache exists for object '${info.locationUri}'" + return info.cached() + } - final info = BlobCacheInfo.create(uri, requestHeaders, responseHeaders) - final target = route.targetPath - if( blobStore.storeIfAbsent(target, info) ) { + if( blobStore.storeIfAbsent(info.id(), info) ) { // start download and caching job - return storeIfAbsent(route, info) - } - else { - final result = awaitCacheStore(target) - // update the download signed uri - return result?.withLocation(uri) + store(route, info) } + + final result = awaitCacheStore(info.id()) + // update the download signed uri + return result?.withLocation(locationUri) } protected boolean blobExists(String uri) { @@ -183,61 +178,21 @@ class BlobCacheServiceImpl implements BlobCacheService { return command } - protected BlobCacheInfo storeIfAbsent(RoutePath route, BlobCacheInfo info) { - BlobCacheInfo result - try { - if( blobExists(info.locationUri) && !debug ) { - log.debug "== Blob cache exists for object '${info.locationUri}'" - result = info.cached() - } - else { - log.debug "== Blob cache begin for object '${info.locationUri}'" - result = store(route, info) - //check if the cached blob size is correct - result = checkUploadedBlobSize(result, route) - } - } - finally { - // use a short time-to-live for failed downloads - // this is needed to allow re-try caching of failure transfers - final ttl = result.succeeded() - ? blobConfig.statusDuration - : blobConfig.failureDuration - - blobStore.storeBlob(route.targetPath, result, ttl) - return result - } - } - - /** - * Check the size of the blob stored in the cache - * - * @return {@link BlobCacheInfo} the blob cache info - */ - protected BlobCacheInfo checkUploadedBlobSize(BlobCacheInfo info, RoutePath route) { - if( !info.succeeded() ) - return info - final blobSize = getBlobSize(route) - if( blobSize == info.contentLength ) - return info - log.warn("== Blob cache mismatch size for uploaded object '${info.locationUri}'; upload blob size: ${blobSize}; expect size: ${info.contentLength}") - CompletableFuture.supplyAsync(() -> deleteBlob(route), executor) - return info.failed("Mismatch cache size for object ${info.locationUri}") - } - - protected BlobCacheInfo store(RoutePath route, BlobCacheInfo info) { - final target = route.targetPath + protected void store(RoutePath route, BlobCacheInfo info) { + log.debug "== Blob cache begin for object '${info.locationUri}'" try { // the transfer command to be executed final cli = transferCommand(route, info) - final result = transferStrategy.transfer(info, cli) - log.debug "== Blob cache completed for object '${target}'; status=$result.exitStatus; duration: ${result.duration()}" - return result + transferStrategy.transfer(info, cli) + // signal the transfer to be started + // note: both `transferQueue` and `blobStore` use the same object `id` + transferQueue.offer(info.id()) } catch (Throwable t) { - log.warn "== Blob cache failed for object '${target}' - cause: ${t.message}", t + log.warn "== Blob cache failed for object '${info.objectUri}' - cause: ${t.message}", t final result = info.failed(t.message) - return result + // update the blob status + blobStore.storeBlob(info.id(), result, blobConfig.failureDuration) } } @@ -294,24 +249,15 @@ class BlobCacheServiceImpl implements BlobCacheService { private static class Waiter { static BlobCacheInfo awaitCompletion(BlobStore store, String key, BlobCacheInfo current) { - final beg = System.currentTimeMillis() - // set the await timeout nearly double as the blob transfer timeout, this because the - // transfer pod can spend `timeout` time in pending status awaiting to be scheduled - // and the same `timeout` time amount carrying out the transfer (upload) operation - final max = (store.timeout.toMillis() * 2.10) as long + final target = current?.locationUri ?: "(unknown)" while( true ) { if( current==null ) { - return BlobCacheInfo.unknown() + return BlobCacheInfo.unknown("Unable to cache blob $target") } - // check is completed if( current.done() ) { return current } - // check if it's timed out - final delta = System.currentTimeMillis()-beg - if( delta > max ) - throw new TransferTimeoutException("Blob cache transfer '$key' timed out") // sleep a bit Thread.sleep(store.delay.toMillis()) // fetch the build status again @@ -320,49 +266,4 @@ class BlobCacheServiceImpl implements BlobCacheService { } } - /** - * get the size of the blob stored in the cache - * - * @return {@link Long} the size of the blob stored in the cache - */ - protected Long getBlobSize(RoutePath route) { - final objectUri = blobStorePath(route) - final object = BucketTokenizer.from(objectUri) - try { - final request = - HeadObjectRequest.builder() - .bucket(object.bucket) - .key(object.key) - .build() - final headObjectResponse = s3Client.headObject(request as HeadObjectRequest) - final contentLength = headObjectResponse.contentLength() - return contentLength!=null ? contentLength : -1L - } - catch (Exception e){ - log.error("== Blob cache Error getting content length of object $objectUri from bucket ${blobConfig.storageBucket}", e) - return -1L - } - } - - /** - * delete the blob stored in the cache - * - */ - protected void deleteBlob(RoutePath route) { - final objectUri = blobStorePath(route) - log.debug "== Blob cache Deleting object $objectUri" - final object = BucketTokenizer.from(objectUri) - try { - final request = - DeleteObjectRequest.builder() - .bucket(object.bucket) - .key(object.key) - .build() - s3Client.deleteObject(request as DeleteObjectRequest) - log.debug("== Blob cache Deleted object $objectUri from bucket ${blobConfig.storageBucket}") - } - catch (Exception e){ - log.error("== Blob cache Error deleting object $objectUri from bucket ${blobConfig.storageBucket}", e) - } - } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy index 431465209..06d87e147 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy @@ -18,16 +18,19 @@ package io.seqera.wave.service.blob.impl -import java.util.concurrent.TimeUnit +import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Replaces import io.micronaut.context.annotation.Requires import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.TransferStrategy +import io.seqera.wave.service.blob.transfer.Transfer +import io.seqera.wave.service.blob.transfer.TransferStrategy import jakarta.inject.Inject +import jakarta.inject.Singleton + /** * Implements {@link TransferStrategy} that runs s5cmd using a docker * container. Meant for development purposes @@ -36,28 +39,30 @@ import jakarta.inject.Inject */ @Slf4j @CompileStatic -@Requires(property = 'wave.blobCache.strategy', value = 'docker') -@Replaces(SimpleTransferStrategy) +@Singleton +@Requires(missingProperty = 'wave.build.k8s') +@Requires(property = 'wave.blobCache.enabled', value = 'true') class DockerTransferStrategy implements TransferStrategy { @Inject private BlobCacheConfig blobConfig @Override - BlobCacheInfo transfer(BlobCacheInfo info, List command) { - final proc = createProcess(command).start() - // wait for the completion and save the result - final completed = proc.waitFor(blobConfig.transferTimeout.toSeconds(), TimeUnit.SECONDS) - final int status = completed ? proc.exitValue() : -1 - final logs = proc.inputStream.text - return info.completed(status, logs) + void transfer(BlobCacheInfo info, List command) { + // create a unique name for the container + createProcess(command, info.jobName, blobConfig.transferTimeout.toSeconds()) + .start() } - protected ProcessBuilder createProcess(List command) { + protected ProcessBuilder createProcess(List command, String name, long timeoutSecs) { // compose the docker command final cli = new ArrayList(10) cli.add('docker') cli.add('run') + cli.add('--name') + cli.add(name) + cli.add('--stop-timeout') + cli.add(String.valueOf(timeoutSecs)) cli.add('-e') cli.add('AWS_ACCESS_KEY_ID') cli.add('-e') @@ -73,4 +78,82 @@ class DockerTransferStrategy implements TransferStrategy { builder.redirectErrorStream(true) return builder } + + @Override + Transfer status(BlobCacheInfo blob) { + final state = getDockerContainerState(blob.jobName) + log.trace "Docker transfer status name=$blob.jobName; state=$state" + + if (state.status == 'running') { + return Transfer.running() + } + else if (state.status == 'exited') { + final logs = getDockerContainerLogs(blob.jobName) + return Transfer.completed(state.exitCode, logs) + } + else if (state.status == 'created' || state.status == 'paused') { + return Transfer.pending() + } + else { + final logs = getDockerContainerLogs(blob.jobName) + return Transfer.unknown(logs) + } + } + + @Override + void cleanup(BlobCacheInfo blob) { + final cli = new ArrayList() + cli.add('docker') + cli.add('rm') + cli.add(blob.jobName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + } + + @ToString(includePackage = false, includeNames = true) + @Canonical + static class State { + String status + Integer exitCode + + static State parse(String result) { + final ret = result.tokenize(',') + final status = ret[0] + final exit = ret[1] ? Integer.valueOf(ret[1]) : null + new State(status,exit) + } + } + + private static State getDockerContainerState(String containerName) { + final cli = new ArrayList() + cli.add('docker') + cli.add('inspect') + cli.add('--format') + cli.add('{{.State.Status}},{{.State.ExitCode}}') + cli.add(containerName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + final result = process.inputStream.text.trim() + return State.parse(result) + } + + private static String getDockerContainerLogs(String containerName) { + final cli = new ArrayList() + cli.add('docker') + cli.add('logs') + cli.add(containerName) + + final builder = new ProcessBuilder(cli) + builder.redirectErrorStream(true) + final process = builder.start() + process.waitFor() + process.inputStream.text + } + } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy index 69afe08dd..24da1496b 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy @@ -21,18 +21,18 @@ package io.seqera.wave.service.blob.impl import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService -import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Replaces import io.micronaut.context.annotation.Requires import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.TransferStrategy +import io.seqera.wave.service.blob.transfer.Transfer +import io.seqera.wave.service.blob.transfer.Transfer.Status +import io.seqera.wave.service.blob.transfer.TransferStrategy import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.k8s.K8sService -import io.seqera.wave.util.K8sHelper +import io.seqera.wave.service.k8s.K8sService.JobStatus import jakarta.inject.Inject import jakarta.inject.Named /** @@ -44,7 +44,7 @@ import jakarta.inject.Named @Slf4j @CompileStatic @Requires(property = 'wave.build.k8s') -@Replaces(SimpleTransferStrategy) +@Requires(property = 'wave.blobCache.enabled', value = 'true') class KubeTransferStrategy implements TransferStrategy { @Inject @@ -61,63 +61,59 @@ class KubeTransferStrategy implements TransferStrategy { private ExecutorService executor @Override - BlobCacheInfo transfer(BlobCacheInfo info, List command) { - final jobName = getJobName(info) + void transfer(BlobCacheInfo info, List command) { // run the transfer job - final result = transfer0(info, command, jobName) - // delete job - cleanupJob(jobName, result.exitStatus) - return result + k8sService.transferJob(info.jobName, blobConfig.s5Image, command, blobConfig) } - protected BlobCacheInfo transfer0(BlobCacheInfo info, List command, String jobName) { - final job = k8sService.transferJob(jobName, blobConfig.s5Image, command, blobConfig) - final timeout = Math.round(blobConfig.transferTimeout.toMillis() *1.1f) - final podList = k8sService.waitJob(job, timeout) - final size = podList?.items?.size() ?: 0 - // verify the upload pod has been created - if( size < 1 ) { - log.error "== Blob cache transfer failed - unable to schedule upload job: $info" - return info.failed("Unable to scheduler transfer job") + @Override + void cleanup(BlobCacheInfo blob) { + if( cleanup.shouldCleanup(blob.exitStatus) ) { + CompletableFuture.supplyAsync (() -> k8sService.deleteJob(blob.jobName), executor) } - // Find the latest created pod among the pods associated with the job - final latestPod = K8sHelper.findLatestPod(podList) - - final pod = k8sService.getPod(latestPod.metadata.name) - final exitCode = k8sService.waitPodCompletion(pod, timeout) - final stdout = k8sService.logsPod(pod) - - return exitCode!=null - ? info.completed(exitCode, stdout) - : info.failed(stdout) } - protected void cleanupJob(String jobName, Integer exitCode) { - if( cleanup.shouldCleanup(exitCode) ) { - CompletableFuture.supplyAsync (() -> k8sService.deleteJob(jobName), executor) + @Override + Transfer status(BlobCacheInfo info) { + final status = k8sService.getJobStatus(info.jobName) + if( !status || !status.completed() ) { + return new Transfer(mapToStatus(status)) } - } - protected static String getJobName(BlobCacheInfo info) { - return 'transfer-' + Hashing - .sipHash24() - .newHasher() - .putUnencodedChars(info.locationUri) - .putUnencodedChars(info.creationTime.toString()) - .hash() + // Find the latest created pod among the pods associated with the job + final pod = k8sService.getLatestPodForJob(info.jobName) + if( !pod ) + throw new IllegalStateException("Missing carried pod for job: ${info.jobName}") + + // determine exit code and logs + final exitCode = pod + .status + ?.containerStatuses + ?.first() + ?.state + ?.terminated + ?.exitCode + final stdout = k8sService.logsPod(pod) + return new Transfer(mapToStatus(status), exitCode, stdout) } - private void cleanupPod(String podName, int exitCode) { - if( !cleanup.shouldCleanup(exitCode) ) { - return + /** + * Map Kubernetes job status to Transfer status + * @param jobStatus + * @return + */ + static Status mapToStatus(JobStatus jobStatus) { + switch (jobStatus) { + case JobStatus.Pending: + return Status.PENDING + case JobStatus.Running: + return Status.RUNNING + case JobStatus.Succeeded: + return Status.SUCCEEDED + case JobStatus.Failed: + return Status.FAILED + default: + return Status.UNKNOWN } - - CompletableFuture.supplyAsync (() -> - k8sService.deletePodWhenReachStatus( - podName, - 'Succeeded', - blobConfig.podDeleteTimeout.toMillis()), - executor) } - } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategy.groovy deleted file mode 100644 index 274e4e624..000000000 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategy.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.blob.impl - -import java.util.concurrent.TimeUnit - -import groovy.transform.CompileStatic -import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.TransferStrategy -import jakarta.inject.Inject -/** - * Simple {@link TransferStrategy} implementation that runs - * s5cmd in the local computer. Meant for development purposes - * - * @author Paolo Di Tommaso - */ -@CompileStatic -class SimpleTransferStrategy implements TransferStrategy { - - @Inject - private BlobCacheConfig blobConfig - - @Override - BlobCacheInfo transfer(BlobCacheInfo info, List cli) { - final proc = createProcess(cli).start() - // wait for the completion and save the result - final completed = proc.waitFor(blobConfig.transferTimeout.toSeconds(), TimeUnit.SECONDS) - final int status = completed ? proc.exitValue() : -1 - final logs = proc.inputStream.text - return info.completed(status, logs) - } - - protected ProcessBuilder createProcess(List cli) { - // builder - final builder = new ProcessBuilder() - builder.environment().putAll(blobConfig.getEnvironment()) - builder.command(cli) - builder.redirectErrorStream(true) - return builder - } -} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy new file mode 100644 index 000000000..600aca519 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy @@ -0,0 +1,72 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.transfer + +import groovy.transform.Canonical +import groovy.transform.ToString + +/** + * Model a transfer operation state + * + * @author Paolo Di Tommaso + */ + +@ToString(includePackage = false, includeNames = true) +@Canonical +class Transfer { + + enum Status { PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN } + + final Status status + final Integer exitCode + final String stdout + + final boolean completed() { + return status==Status.SUCCEEDED || status==Status.FAILED + } + + final boolean succeeded() { + status==Status.SUCCEEDED && exitCode==0 + } + + static Transfer pending() { + return new Transfer(Status.PENDING) + } + + static Transfer running() { + return new Transfer(Status.RUNNING) + } + + static Transfer failed(Integer exit, String logs) { + return new Transfer(Status.FAILED, exit, logs) + } + + static Transfer succeeded(String logs) { + return new Transfer(Status.SUCCEEDED, 0, logs) + } + + static Transfer completed(Integer exit, String logs) { + final st = exit==0 ? Status.SUCCEEDED : Status.FAILED + return new Transfer(st, exit, logs) + } + + static Transfer unknown(String logs) { + return new Transfer(Status.UNKNOWN,null,logs) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy new file mode 100644 index 000000000..acae180e0 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy @@ -0,0 +1,163 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.transfer + +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Context +import io.micronaut.context.annotation.Requires +import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.impl.BlobCacheStore +import io.seqera.wave.util.ExponentialAttempt +import jakarta.annotation.PostConstruct +import jakarta.inject.Inject +import jakarta.inject.Named +/** + * Implement the logic to handle Blob cache transfer (uploads) + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Context +@CompileStatic +@Requires(property = 'wave.blobCache.enabled', value = 'true') +class TransferManager { + + @Inject + private TransferStrategy transferStrategy + + @Inject + private BlobCacheStore blobStore + + @Inject + private BlobCacheConfig blobConfig + + @Inject + private TransferQueue queue + + @Inject + @Named(TaskExecutors.IO) + private ExecutorService executor + + private final ExponentialAttempt attempt = new ExponentialAttempt() + + @PostConstruct + private init() { + CompletableFuture.supplyAsync(()->run(), executor) + } + + void run() { + log.info "+ Starting Blob cache transfer manager" + while( !Thread.currentThread().isInterrupted() ) { + try { + final transferId = queue.poll(blobConfig.statusDelay) + + if( transferId ) { + handle(transferId) + attempt.reset() + } + } + catch (InterruptedException e) { + log.debug "Interrupting transfer manager watcher thread" + break + } + catch (Throwable e) { + final d0 = attempt.delay() + log.error("Transfer manager unexpected error (await: ${d0}) - cause: ${e.message}", e) + sleep(d0.toMillis()) + } + } + } + + /** + * Handles the blob transfer operation i.e. check and update the current upload status + * + * @param blobId the blob cache id i.e. {@link BlobCacheInfo#id()} + */ + protected void handle(String blobId) { + try { + final blob = blobStore.get(blobId) + if( !blob ) { + log.error "Unknown blob transfer with id: $blobId" + return + } + try { + handle0(blob) + } + catch (Throwable t) { + log.error("Unexpected error caching blob '${blob.objectUri}' - job name '${blob.jobName}", t) + blobStore.put(blobId, blob.failed("Unexpected error caching blob '${blob.locationUri}' - job name '${blob.jobName}'")) + } + } + catch (InterruptedException e) { + // re-queue the transfer to not lose it + queue.offer(blobId) + // re-throw the exception + throw e + } + } + + protected void handle0(BlobCacheInfo info) { + final duration = Duration.between(info.creationTime, Instant.now()) + final transfer = transferStrategy.status(info) + log.trace "Blob cache transfer name=${info.jobName}; state=${transfer}; object=${info.objectUri}" + final done = + transfer.completed() || + // considered failed when remain in unknown status too long + (transfer.status==Transfer.Status.UNKNOWN && duration>blobConfig.graceDuration) + if( done ) { + // use a short time-to-live for failed downloads + // this is needed to allow re-try caching of failure transfers + final ttl = transfer.succeeded() + ? blobConfig.statusDuration + : blobConfig.failureDuration + // update the blob status + final result = transfer.succeeded() + ? info.completed(transfer.exitCode, transfer.stdout) + : info.failed(transfer.stdout) + blobStore.storeBlob(info.id(), result, ttl) + log.debug "== Blob cache completed for object '${info.objectUri}'; id=${info.objectUri}; status=${result.exitStatus}; duration=${result.duration()}" + // finally cleanup the job + transferStrategy.cleanup(result) + return + } + // set the await timeout nearly double as the blob transfer timeout, this because the + // transfer pod can spend `timeout` time in pending status awaiting to be scheduled + // and the same `timeout` time amount carrying out the transfer (upload) operation + final max = (blobConfig.transferTimeout.toMillis() * 2.10) as long + if( duration.toMillis()>max ) { + final result = info.failed("Blob cache transfer timed out - id: ${info.objectUri}; object: ${info.objectUri}") + log.warn "== Blob cache completed for object '${info.objectUri}'; id=${info.objectUri}; duration=${result.duration()}" + blobStore.storeBlob(info.id(), result, blobConfig.failureDuration) + } + else { + log.trace "== Blob cache pending for completion $info" + // re-schedule for a new check + queue.offer(info.id()) + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy new file mode 100644 index 000000000..183a0cbaf --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy @@ -0,0 +1,48 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.transfer + +import java.time.Duration + +import io.seqera.wave.service.data.queue.MessageQueue +import jakarta.inject.Inject +import jakarta.inject.Singleton + +/** + * Implements a simple persistent FIFO queue + * + * @author Paolo Di Tommaso + */ +@Singleton +class TransferQueue { + + final private static String QUEUE_NAME = 'transfer-queue/v1' + + @Inject + private MessageQueue transferQueue + + void offer(String transferId) { + transferQueue.offer(QUEUE_NAME, transferId) + } + + String poll(Duration timeout) { + transferQueue.poll(QUEUE_NAME, timeout) + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy similarity index 80% rename from src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy rename to src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy index d7669185d..08452b64f 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/TransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy @@ -16,7 +16,10 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob +package io.seqera.wave.service.blob.transfer + + +import io.seqera.wave.service.blob.BlobCacheInfo /** * Defines the contract to transfer a layer blob into a remote object storage * @@ -24,6 +27,10 @@ package io.seqera.wave.service.blob */ interface TransferStrategy { - BlobCacheInfo transfer(BlobCacheInfo info, List command) + void transfer(BlobCacheInfo blob, List command) + + Transfer status(BlobCacheInfo blob) + + void cleanup(BlobCacheInfo blob) } diff --git a/src/main/groovy/io/seqera/wave/service/blob/TransferTimeoutException.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy similarity index 96% rename from src/main/groovy/io/seqera/wave/service/blob/TransferTimeoutException.groovy rename to src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy index 1debd74aa..d16412a2e 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/TransferTimeoutException.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob +package io.seqera.wave.service.blob.transfer import groovy.transform.CompileStatic import io.seqera.wave.exception.WaveException diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy index 7eaaeb5b1..f93c4daaf 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/impl/RedisMessageQueue.groovy @@ -61,7 +61,8 @@ class RedisMessageQueue implements MessageQueue { String poll(String target, Duration duration) { try (Jedis conn = pool.getResource()) { double d = duration.toMillis() / 1000.0 - return conn.brpop(d, target) + final entry = conn.brpop(d, target) + return entry ? entry.getValue() : null } } } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 915f5f313..1ecb602d9 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -33,13 +33,7 @@ import io.seqera.wave.configuration.SpackConfig */ interface K8sService { - enum JobStatus { Pending, Running, Succeeded, Failed } - - V1Job createJob(String name, String containerImage, List args) - - V1Job getJob(String name) - - JobStatus getJobStatus(String name) + enum JobStatus { Pending, Running, Succeeded, Failed; boolean completed() { return this == Succeeded || this == Failed } } V1Pod getPod(String name) @@ -53,12 +47,20 @@ interface K8sService { Integer waitPodCompletion(V1Pod pod, long timeout) + void deletePodWhenReachStatus(String podName, String statusName, long timeout) + + V1Job createJob(String name, String containerImage, List args) + + V1Job getJob(String name) + + JobStatus getJobStatus(String name) + void deleteJob(String name) V1Job transferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) V1PodList waitJob(V1Job job, Long timeout) - void deletePodWhenReachStatus(String podName, String statusName, long timeout) + V1Pod getLatestPodForJob(String jobName) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index b7d96dafc..21a258222 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -199,7 +199,7 @@ class K8sServiceImpl implements K8sService { */ @Override JobStatus getJobStatus(String name) { - def job = k8sClient + final job = k8sClient .batchV1Api() .readNamespacedJob(name, namespace, null) if( !job ) @@ -678,4 +678,23 @@ class K8sServiceImpl implements K8sService { .deleteNamespacedJob(name, namespace, null, null, null, null,"Foreground", null) } + @Override + V1Pod getLatestPodForJob(String jobName) { + // list all pods for the given job + final allPods = k8sClient + .coreV1Api() + .listNamespacedPod(namespace, null, null, null, null, "job-name=${jobName}", null, null, null, null, null, null) + + if( !allPods ) + return null + + // Find the latest created pod among the pods associated with the job + def latest = allPods.getItems().get(0) + for (def pod : allPods.items) { + if (pod.metadata?.creationTimestamp?.isAfter(latest.metadata.creationTimestamp)) { + latest = pod + } + } + return latest + } } diff --git a/src/main/groovy/io/seqera/wave/service/stream/StreamServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/stream/StreamServiceImpl.groovy index 78da8edfa..e2b1c69ee 100644 --- a/src/main/groovy/io/seqera/wave/service/stream/StreamServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/stream/StreamServiceImpl.groovy @@ -114,8 +114,8 @@ class StreamServiceImpl implements StreamService { } else { // this approach should be avoided because it can put too much pressure on the wave backend - // this is only meant to be used when the 'BlobCacheService' is not aivalable - log.debug "Streaming flux for route: $route" + // this is only meant to be used when the 'BlobCacheService' is not available + log.warn "Streaming flux for route: $route - Streaming is not scalable. Blob cache service should be configured instead." final result = proxyService.streamBlob(route, Map.>of()) return fluxToInputStream(result); } diff --git a/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy b/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy index 336890e79..032709b6d 100644 --- a/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/K8sHelper.groovy @@ -19,11 +19,8 @@ package io.seqera.wave.util import groovy.transform.CompileStatic -import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException - /** * * @author Paolo Di Tommaso @@ -75,14 +72,4 @@ class K8sHelper { return Map.of(parts[0], parts[1]) } - static V1Pod findLatestPod(V1PodList allPods) { - // Find the latest created pod among the pods associated with the job - def latest = allPods.getItems().get(0) - for (def pod : allPods.items) { - if (pod.metadata?.creationTimestamp?.isAfter(latest.metadata.creationTimestamp)) { - latest = pod - } - } - return latest - } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy index 15984ee6b..fb54a105b 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy @@ -31,21 +31,23 @@ import java.time.Instant class BlobCacheInfoTest extends Specification { def 'should create blob info' () { + when: + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + then: + blob.locationUri == 'http://foo.com' + blob.objectUri == 's3://foo/com' + blob.headers == [:] + blob.id() == 's3://foo/com' + expect: - BlobCacheInfo.create('http://foo.com', [:], [:]) - .locationUri == 'http://foo.com' - and: - BlobCacheInfo.create('http://foo.com', [:], [:]) - .headers == [:] - and: - BlobCacheInfo.create('http://foo.com', [Foo:['alpha'], Bar:['delta', 'gamma', 'omega']], [:]) + BlobCacheInfo.create('http://foo.com', 's3://foo/com', [Foo:['alpha'], Bar:['delta', 'gamma', 'omega']], [:]) .headers == [Foo:'alpha', Bar: 'delta,gamma,omega'] } def 'should find content type' () { expect: - BlobCacheInfo.create('http://foo', [:], HEADERS ).getContentType() == EXPECTED + BlobCacheInfo.create('http://foo', 's3://foo/com', [:], HEADERS ).getContentType() == EXPECTED where: HEADERS | EXPECTED @@ -57,7 +59,7 @@ class BlobCacheInfoTest extends Specification { def 'should find cache control' () { expect: - BlobCacheInfo.create('http://foo', [:], HEADERS ).getCacheControl() == EXPECTED + BlobCacheInfo.create('http://foo', 's3://foo/com', [:], HEADERS ).getCacheControl() == EXPECTED where: HEADERS | EXPECTED @@ -69,7 +71,7 @@ class BlobCacheInfoTest extends Specification { def 'should find content length' () { expect: - BlobCacheInfo.create('http://foo', [:], HEADERS ).getContentLength() == EXPECTED + BlobCacheInfo.create('http://foo', 's3://foo/com', [:], HEADERS ).getContentLength() == EXPECTED where: HEADERS | EXPECTED @@ -82,16 +84,18 @@ class BlobCacheInfoTest extends Specification { def 'should complete blob info' () { given: - def location = 'http://foo.com' + String location = 'http://foo.com' + String object = 's3://foo/bar' def headers = [Foo:['something']] def response = ['Content-Length':['100'], 'Content-Type':['text'], 'Cache-Control': ['12345']] - def cache = BlobCacheInfo.create(location, headers, response) + def cache = BlobCacheInfo.create(location, object, headers, response) when: def result = cache.completed(0, 'OK') then: result.headers == [Foo:'something'] result.locationUri == 'http://foo.com' + result.objectUri == 's3://foo/bar' result.creationTime == cache.creationTime result.completionTime >= cache.creationTime result.exitStatus == 0 @@ -100,6 +104,8 @@ class BlobCacheInfoTest extends Specification { result.contentType == 'text' result.cacheControl == '12345' and: + result.id() == 's3://foo/bar' + and: result.done() result.succeeded() @@ -108,6 +114,7 @@ class BlobCacheInfoTest extends Specification { then: result.headers == [Foo:'something'] result.locationUri == 'http://foo.com' + result.objectUri == 's3://foo/bar' result.creationTime == cache.creationTime result.completionTime >= cache.creationTime result.exitStatus == 1 @@ -115,6 +122,8 @@ class BlobCacheInfoTest extends Specification { result.contentType == 'text' result.cacheControl == '12345' and: + result.id() == 's3://foo/bar' + and: result.done() !result.succeeded() } @@ -122,15 +131,17 @@ class BlobCacheInfoTest extends Specification { def 'should fail blob info' () { given: def location = 'http://foo.com' + def object = 's3://foo/bar' def headers = [Foo:['something']] def response = ['Content-Length':['100'], 'Content-Type':['text'], 'Cache-Control': ['12345']] - def cache = BlobCacheInfo.create(location, headers, response) + def cache = BlobCacheInfo.create(location, object, headers, response) when: def result = cache.failed('Oops') then: result.headers == [Foo:'something'] result.locationUri == 'http://foo.com' + result.objectUri == 's3://foo/bar' result.creationTime == cache.creationTime result.completionTime >= cache.creationTime result.exitStatus == null @@ -139,6 +150,8 @@ class BlobCacheInfoTest extends Specification { result.contentType == 'text' result.cacheControl == '12345' and: + result.id() == 's3://foo/bar' + and: result.done() !result.succeeded() } @@ -146,15 +159,17 @@ class BlobCacheInfoTest extends Specification { def 'should cache blob info' () { given: def location = 'http://foo.com' + def object = 's3://foo/bar' def headers = [Foo:['something']] def response = ['Content-Length':['100'], 'Content-Type':['text'], 'Cache-Control': ['12345']] and: - def cache = BlobCacheInfo.create(location, headers, response) + def cache = BlobCacheInfo.create(location, object, headers, response) when: def result = cache.cached() then: result.headers == [Foo:'something'] result.locationUri == 'http://foo.com' + result.objectUri == 's3://foo/bar' result.creationTime == cache.creationTime result.completionTime == cache.creationTime result.exitStatus == 0 @@ -163,20 +178,25 @@ class BlobCacheInfoTest extends Specification { result.contentType == 'text' result.contentLength == 100L and: + result.id() == 's3://foo/bar' + and: result.done() result.succeeded() } def 'should unknown blob info' () { given: - def result = BlobCacheInfo.unknown() + def result = BlobCacheInfo.unknown('Foo bar') expect: result.headers == null result.locationUri == null + result.objectUri == null result.creationTime == Instant.ofEpochMilli(0) result.completionTime == Instant.ofEpochMilli(0) result.exitStatus == null - result.logs == null + result.logs == 'Foo bar' + and: + result.id() == null and: !result.done() !result.succeeded() @@ -191,19 +211,25 @@ class BlobCacheInfoTest extends Specification { def response = ['Content-Length':['100'], 'Content-Type':['text'], 'Cache-Control': ['12345']] when: - def result1 = BlobCacheInfo.create('http://foo.com', headers, response) + def result1 = BlobCacheInfo.create('http://foo.com', 's3://foo/bar', headers, response) then: result1.locationUri == 'http://foo.com' + result1.objectUri == 's3://foo/bar' and: result1.headers == [Foo:'something'] result1.contentType == 'text' result1.contentLength == 100L result1.cacheControl == '12345' + result1.id() == 's3://foo/bar' + and: when: def result2 = result1.withLocation('http://bar.com') then: result2.locationUri == 'http://bar.com' + result2.objectUri == 's3://foo/bar' + and: + result2.id() == 's3://foo/bar' and: result2.headers == [Foo:'something'] result2.contentType == 'text' @@ -222,11 +248,15 @@ class BlobCacheInfoTest extends Specification { null, null, null, + null, + null, CREATE, COMPLETE ) expect: info.duration() == EXPECTED + and: + info.id() == null where: CREATE | COMPLETE | EXPECTED diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy index e7d8a5b31..a8f8f07f4 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImplTest.groovy @@ -52,7 +52,7 @@ class BlobCacheServiceImplTest extends Specification implements AwsS3TestContain result == ['s5cmd', '--json', 'pipe', 's3://store/blobs/docker.io/v2/library/ubuntu/manifests/sha256:aabbcc'] when: - result = service.s5cmd(route, BlobCacheInfo.create('http://foo', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar']])) + result = service.s5cmd(route, BlobCacheInfo.create('http://foo', 'http://bar', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar']])) then: result == ['s5cmd', '--json', 'pipe', '--content-type', 'foo', '--cache-control', 'bar', 's3://store/blobs/docker.io/v2/library/ubuntu/manifests/sha256:aabbcc'] @@ -77,7 +77,7 @@ class BlobCacheServiceImplTest extends Specification implements AwsS3TestContain def route = RoutePath.v2manifestPath(ContainerCoordinates.parse('ubuntu@sha256:aabbcc')) and: def response = ['content-type': ['something']] - def blobCache = BlobCacheInfo.create('http://foo', ['foo': ['one']], response) + def blobCache = BlobCacheInfo.create('http://foo','http://bar', ['foo': ['one']], response) when: def result = service.transferCommand(route, blobCache) @@ -91,114 +91,4 @@ class BlobCacheServiceImplTest extends Specification implements AwsS3TestContain ] } - def 'should return blob size when blob exists'() { - given: - def bucket = 's3://my-cache-bucket' - def expectedSize = 1024 - def s3Client = Mock(S3Client) - def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) - and: - def route = Mock(RoutePath) { - getTargetPath() >> 'docker.io/repo/container/latest' - } - and: - final request = - HeadObjectRequest.builder() - .bucket('my-cache-bucket') - .key('docker.io/repo/container/latest') - .build() - - when: - def size = blobCacheService.getBlobSize(route) - - then: - 1 * s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(expectedSize).build() - and: - size == expectedSize - } - - def 'should return zero when blob does not exist'() { - given: - def bucket = 's3://my-cache-bucket' - def s3Client = Mock(S3Client) - def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) - and: - def route = Mock(RoutePath) { - getTargetPath() >> 'docker.io/repo/container/latest' - } - and: - final request = - HeadObjectRequest.builder() - .bucket('my-cache-bucket') - .key('docker.io/repo/container/latest') - .build() - - when: - def size = blobCacheService.getBlobSize(route) - - then: - 1 * s3Client.headObject(request) >> { throw S3Exception.builder().message('Not Found').build() } - and: - size == -1L - } - - def 'should delete blob when blob exists'() { - given: - def bucket = 's3://my-cache-bucket/base/dir' - def s3Client = Mock(S3Client) - def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: bucket)) - and: - def route = Mock(RoutePath) { - getTargetPath() >> 'docker.io/repo/container/latest' - } - and: - def request = DeleteObjectRequest.builder() - .bucket('my-cache-bucket') - .key('base/dir/docker.io/repo/container/latest') - .build() - - when: - blobCacheService.deleteBlob(route) - then: - 1 * s3Client.deleteObject(request) >> { } - } - - def 'should return failed BlobCacheInfo when blob size mismatch'() { - given: - def executor = Mock(ExecutorService) - def s3Client = Mock(S3Client) - s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(1234L).build() - def blobStore = Mock(BlobStore) - def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: 's3://store/blobs/'), blobStore: blobStore, executor: executor, ) - def route = RoutePath.v2manifestPath(ContainerCoordinates.parse('ubuntu@sha256:aabbcc')) - def info = BlobCacheInfo.create('http://foo', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar'], 'Content-Length': ['4321']]) - info = info.completed(0, 'Blob uploaded') - - when: - def result = blobCacheService.checkUploadedBlobSize(info, route) - - then: - !result.succeeded() - result.logs == "Mismatch cache size for object http://foo" - } - - def 'should return succeeded BlobCacheInfo when blob size matches'() { - given: - def executor = Mock(ExecutorService) - def s3Client = Mock(S3Client) - s3Client.headObject(_) >> HeadObjectResponse.builder().contentLength(4321L).build() - def blobStore = Mock(BlobStore) - def blobCacheService = new BlobCacheServiceImpl(s3Client: s3Client, blobConfig: new BlobCacheConfig(storageBucket: 's3://store/blobs/'), blobStore: blobStore, executor: executor) - def route = RoutePath.v2manifestPath(ContainerCoordinates.parse('ubuntu@sha256:aabbcc')) - def info = BlobCacheInfo.create('http://foo', [:], ['Content-Type':['foo'], 'Cache-Control': ['bar'], 'Content-Length': ['4321']]) - info = info.completed(0, 'Blob uploaded') - - when: - def result = blobCacheService.checkUploadedBlobSize(info, route) - - then: - result.succeeded() - result.logs == "Blob uploaded" - } - } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy index adf5582f7..b361fcde2 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy @@ -19,9 +19,9 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification +import spock.lang.Unroll import io.seqera.wave.configuration.BlobCacheConfig - /** * * @author Paolo Di Tommaso @@ -41,12 +41,16 @@ class DockerTransferStrategyTest extends Specification { def strategy = new DockerTransferStrategy(blobConfig: config) when: - def result = strategy.createProcess(['s5cmd', 'run', '--this']) + def result = strategy.createProcess(['s5cmd', 'run', '--this'], "job-name", 10) then: result.command() == [ 'docker', 'run', + '--name', + 'job-name', + '--stop-timeout', + '10', '-e', 'AWS_ACCESS_KEY_ID', '-e', 'AWS_SECRET_ACCESS_KEY', 'cr.seqera.io/public/s5cmd:latest', @@ -60,4 +64,18 @@ class DockerTransferStrategyTest extends Specification { and: result.redirectErrorStream() } + + @Unroll + def 'should parse state string' () { + expect: + DockerTransferStrategy.State.parse(STATE) == EXPECTED + + where: + STATE | EXPECTED + 'running' | new DockerTransferStrategy.State('running') + 'exited' | new DockerTransferStrategy.State('exited') + 'exited,' | new DockerTransferStrategy.State('exited') + 'exited,0' | new DockerTransferStrategy.State('exited', 0) + 'exited,10' | new DockerTransferStrategy.State('exited', 10) + } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy index 33f8320d9..54fff307a 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -19,11 +19,15 @@ package io.seqera.wave.service.blob.impl import spock.lang.Specification +import spock.lang.Unroll import java.time.Duration import java.time.OffsetDateTime import java.util.concurrent.Executors +import io.kubernetes.client.openapi.models.V1ContainerState +import io.kubernetes.client.openapi.models.V1ContainerStateTerminated +import io.kubernetes.client.openapi.models.V1ContainerStatus import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList @@ -31,6 +35,8 @@ import io.kubernetes.client.openapi.models.V1PodStatus import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.transfer.Transfer +import io.seqera.wave.service.k8s.K8sService.JobStatus import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.k8s.K8sService /** @@ -44,9 +50,9 @@ class KubeTransferStrategyTest extends Specification { CleanupStrategy cleanup = new CleanupStrategy(buildConfig: new BuildConfig(cleanup: "OnSuccess")) KubeTransferStrategy strategy = new KubeTransferStrategy(k8sService: k8sService, blobConfig: blobConfig, cleanup: cleanup, executor: Executors.newSingleThreadExecutor()) - def "transfer should return completed info when job is successful"() { + def "transfer should start a transferJob"() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", null, null) + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) def command = ["transfer", "blob"] final jobName = "job-123" def podName = "$jobName-abc" @@ -60,37 +66,81 @@ class KubeTransferStrategyTest extends Specification { k8sService.logsPod(_) >> "transfer successful" when: - def result = strategy.transfer(info, command) + strategy.transfer(info, command) then: - result.exitStatus == 0 - result.logs == "transfer successful" - result.done() - result.succeeded() + 1 * k8sService.transferJob(info.jobName, blobConfig.s5Image, command, blobConfig) } - def "transfer should return failed info when job is failed"() { + def 'status should return correct status when job is not completed'() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", null, null) - def command = ["transfer", "blob"] - final jobName = "job-123" - def podName = "$jobName-abc" - def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) - pod.status = new V1PodStatus(phase: "Succeeded") - def podList = new V1PodList(items: [pod]) - k8sService.transferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) - k8sService.waitJob(_, _) >> podList - k8sService.getPod(_) >> pod - k8sService.waitPodCompletion(pod, _) >> 1 - k8sService.logsPod(pod) >> "transfer failed" + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + k8sService.getJobStatus(info.jobName) >> K8sService.JobStatus.Running + + when: + def result = strategy.status(info) + + then: + result.status == Transfer.Status.RUNNING + } + + + void 'status should return correct transfer status when pods are created'() { + given: + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + def status = new V1PodStatus(phase: "Succeeded", containerStatuses: [new V1ContainerStatus( state: new V1ContainerState(terminated: new V1ContainerStateTerminated(exitCode: 0)))]) + def pod = new V1Pod(metadata: [name: "pod-123"], status: status) + k8sService.getJobStatus(_) >> K8sService.JobStatus.Succeeded + k8sService.logsPod(_) >> "transfer successful" + k8sService.getLatestPodForJob(info.jobName) >> pod when: - def result = strategy.transfer(info, command) + def result = strategy.status(info) then: - result.exitStatus == 1 - result.logs == "transfer failed" - result.done() - !result.succeeded() + result.status == Transfer.Status.SUCCEEDED + result.exitCode == 0 + result.stdout == "transfer successful" + } + + def 'status should return failed transfer when no pods are created'() { + given: + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + def status = new V1PodStatus(phase: "Failed") + def pod = new V1Pod(metadata: [name: "pod-123"], status: status) + k8sService.getJobStatus(info.jobName) >> K8sService.JobStatus.Failed + k8sService.getLatestPodForJob(info.jobName) >> pod + + when: + def result = strategy.status(info) + + then: + result.status == Transfer.Status.FAILED + } + + def 'status should handle null job status'() { + given: + def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + k8sService.getJobStatus(info.id()) >> null + + when: + def result = strategy.status(info) + + then: + result.status == Transfer.Status.UNKNOWN + } + + @Unroll + def "mapToStatus should return correct transfer status for jobStatus #JOB_STATUS that is #TRANSFER_STATUS"() { + expect: + KubeTransferStrategy.mapToStatus(JOB_STATUS) == TRANSFER_STATUS + + where: + JOB_STATUS | TRANSFER_STATUS + JobStatus.Pending | Transfer.Status.PENDING + JobStatus.Running | Transfer.Status.RUNNING + JobStatus.Succeeded | Transfer.Status.SUCCEEDED + JobStatus.Failed | Transfer.Status.FAILED + null | Transfer.Status.UNKNOWN } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategyTest.groovy deleted file mode 100644 index 21d238ba0..000000000 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/SimpleTransferStrategyTest.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.blob.impl - -import spock.lang.Specification - -import io.seqera.wave.configuration.BlobCacheConfig - -/** - * - * @author Paolo Di Tommaso - */ -class SimpleTransferStrategyTest extends Specification { - - def 'should create transfer cli' () { - given: - def config = new BlobCacheConfig( - storageBucket: 's3://foo', - storageEndpoint: 'https://foo.com', - storageRegion: 'some-region', - storageAccessKey: 'xyz', - storageSecretKey: 'secret' - ) - def strategy = new SimpleTransferStrategy(blobConfig: config) - - when: - def result = strategy.createProcess(['s5cmd', 'run', '--this']) - then: - result.command() == ['s5cmd', 'run', '--this'] - result.redirectErrorStream() - and: - def env = result.environment() - env.AWS_REGION == 'some-region' - env.AWS_DEFAULT_REGION == 'some-region' - env.AWS_ACCESS_KEY_ID == 'xyz' - env.AWS_SECRET_ACCESS_KEY == 'secret' - } -} diff --git a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy new file mode 100644 index 000000000..71681a23e --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy @@ -0,0 +1,143 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.transfer + +import spock.lang.Specification + +import java.time.Duration + +import io.seqera.wave.configuration.BlobCacheConfig +import io.seqera.wave.service.blob.BlobCacheInfo +import io.seqera.wave.service.blob.impl.BlobCacheStore +/** + * + * @author Munish Chouhan + */ +class TransferManagerTest extends Specification { + + def "handle should process valid transferId"() { + given: + def blobStore = Mock(BlobCacheStore) + def queue = Mock(TransferQueue) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = Mock(BlobCacheConfig) + def manager = new TransferManager(blobStore: blobStore, queue: queue, transferStrategy: transferStrategy, blobConfig: blobConfig) + + and: + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + + when: + manager.handle(blob.id()) + + then: + 1 * blobStore.get(blob.id()) >> blob + 1 * transferStrategy.status(blob) >> Transfer.completed(0, 'logs') + } + + def "handle should log error for unknown transferId"() { + given: + def transferId = 'unknown' + def blobStore = Mock(BlobCacheStore) + def queue = Mock(TransferQueue) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = Mock(BlobCacheConfig) + def manager = new TransferManager(blobStore: blobStore, queue: queue, transferStrategy: transferStrategy, blobConfig: blobConfig) + + when: + manager.handle(transferId) + + then: + 1 * blobStore.get(transferId) >> null + 0 * manager.handle0(_) + } + + def "handle0 should complete transfer when status is completed"() { + given: + def blobStore = Mock(BlobCacheStore) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = Mock(BlobCacheConfig) + def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) + def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + def transfer = Transfer.succeeded('logs') + blobConfig.statusDuration >> Duration.ofMinutes(5) + transferStrategy.status(blob) >> transfer + + when: + manager.handle0(blob) + + then: + 1 * blobStore.storeBlob(blob.id(), _, blobConfig.statusDuration) + 1 * transferStrategy.cleanup(_) + } + + def "handle0 should fail transfer when status is unknown and duration exceeds grace period"() { + given: + def blobStore = Mock(BlobCacheStore) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1), graceDuration: Duration.ofSeconds(1)) + def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) + def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + def transfer = Transfer.unknown('logs') + transferStrategy.status(info) >> transfer + + when: + sleep 1_000 //sleep for grace period + manager.handle0(info) + + then: + 1 * blobStore.storeBlob(info.id(), _, blobConfig.failureDuration) + 1 * transferStrategy.cleanup(_) + } + + def "handle0 should requeue transfer when duration is within limits"() { + given: + def blobStore = Mock(BlobCacheStore) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1)) + def queue = Mock(TransferQueue) + def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig, queue: queue) + def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + def transfer = Transfer.running() + transferStrategy.status(info) >> transfer + + when: + manager.handle0(info) + + then: + 1 * queue.offer(info.id()) + } + + def "handle0 should timeout transfer when duration exceeds max limit"() { + given: + def blobStore = Mock(BlobCacheStore) + def transferStrategy = Mock(TransferStrategy) + def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1)) + def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) + def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) + def transfer = Transfer.running() + transferStrategy.status(info) >> transfer + + when: + sleep 1_100 * 2 //await timeout + manager.handle0(info) + + then: + 1 * blobStore.storeBlob(info.id(), _, blobConfig.failureDuration) + } +} diff --git a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy new file mode 100644 index 000000000..f8b3e615f --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy @@ -0,0 +1,61 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.blob.transfer + +import spock.lang.Specification +import spock.lang.Unroll + +/** + * + * @author Paolo Di Tommaso + */ +class TransferTest extends Specification { + + @Unroll + def 'should validate completed status' () { + expect: + new Transfer(STATUS).completed() == EXPECTED + + where: + STATUS | EXPECTED + Transfer.Status.PENDING | false + Transfer.Status.RUNNING | false + Transfer.Status.UNKNOWN | false + and: + Transfer.Status.SUCCEEDED | true + Transfer.Status.FAILED | true + } + + @Unroll + def 'should validate succeeded status' () { + expect: + new Transfer(STATUS, EXIT).succeeded() == EXPECTED + + where: + STATUS | EXIT | EXPECTED + Transfer.Status.PENDING | null | false + Transfer.Status.RUNNING | null | false + Transfer.Status.UNKNOWN | null | false + Transfer.Status.FAILED | null | false + Transfer.Status.SUCCEEDED | 1 | false + Transfer.Status.SUCCEEDED | 0 | true + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy index 6361907ce..84548ea2f 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/LocalMessageQueueTest.groovy @@ -47,7 +47,7 @@ class LocalMessageQueueTest extends Specification { and: def delta = System.currentTimeMillis()-start then: - assert delta>500 + assert delta>=500 assert delta<1000 } diff --git a/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy index 87b347146..80286d78a 100644 --- a/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/data/queue/RedisMessageQueueTest.groovy @@ -76,4 +76,17 @@ class RedisMessageQueueTest extends Specification implements RedisTestContainer broker.poll('bar') == 'beta' } + def 'should offer and poll a value after wait' () { + given: + def broker = context.getBean(RedisMessageQueue) + def wait = Duration.ofMillis(500) + and: + broker.offer('bar1', 'alpha1') + broker.offer('bar1', 'beta1') + + expect: + broker.poll('foo1', wait) == null + broker.poll('bar1', wait) == 'alpha1' + broker.poll('bar1', wait) == 'beta1' + } } diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index c715a1cf2..6b45d4bd4 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -22,18 +22,23 @@ import spock.lang.Specification import java.nio.file.Path import java.time.Duration +import java.time.OffsetDateTime import io.kubernetes.client.custom.Quantity import io.kubernetes.client.openapi.ApiClient import io.kubernetes.client.openapi.apis.CoreV1Api import io.kubernetes.client.openapi.models.V1EnvVar +import io.kubernetes.client.openapi.models.V1ObjectMeta import io.kubernetes.client.openapi.models.V1Pod +import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1PodStatus import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.SpackConfig +import io.seqera.wave.util.K8sHelper + /** * * @author Paolo Di Tommaso @@ -634,4 +639,55 @@ class K8sServiceImplTest extends Specification { 0 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) } + def "getLatestPodForJob should return the latest pod when multiple pods are present"() { + given: + def jobName = "test-job" + def pod1 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now().minusDays(1))) + def pod2 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now())) + def allPods = new V1PodList().items(Arrays.asList(pod1, pod2)) + def api = Mock(CoreV1Api) + api.listNamespacedPod(_, _, _, _, _, "job-name=${jobName}", _, _, _, _, _, _) >> allPods + def k8sClient = new K8sClient() { + @Override + ApiClient apiClient() { + return null + } + CoreV1Api coreV1Api() { + return api + } + } + and: + def k8sService = new K8sServiceImpl(k8sClient: k8sClient) + + when: + def latestPod = k8sService.getLatestPodForJob(jobName) + + then: + latestPod == pod2 + } + + def "getLatestPodForJob should return null when no pod is present"() { + given: + def jobName = "test-job" + def api = Mock(CoreV1Api) + api.listNamespacedPod(_, _, _, _, _, "job-name=${jobName}", _, _, _, _, _, _) >> null + def k8sClient = new K8sClient() { + @Override + ApiClient apiClient() { + return null + } + CoreV1Api coreV1Api() { + return api + } + } + and: + def k8sService = new K8sServiceImpl(k8sClient: k8sClient) + + when: + def latestPod = k8sService.getLatestPodForJob(jobName) + + then: + latestPod == null + } + } diff --git a/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy index 4c606d0d9..d9507e3e6 100644 --- a/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/K8sHelperTest.groovy @@ -20,14 +20,8 @@ package io.seqera.wave.util import spock.lang.Specification -import java.time.OffsetDateTime - -import io.kubernetes.client.openapi.models.V1ObjectMeta -import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException - /** * * @author Paolo Di Tommaso @@ -60,29 +54,4 @@ class K8sHelperTest extends Specification { err.message == "Unsupported container platform 'linux/amd64'" } - def "should return the latest pod when multiple pods are present"() { - given: - def pod1 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now().minusDays(1))) - def pod2 = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now())) - def allPods = new V1PodList().items(Arrays.asList(pod1, pod2)) - - when: - def latestPod = K8sHelper.findLatestPod(allPods) - - then: - latestPod == pod2 - } - - def "should return the only pod when one pod is present"() { - given: - def pod = new V1Pod().metadata(new V1ObjectMeta().creationTimestamp(OffsetDateTime.now())) - def allPods = new V1PodList().items(Collections.singletonList(pod)) - - when: - def latestPod = K8sHelper.findLatestPod(allPods) - - then: - latestPod == pod - } - } From 63b58088aa0ba0613cbedd9feff901923d814bf3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 18 Aug 2024 09:30:13 +0200 Subject: [PATCH 36/50] Add trusted builds timeout (#600) This commit adds a the setting `wave.build.trusted-timeout` that defines the max duration for jobs for container build when freeze is enabled and the user identity is know. Signed-off-by: Paolo Di Tommaso --- .../wave/configuration/BuildConfig.groovy | 37 ++- .../controller/ContainerController.groovy | 5 +- .../service/builder/BuildCacheStore.groovy | 22 +- .../wave/service/builder/BuildRequest.groovy | 36 ++- .../wave/service/builder/BuildStore.groovy | 4 - .../builder/DockerBuildStrategy.groovy | 3 +- .../service/builder/KubeBuildStrategy.groovy | 5 +- .../seqera/wave/service/k8s/K8sService.groovy | 5 +- .../wave/service/k8s/K8sServiceImpl.groovy | 11 +- .../wave/controller/BuildConfigTest.groovy | 52 ++++ .../controller/BuildControllerTest.groovy | 32 +-- .../RegistryControllerRedisTest.groovy | 1 + .../encoder/MoshiEncodingStrategyTest.groovy | 29 +- .../BuildServiceRateLimitTest.groovy | 24 +- .../builder/BuildCacheStoreRedisTest.groovy | 2 +- .../service/builder/BuildRequestTest.groovy | 30 +- .../service/builder/BuildStrategyTest.groovy | 10 +- .../builder/ContainerBuildServiceTest.groovy | 262 +++++++++++++----- .../builder/DockerBuildStrategyTest.groovy | 2 +- .../FutureContainerBuildServiceTest.groovy | 2 +- .../builder/KubeBuildStrategyTest.groovy | 17 +- .../service/k8s/K8sServiceImplTest.groovy | 20 +- .../persistence/WaveBuildRecordTest.groovy | 6 +- .../impl/SurrealPersistenceServiceTest.groovy | 9 +- .../wave/service/scan/ScanRequestTest.groovy | 33 +-- 25 files changed, 455 insertions(+), 204 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index acc21afb8..2ea625fde 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -22,8 +22,10 @@ import javax.annotation.Nullable import javax.annotation.PostConstruct import groovy.transform.CompileStatic +import groovy.transform.Memoized import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerPlatform import jakarta.inject.Singleton /** @@ -66,11 +68,28 @@ class BuildConfig { Duration statusDelay @Value('${wave.build.timeout:5m}') - Duration buildTimeout + Duration defaultTimeout + + @Value('${wave.build.trusted-timeout:10m}') + Duration trustedTimeout @Value('${wave.build.status.duration}') Duration statusDuration + @Memoized + Duration getStatusInitialDelay() { + final d1 = defaultTimeout.toMillis() * 2.5f + final d2 = trustedTimeout.toMillis() * 1.5f + return Duration.ofMillis(Math.round(Math.max(d1,d2))) + } + + @Memoized + Duration getStatusAwaitDuration() { + final d1 = defaultTimeout.toMillis() * 2.1f + final d2 = trustedTimeout.toMillis() * 1.1f + return Duration.ofMillis(Math.round(Math.max(d1,d2))) + } + @Value('${wave.build.cleanup}') @Nullable String cleanup @@ -93,7 +112,7 @@ class BuildConfig { @PostConstruct private void init() { - log.debug("Builder config: " + + log.info("Builder config: " + "buildkit-image=${buildkitImage}; " + "singularity-image=${singularityImage}; " + "singularity-image-amr64=${singularityImageArm64}; " + @@ -101,7 +120,8 @@ class BuildConfig { "default-cache-repository=${defaultCacheRepository}; " + "default-public-repository=${defaultPublicRepository}; " + "build-workspace=${buildWorkspace}; " + - "build-timeout=${buildTimeout}; " + + "build-timeout=${defaultTimeout}; " + + "build-trusted-timeout=${trustedTimeout}; " + "status-delay=${statusDelay}; " + "status-duration=${statusDuration}; " + "record-duration=${recordDuration}; " + @@ -109,6 +129,10 @@ class BuildConfig { "oci-mediatypes=${ociMediatypes}; " + "compression=${compression}; " + "force-compression=${forceCompression}; ") + // minimal validation + if( trustedTimeout < defaultTimeout ) { + log.warn "Trusted build timeout should be longer than default timeout - check configuration setting 'wave.build.trusted-timeout'" + } } String singularityImage(ContainerPlatform containerPlatform){ @@ -121,4 +145,11 @@ class BuildConfig { return singularityImageArm64 ?: singularityImage + "-arm64" } + Duration buildMaxDuration(SubmitContainerTokenRequest request) { + // build max duration - when the user identity is provided and freeze is enabled + // use `trustedTimeout` which is expected to be longer than `defaultTimeout` + return request.towerAccessToken && request.freeze && trustedTimeout>defaultTimeout + ? trustedTimeout + : defaultTimeout + } } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index fcfa3d7d7..be89c905a 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -329,7 +329,7 @@ class ContainerController { // create a unique digest to identify the build request final containerId = makeContainerId(containerFile, condaContent, spackContent, platform, buildRepository, req.buildContext) final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, spackContent, nameStrategy) - + final maxDuration = buildConfig.buildMaxDuration(req) return new BuildRequest( containerId, containerFile, @@ -346,7 +346,8 @@ class ContainerController { containerConfig, scanId, req.buildContext, - format + format, + maxDuration ) } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy index c6890036f..edd7c6583 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy @@ -61,17 +61,7 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt protected Duration getDuration() { return buildConfig.statusDuration } - - @Override - Duration getTimeout() { - return buildConfig.buildTimeout - } - - @Override - Duration getDelay() { - return buildConfig.statusDelay - } - + @Override BuildResult getBuild(String imageName) { return get(imageName) @@ -92,8 +82,7 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt // store up 2.5 time the build timeout to prevent a missed cache // update on job termination remains too long in the store // note: this should be longer than the max await time used in the Waiter#awaitCompletion method - final ttl = Duration.ofMillis(Math.round(getTimeout().toMillis() * 2.5f)) - return putIfAbsent(imageName, build, ttl) + return putIfAbsent(imageName, build, buildConfig.statusInitialDelay) } @Override @@ -114,12 +103,13 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt */ private static class Waiter { - static BuildResult awaitCompletion(BuildStore store, String imageName, BuildResult current) { + static BuildResult awaitCompletion(BuildCacheStore store, String imageName, BuildResult current) { + final await = store.buildConfig.statusDelay final beg = System.currentTimeMillis() // await nearly double of the build timeout time because the build job // can require additional time, other than the build time, to be scheduled // note: see also #storeIfAbsent method - final max = (store.timeout.toMillis() * 2.10) as long + final max = store.buildConfig.statusAwaitDuration.toMillis() while( true ) { if( current==null ) { return BuildResult.unknown() @@ -134,7 +124,7 @@ class BuildCacheStore extends AbstractCacheStore implements BuildSt if( delta > max ) throw new BuildTimeoutException("Build of container '$imageName' timed out") // sleep a bit - Thread.sleep(store.delay.toMillis()) + Thread.sleep(await.toMillis()) // fetch the build status again current = store.getBuild(imageName) } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 940c1dac6..545a52ca7 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.builder import java.nio.file.Path +import java.time.Duration import java.time.Instant import java.time.OffsetDateTime @@ -133,12 +134,35 @@ class BuildRequest { * The target build format, either Docker or Singularity */ final BuildFormat format + + /** + * Max allow time duration for this build + */ + final Duration maxDuration volatile String buildId volatile Path workDir - BuildRequest(String containerId, String containerFile, String condaFile, String spackFile, Path workspace, String targetImage, PlatformId identity, ContainerPlatform platform, String cacheRepository, String ip, String configJson, String offsetId, ContainerConfig containerConfig, String scanId, BuildContext buildContext, BuildFormat format) { + BuildRequest(String containerId, + String containerFile, + String condaFile, + String spackFile, + Path workspace, + String targetImage, + PlatformId identity, + ContainerPlatform platform, + String cacheRepository, + String ip, + String configJson, + String offsetId, + ContainerConfig containerConfig, + String scanId, + BuildContext buildContext, + BuildFormat format, + Duration maxDuration + ) + { this.containerId = containerId this.containerFile = containerFile this.condaFile = condaFile @@ -157,10 +181,11 @@ class BuildRequest { this.scanId = scanId this.buildContext = buildContext this.format = format + this.maxDuration = maxDuration } BuildRequest(Map opts) { - this.containerId = opts.id + this.containerId = opts.containerId this.containerFile = opts.containerFile this.condaFile = opts.condaFile this.spackFile = opts.spackFile @@ -180,11 +205,12 @@ class BuildRequest { this.format = opts.format as BuildFormat this.workDir = opts.workDir as Path this.buildId = opts.buildId + this.maxDuration = opts.maxDuration as Duration } @Override String toString() { - return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; spackFile=${trunc(spackFile)}; buildId=$buildId]" + return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; spackFile=${trunc(spackFile)}; buildId=$buildId, maxDuration=$maxDuration]" } String getContainerId() { @@ -244,6 +270,10 @@ class BuildRequest { return offsetId } + Duration getMaxDuration() { + return maxDuration + } + boolean formatDocker() { !format || format==DOCKER } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy index 1f1416443..f6e246a06 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStore.groovy @@ -30,10 +30,6 @@ import groovy.transform.CompileStatic @CompileStatic interface BuildStore { - Duration getTimeout() - - Duration getDelay() - /** * Retrieve a container image {@link BuildResult} * diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index f0460f95d..966f3e2aa 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -95,7 +95,8 @@ class DockerBuildStrategy extends BuildStrategy { .redirectErrorStream(true) .start() - final completed = proc.waitFor(buildConfig.buildTimeout.toSeconds(), TimeUnit.SECONDS) + final timeout = req.maxDuration ?: buildConfig.defaultTimeout + final completed = proc.waitFor(timeout.toSeconds(), TimeUnit.SECONDS) final stdout = proc.inputStream.text if( completed ) { final digest = proc.exitValue()==0 ? proxyService.getImageDigest(req, true) : null diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 6f79aa5aa..7ac6d20c5 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -97,10 +97,11 @@ class KubeBuildStrategy extends BuildStrategy { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) final name = podName(req) + final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) final spackCfg0 = req.isSpackBuild ? spackConfig : null - final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, spackCfg0, selector) - final exitCode = k8sService.waitPodCompletion(pod, buildConfig.buildTimeout.toMillis()) + final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) + final exitCode = k8sService.waitPodCompletion(pod, timeout.toMillis()) final stdout = k8sService.logsPod(pod) if( exitCode!=null ) { final digest = exitCode==0 ? proxyService.getImageDigest(req, true) : null diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 1ecb602d9..d7005ab7b 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.k8s import java.nio.file.Path +import java.time.Duration import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod @@ -41,9 +42,9 @@ interface K8sService { void deletePod(String name) - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, SpackConfig spackConfig, Map nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) Integer waitPodCompletion(V1Pod pod, long timeout) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 21a258222..fa886a81f 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.k8s import java.nio.file.Path +import java.time.Duration import javax.annotation.PostConstruct import groovy.transform.CompileDynamic @@ -120,7 +121,7 @@ class K8sServiceImpl implements K8sService { */ @PostConstruct private void init() { - log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; buildTimeout=$buildConfig.buildTimeout; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspace=$buildConfig.buildWorkspace; storageClaimName=$storageClaimName; storageMountPath=$storageMountPath; " + log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspace=$buildConfig.buildWorkspace; storageClaimName=$storageClaimName; storageMountPath=$storageMountPath; " if( storageClaimName && !storageMountPath ) throw new IllegalArgumentException("Missing 'wave.build.k8s.storage.mountPath' configuration attribute") if( storageMountPath ) { @@ -331,14 +332,14 @@ class K8sServiceImpl implements K8sService { * The {@link V1Pod} description the submitted pod */ @Override - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, SpackConfig spackConfig, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, spackConfig, nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, SpackConfig spackConfig, Map nodeSelector) { + V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { // dirty dependency to avoid introducing another parameter final singularity = containerImage.contains('singularity') @@ -380,7 +381,7 @@ class K8sServiceImpl implements K8sService { .withNewSpec() .withNodeSelector(nodeSelector) .withServiceAccount(serviceAccount) - .withActiveDeadlineSeconds( buildConfig.buildTimeout.toSeconds() ) + .withActiveDeadlineSeconds( timeout.toSeconds() ) .withRestartPolicy("Never") .addAllToVolumes(volumes) diff --git a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy index 6d3f633a4..f71941ea6 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildConfigTest.groovy @@ -19,7 +19,11 @@ package io.seqera.wave.controller import spock.lang.Specification +import spock.lang.Unroll +import java.time.Duration + +import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.core.ContainerPlatform @@ -52,4 +56,52 @@ class BuildConfigTest extends Specification { config.singularityImage( ContainerPlatform.of('arm64') ) == 'bar' } + @Unroll + def 'should validate initial duration' () { + given: + def config = new BuildConfig(defaultTimeout:DEFAULT, trustedTimeout:TRUSTED) + expect: + config.statusInitialDelay == EXPECTED + + where: + DEFAULT | TRUSTED | EXPECTED + Duration.ofMillis(1000) | Duration.ofMillis(1000) | Duration.ofMillis(Math.round(1000 * 2.5)) + Duration.ofMinutes(15) | Duration.ofMinutes(15) | Duration.ofMillis(Math.round( 37.5 * 60 * 1000)) + Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 2.5 * 15 * 60 * 1000)) + Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 1.5 * 25 * 60 * 1000)) + Duration.ofMinutes(15) | Duration.ofMinutes(30) | Duration.ofMillis(Math.round( 1.5 * 30 * 60 * 1000)) + } + + @Unroll + def 'should validate await duration' () { + given: + def config = new BuildConfig(defaultTimeout:DEFAULT, trustedTimeout:TRUSTED) + expect: + config.statusAwaitDuration == EXPECTED + + where: + DEFAULT | TRUSTED | EXPECTED + Duration.ofMillis(1000) | Duration.ofMillis(1000) | Duration.ofMillis(Math.round(1000 * 2.1)) + Duration.ofMinutes(15) | Duration.ofMinutes(15) | Duration.ofMillis(Math.round( 2.1 * 15 * 60 * 1000)) + Duration.ofMinutes(15) | Duration.ofMinutes(25) | Duration.ofMillis(Math.round( 2.1 * 15 * 60 * 1000)) + Duration.ofMinutes(15) | Duration.ofMinutes(30) | Duration.ofMillis(Math.round( 1.1 * 30 * 60 * 1000)) + } + + @Unroll + def 'should validate build max duration' () { + given: + def config = new BuildConfig(defaultTimeout: Duration.ofMinutes(DEFAULT), trustedTimeout: Duration.ofMinutes(TRUSTED)) + expect: + config.buildMaxDuration(new SubmitContainerTokenRequest(towerAccessToken: TOKEN, freeze: FREEZE)) == Duration.ofMinutes(EXPECTED) + + where: + TOKEN | FREEZE | DEFAULT | TRUSTED | EXPECTED + null | false | 5 | 10 | 5 + null | true | 5 | 10 | 5 + null | true | 5 | 1 | 5 + 'xyz' | false | 5 | 10 | 5 + 'xtz' | true | 5 | 10 | 10 // <-- pick "trusted" because both "freeze" and "token" are provided + 'xtz' | true | 20 | 10 | 20 // <-- pick "default" when it's greater than "trusted" + + } } diff --git a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy index c72b899f5..15344253e 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy @@ -34,6 +34,7 @@ import io.micronaut.http.client.exceptions.HttpClientResponseException import io.micronaut.http.server.types.files.StreamedFile import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.BuildStatusResponse import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -44,10 +45,8 @@ import io.seqera.wave.service.logs.BuildLogServiceImpl import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.tower.PlatformId -import jakarta.inject.Inject -import io.seqera.wave.api.BuildStatusResponse import io.seqera.wave.util.ContainerHelper - +import jakarta.inject.Inject /** * * @author Paolo Di Tommaso @@ -80,22 +79,17 @@ class BuildControllerTest extends Specification { final containerId = ContainerHelper.makeContainerId(containerFile, null, null, platform, 'buildrepo', null) final targetImage = ContainerHelper.makeTargetImage(format, repo, containerId, null, null, null) final build = new BuildRequest( - containerId, - containerFile, - null, - null, - Path.of("/some/path"), - targetImage, - PlatformId.NULL, - platform, - 'cacherepo', - "1.2.3.4", - '{"config":"json"}', - null, - null, - 'scan12345', - null, - format) + containerId: containerId, + containerFile: containerFile, + workspace: Path.of("/some/path"), + targetImage: targetImage, + identity: PlatformId.NULL, + platform: platform, + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: format ) .withBuildId('1') final result = new BuildResult(build.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(build, result) diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy index 089c494a0..56e1daf22 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy @@ -56,6 +56,7 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr REDIS_HOST : redisHostName, REDIS_PORT : redisPort, 'wave.build.timeout':'2s', + 'wave.build.trusted-timeout':'2s', 'micronaut.server.port': port, 'micronaut.http.services.default.url' : "http://localhost:$port".toString(), ], 'test', 'h2', 'redis') diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index cc608a0d6..c8950916a 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -324,22 +324,19 @@ class MoshiEncodingStrategyTest extends Specification { def context = new BuildContext('http://foo.com', '12345', 100, '67890') and: def build = new BuildRequest( - '12345', - 'from foo', - 'conda spec', - 'spack spec', - Path.of("/some/path"), - 'docker.io/some:image:12345', - PlatformId.NULL, - ContainerPlatform.of('linux/amd64'), - 'cacherepo', - "1.2.3.4", - '{"config":"json"}', - null, - null, - 'scan12345', - context, - null) + containerId: '12345', + containerFile: 'from foo', + condaFile: 'conda spec', + spackFile: 'spack spec', + workspace: Path.of("/some/path"), + targetImage: 'docker.io/some:image:12345', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345', + buildContext: context ) .withBuildId('1') def result = new BuildResult(build.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(build, result) diff --git a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy index 05d702d62..2a288c150 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/BuildServiceRateLimitTest.groovy @@ -71,7 +71,17 @@ class BuildServiceRateLimitTest extends Specification { and: def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null, null) - def REQ = new BuildRequest(CONTAINER_ID, dockerfile, null, null, folder, TARGET_IMAGE, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "127.0.0.1", '{"config":"json"}', null, null, null, null,BuildFormat.DOCKER) + def REQ = new BuildRequest( + containerId: CONTAINER_ID, + containerFile: dockerfile, + workspace: folder, + targetImage: TARGET_IMAGE, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + ip: "127.0.0.1", + configJson: '{"config":"json"}', + format: BuildFormat.DOCKER ) when: (0..configuration.build.authenticated.max).each { @@ -95,7 +105,17 @@ class BuildServiceRateLimitTest extends Specification { and: def CONTAINER_ID = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def TARGET_IMAGE = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, CONTAINER_ID, null, null, null) - def REQ = new BuildRequest(CONTAINER_ID, dockerfile, null, null, folder, TARGET_IMAGE, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "127.0.0.1", '{"config":"json"}', null, null, null, null,BuildFormat.DOCKER) + def REQ = new BuildRequest( + containerId: CONTAINER_ID, + containerFile: dockerfile, + workspace: folder, + targetImage: TARGET_IMAGE, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + ip: "127.0.0.1", + configJson: '{"config":"json"}', + format: BuildFormat.DOCKER) when: (0..configuration.build.anonymous.max).each { diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy index c0fabcacc..48432edb6 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildCacheStoreRedisTest.groovy @@ -41,7 +41,7 @@ class BuildCacheStoreRedisTest extends Specification implements RedisTestContain def setup() { applicationContext = ApplicationContext.run([ - wave:[ build:[ timeout: '5s' ]], + wave:[ build:[ timeout: '5s', 'trusted-timeout': '5s' ]], REDIS_HOST: redisHostName, REDIS_PORT: redisPort ], 'test', 'redis') diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index bee4d8468..e5eaa2172 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -22,6 +22,7 @@ package io.seqera.wave.service.builder import spock.lang.Specification import java.nio.file.Path +import java.time.Duration import java.time.OffsetDateTime import io.seqera.wave.api.BuildContext @@ -38,6 +39,7 @@ class BuildRequestTest extends Specification { def 'should create docker build request'() { given: + def TIMEOUT = Duration.ofMinutes(5) def USER = new PlatformId(new User(id:1, email: 'foo@user.com')) def CONTENT = 'FROM foo' def PATH = Path.of('somewhere') @@ -70,7 +72,8 @@ class BuildRequestTest extends Specification { CONFIG, SCAN_ID, CONTEXT, - FORMAT + FORMAT, + TIMEOUT ) then: @@ -118,7 +121,8 @@ class BuildRequestTest extends Specification { CONFIG, SCAN_ID, CONTEXT, - FORMAT + FORMAT, + TIMEOUT ) then: req.containerId == '8026e3a63b5c863f' @@ -153,7 +157,8 @@ class BuildRequestTest extends Specification { CONFIG, SCAN_ID, CONTEXT, - FORMAT + FORMAT, + TIMEOUT ) then: req.containerId == '8726782b1d9bb8fb' @@ -166,6 +171,7 @@ class BuildRequestTest extends Specification { def 'should create singularity build request'() { given: + def TIMEOUT = Duration.ofMinutes(5) def USER = new PlatformId(new User(id:1, email: 'foo@user.com')) def CONTENT = 'From: foo' def PATH = Path.of('somewhere') @@ -197,7 +203,8 @@ class BuildRequestTest extends Specification { CONFIG, null, CONTEXT, - FORMAT + FORMAT, + TIMEOUT ) then: req.containerId == 'd78ba9cb01188668' @@ -220,6 +227,7 @@ class BuildRequestTest extends Specification { def 'should check equals and hash code'() { given: + def TIMEOUT = Duration.ofMinutes(5) def USER = new PlatformId(new User(id:1, email: 'foo@user.com')) def PATH = Path.of('somewhere') def BUILD_REPO = 'docker.io/wave' @@ -232,26 +240,26 @@ class BuildRequestTest extends Specification { and: def CONTAINER_ID1 = ContainerHelper.makeContainerId(FOO_CONTENT, null, null, PLATFORM, BUILD_REPO, null) def TARGET_IMAGE1 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID1, null, null, null) - def req1 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req1 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req2 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req2 = new BuildRequest(CONTAINER_ID1, FOO_CONTENT, null, null, PATH, TARGET_IMAGE1, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: def CONTAINER_ID3 = ContainerHelper.makeContainerId(BAR_CONTENT, null, null, PLATFORM, BUILD_REPO, null) def TARGET_IMAGE3 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID3, null, null, null) - def req3 = new BuildRequest(CONTAINER_ID3, BAR_CONTENT, null, null, PATH, TARGET_IMAGE3, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req3 = new BuildRequest(CONTAINER_ID3, BAR_CONTENT, null, null, PATH, TARGET_IMAGE3, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: def CONTAINER_ID4 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, null, PLATFORM, BUILD_REPO, null) def TARGET_IMAGE4 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID4, CONDA_CONTENT, null, null) - def req4 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req4 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req5 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req5 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE4, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: CONDA_CONTENT = 'salmon=1.2.5' def CONTAINER_ID6 = ContainerHelper.makeContainerId(BAR_CONTENT, CONDA_CONTENT, null, PLATFORM, BUILD_REPO, null) def TARGET_IMAGE6 = ContainerHelper.makeTargetImage(FORMAT, BUILD_REPO, CONTAINER_ID6, CONDA_CONTENT, null, null) - def req6 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT) + def req6 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', null, null, null, null, FORMAT, TIMEOUT) and: - def req7 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', "UTC+2", null, null, null, FORMAT) + def req7 = new BuildRequest(CONTAINER_ID4, BAR_CONTENT, CONDA_CONTENT, null, PATH, TARGET_IMAGE6, USER, PLATFORM, CACHE_REPO, "10.20.30.40", '{"config":"json"}', "UTC+2", null, null, null, FORMAT, TIMEOUT) expect: req1 == req2 diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy index db621a613..8151c836f 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.service.builder import spock.lang.Specification import java.nio.file.Path +import java.time.Duration import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.core.ContainerPlatform @@ -40,7 +41,7 @@ class BuildStrategyTest extends Specification { def 'should get buildkit command' () { given: def req = new BuildRequest( - id: 'c168dba125e28777', + containerId: 'c168dba125e28777', workDir: Path.of('/work/foo/c168dba125e28777'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'quay.io/wave:c168dba125e28777', @@ -73,7 +74,7 @@ class BuildStrategyTest extends Specification { def 'should get buildkit command with build context' () { given: def req = new BuildRequest( - id: 'c168dba125e28777', + containerId: 'c168dba125e28777', workDir: Path.of('/work/foo/3980470531b4a52a'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'quay.io/wave:3980470531b4a52a', @@ -123,6 +124,7 @@ class BuildStrategyTest extends Specification { def 'should create request' () { when: + def timeout = Duration.ofMinutes(5) def content = 'FROM foo:latest' def workspace = Path.of("some/path") def buildrepo = 'foo.com/repo' @@ -144,7 +146,8 @@ class BuildStrategyTest extends Specification { null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + timeout ) then: @@ -162,5 +165,6 @@ class BuildStrategyTest extends Specification { and: build.buildId == 'af15cb0a413a2d48_100' build.workDir == Path.of('.').toRealPath().resolve('some/path/af15cb0a413a2d48_100') + build.maxDuration == timeout } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 0f846366a..cf436f481 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -87,8 +87,20 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, cacheRepo, Mock(PlatformId)) def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", cfg, null,null , null, null, BuildFormat.DOCKER) - .withBuildId('1') + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('1') when: def result = service.launch(req) @@ -117,7 +129,19 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", cfg, null,null , null, null, BuildFormat.DOCKER) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now(), + ) .withBuildId('1') when: @@ -150,7 +174,19 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", cfg, null,null , null, null, BuildFormat.DOCKER) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) .withBuildId('1') when: @@ -182,7 +218,19 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", cfg, null,null , null, null, BuildFormat.DOCKER) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) .withBuildId('1') when: @@ -227,7 +275,21 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def spackConfig = new SpackConfig(cacheBucket: 's3://bucket/cache', secretMountPath: '/mnt/secret') def containerId = ContainerHelper.makeContainerId(dockerFile, condaFile, spackFile, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, condaFile, spackFile, null) - def req = new BuildRequest(containerId, dockerFile, condaFile, spackFile, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", null, null,null , null, null, BuildFormat.DOCKER) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + condaFile: condaFile, + spackFile: spackFile, + isSpackBuild: true, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) .withBuildId('1') and: def store = Mock(BuildStore) @@ -261,8 +323,18 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def dockerFile = 'FROM something; {{foo}}' def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", null, null,null , null, null, BuildFormat.DOCKER) - .withBuildId('1') + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('1') and: def spack = Mock(SpackConfig) @@ -288,7 +360,18 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def spackFile = 'some spack packages' def containerId = ContainerHelper.makeContainerId(dockerFile, null, spackFile, ContainerPlatform.of('amd64'), 'buildRepo', null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, 'foo.com/repo', containerId, null, spackFile, null) - def req = new BuildRequest(containerId, dockerFile, null, spackFile, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), 'cacheRepo', "10.20.30.40", null, null,null , null, null, BuildFormat.DOCKER) + def req = new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + spackFile: spackFile, + isSpackBuild: true, + workspace: folder, + targetImage: targetImage, + identity:Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) .withBuildId('1') and: def spack = Mock(SpackConfig) @@ -320,7 +403,20 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def spackFile = 'some spack packages' def containerId = ContainerHelper.makeContainerId(dockerFile, null, spackFile, ContainerPlatform.of('amd64'), 'buildRepo', null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'foo.com/repo', containerId, null, spackFile, null) - def req = new BuildRequest(containerId, dockerFile, null, spackFile, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), 'cacheRepo', "10.20.30.40", null, null,null , null, null, BuildFormat.SINGULARITY) + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + spackFile: spackFile, + isSpackBuild: true, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + format: BuildFormat.SINGULARITY, + startTime: Instant.now() + ) + .withBuildId('1') and: def spack = Mock(SpackConfig) @@ -358,7 +454,18 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def builder = new ContainerBuildServiceImpl() def containerId = ContainerHelper.makeContainerId(containerFile, null, null, ContainerPlatform.of('amd64'), 'buildRepo', null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.SINGULARITY, 'foo.com/repo', containerId, null, null, null) - def req = new BuildRequest(containerId, containerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), 'cacheRepo', "10.20.30.40", null, null,null , null, null, BuildFormat.SINGULARITY).withBuildId('1') + def req = + new BuildRequest( + containerId: containerId, + containerFile: containerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + format: BuildFormat.SINGULARITY, + startTime: Instant.now() + ) + .withBuildId('1') when: def result = builder.containerFile0(req, Path.of('/some/context/'), null) @@ -393,7 +500,21 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def cfg = dockerAuthService.credentialsConfigJson(dockerFile, buildRepo, null, Mock(PlatformId)) def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", cfg, null,containerConfig , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: folder, + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + cacheRepository: cacheRepo, + configJson: cfg, + containerConfig: containerConfig , + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('1') when: def result = service.launch(req) @@ -465,7 +586,20 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai def buildRepo = 'quay.io/org/name' def containerId = ContainerHelper.makeContainerId(dockerFile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerFile, null, null, Path.of('/wsp'), targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), 'cacheRepo', "10.20.30.40", '{"config":"json"}', null,config , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = + new BuildRequest( + containerId: containerId, + containerFile: dockerFile, + workspace: Path.of('/wsp'), + targetImage: targetImage, + identity: Mock(PlatformId), + platform: ContainerPlatform.of('amd64'), + configJson: '{"config":"json"}', + containerConfig: config , + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('1') when: service.saveLayersToContext(req, folder) @@ -481,23 +615,20 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai void "an event insert a build"() { given: final request = new BuildRequest( - 'container1234', - 'test', - 'test', - 'test', - Path.of("."), - 'docker.io/my/repo:container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER - ).withBuildId('123') + containerId: 'container1234', + containerFile: 'test', + condaFile: 'test', + spackFile: 'test', + workspace: Path.of("."), + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('123') and: def result = new BuildResult(request.buildId, 0, "content", Instant.now(), Duration.ofSeconds(1), 'abc123') @@ -514,24 +645,24 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai void "should create build record in redis"() { given: - final request = new BuildRequest( - 'container1234', - 'test', - 'test', - 'test', - Path.of("."), - 'docker.io/my/repo:container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER - ).withBuildId('123') + final request = + new BuildRequest( + containerId: 'container1234', + containerFile:'test', + condaFile: 'test', + spackFile: 'test', + workspace: Path.of("."), + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + cacheRepository: 'docker.io/my/cache', + ip: '127.0.0.1', + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + startTime: Instant.now() + ) + .withBuildId('123') and: def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') @@ -550,24 +681,23 @@ class ContainerBuildServiceTest extends Specification implements RedisTestContai void "should save build record in redis and surrealdb"() { given: - final request = new BuildRequest( - 'container1234', - 'test', - 'test', - 'test', - Path.of("."), - 'docker.io/my/repo:container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER - ).withBuildId('123') + final request = + new BuildRequest( + containerId: 'container1234', + containerFile: 'test', + condaFile: 'test', + spackFile: 'test', + workspace: Path.of("."), + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + cacheRepository: 'docker.io/my/cache', + ip: '127.0.0.1', + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER + ) + .withBuildId('123') and: def result = new BuildResult(request.buildId, 0, "content", Instant.now(), Duration.ofSeconds(1), 'abc123') diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 31cbc7aef..7d83bf257 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -96,7 +96,7 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( - id: '89fb83ce6ec8627b', + containerId: '89fb83ce6ec8627b', workDir: Path.of('/work/foo/89fb83ce6ec8627b'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'repo:89fb83ce6ec8627b', diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy index 68fc80add..5c96184d0 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy @@ -71,7 +71,7 @@ class FutureContainerBuildServiceTest extends Specification { and: def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), buildRepo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') when: exitCode = EXIT_CODE diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index d50f44ca3..628ef2fe4 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.service.builder import spock.lang.Specification import java.nio.file.Files +import java.time.Duration import io.micronaut.context.annotation.Property import io.micronaut.test.annotation.MockBean @@ -69,24 +70,24 @@ class KubeBuildStrategyTest extends Specification { when: def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req.workDir) def resp = strategy.build(req) then: resp and: - 1 * k8sService.buildContainer(_, _, _, _, _, _, [service:'wave-build']) >> null + 1 * k8sService.buildContainer(_, _, _, _, _, _, _, [service:'wave-build']) >> null when: - def req2 = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') + def req2 = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') Files.createDirectories(req2.workDir) def resp2 = strategy.build(req2) then: resp2 and: - 1 * k8sService.buildContainer(_, _, _, _, _, _, [service:'wave-build-arm64']) >> null + 1 * k8sService.buildContainer(_, _, _, _, _, _, _, [service:'wave-build-arm64']) >> null } @@ -101,19 +102,19 @@ class KubeBuildStrategyTest extends Specification { when:'getting docker with amd64 arch in build request' def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') then: 'should return buildkit image' strategy.getBuildImage(req) == 'moby/buildkit:v0.14.1-rootless' when:'getting singularity with amd64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY,Duration.ofMinutes(1)).withBuildId('1') then:'should return singularity amd64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim' when:'getting singularity with arm64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY, Duration.ofMinutes(1)).withBuildId('1') then:'should return singularity arm64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim-arm64' @@ -128,7 +129,7 @@ class KubeBuildStrategyTest extends Specification { def dockerfile = 'from foo' def containerId = ContainerHelper.makeContainerId(dockerfile, null, null, ContainerPlatform.of('amd64'), repo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) - def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') when: def podName = strategy.podName(req) diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 6b45d4bd4..2d7f89dd3 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -37,8 +37,6 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.SpackConfig -import io.seqera.wave.util.K8sHelper - /** * * @author Paolo Di Tommaso @@ -203,7 +201,6 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace' : '/build/work', - 'wave.build.timeout' : '10s', 'wave.build.k8s.namespace' : 'my-ns', 'wave.build.k8s.configPath' : '/home/kube.config', 'wave.build.k8s.storage.claimName': 'build-claim', @@ -213,7 +210,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this', 'that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), null, [:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this', 'that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), Duration.ofSeconds(10), null, [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -247,7 +244,6 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace': '/build/work', - 'wave.build.timeout': '10s', 'wave.build.k8s.namespace': 'my-ns', 'wave.build.k8s.configPath': '/home/kube.config', 'wave.build.k8s.storage.claimName': 'build-claim', @@ -257,7 +253,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) def workDir = Path.of('/build/work/xyz') when: - def result = k8sService.buildSpec('foo', 'singularity:latest', ['this','that'], workDir, workDir.resolve('config.json'), null, [:]) + def result = k8sService.buildSpec('foo', 'singularity:latest', ['this','that'], workDir, workDir.resolve('config.json'), Duration.ofSeconds(10), null, [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -295,7 +291,6 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace': '/build/work', - 'wave.build.timeout': '10s', 'wave.build.k8s.namespace': 'my-ns', 'wave.build.k8s.configPath': '/home/kube.config', 'wave.build.k8s.storage.claimName': 'build-claim', @@ -308,7 +303,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) def spackConfig = ctx.getBean(SpackConfig) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, spackConfig, [:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null,Duration.ofSeconds(10), spackConfig, [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -342,7 +337,6 @@ class K8sServiceImplTest extends Specification { given: def PROPS = [ 'wave.build.workspace': '/build/work', - 'wave.build.timeout': '10s', 'wave.build.k8s.namespace': 'my-ns', 'wave.build.k8s.configPath': '/home/kube.config', 'wave.build.k8s.storage.claimName': 'build-claim', @@ -352,7 +346,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -396,7 +390,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) then: result.metadata.name == 'foo' result.metadata.labels.toString() == PROPS['wave.build.k8s.labels'].toString() @@ -424,7 +418,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null, PROPS['wave.build.k8s.node-selector'] as Map) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null, PROPS['wave.build.k8s.node-selector'] as Map) then: result.spec.nodeSelector.toString() == PROPS['wave.build.k8s.node-selector'].toString() and: @@ -449,7 +443,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, Duration.ofSeconds(10), null,[:]) then: result.spec.serviceAccount == PROPS['wave.build.k8s.service-account'] and: diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index b9270130c..fa9d5cf36 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -57,7 +57,8 @@ class WaveBuildRecordTest extends Specification { null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + Duration.ofMinutes(1) ) final result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(request, result) @@ -88,7 +89,8 @@ class WaveBuildRecordTest extends Specification { null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + Duration.ofMinutes(1) ).withBuildId('123') and: diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index 135ef8de0..fe8050177 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -116,7 +116,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + Duration.ofMinutes(1) ).withBuildId('1') def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(request, result) @@ -152,7 +153,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + Duration.ofMinutes(1) ).withBuildId('123') def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(request, result) @@ -188,7 +190,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe null, 'scan12345', null, - BuildFormat.DOCKER + BuildFormat.DOCKER, + Duration.ofMinutes(1) ).withBuildId('123') and: def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy index ce55fc628..a122f30af 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy @@ -22,8 +22,6 @@ import spock.lang.Specification import java.nio.file.Path -import io.seqera.wave.api.BuildContext -import io.seqera.wave.api.ContainerConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest @@ -38,24 +36,19 @@ class ScanRequestTest extends Specification { given: def workspace = Path.of('/some/workspace') def platform = ContainerPlatform.of('amd64') - final build = new BuildRequest( - 'container1234', - 'FROM ubuntu', - null, - null, - workspace, - 'docker.io/my/repo:container1234', - PlatformId.NULL, - platform, - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER - ).withBuildId('123') + final build = + new BuildRequest( + containerId: 'container1234', + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER + ) + .withBuildId('123') when: def scan = ScanRequest.fromBuild(build) From f844bec748e1d47ee4de4fe39055a48309800666 Mon Sep 17 00:00:00 2001 From: Gavin Date: Tue, 20 Aug 2024 07:22:33 +0100 Subject: [PATCH 37/50] docs: Update wave rate limits (#606) --- docs/cli/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index 7d625afe1..562d3846f 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -39,8 +39,8 @@ The following usage limits apply: - 25 container builds per day - 250 container pulls per hour - Seqera Platform authenticated users - - 100 container images per hour - - 1,000 container images per minute + - 100 container builds per hour + - 1,000 container pulls per minute To authenticate with Seqera, define an access token in the `TOWER_ACCESS_TOKEN` environment variable or specify the token with the `--tower-token` CLI argument. From b31909f8382d4ceda3de892bf2b76f4968598a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20=CE=92oxman?= <141646877+jason-seqera@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:11:50 -0400 Subject: [PATCH 38/50] Refresh Wave documentation (#516) --- docs/api.mdx | 14 ++ docs/cli/build-conda.mdx | 28 --- docs/cli/build-directory.mdx | 34 --- docs/cli/build-docker.mdx | 60 ------ docs/cli/build-freeze.mdx | 31 --- docs/cli/build-singularity.mdx | 47 ---- docs/cli/build-spack.mdx | 29 --- docs/cli/index.mdx | 2 +- docs/cli/install.mdx | 41 ---- docs/cli/reference.mdx | 271 ++++++++++++++++++++++++ docs/get-started.mdx | 377 +++++++++++++++++++++++++++++++++ docs/guide.mdx | 161 -------------- docs/index.mdx | 75 ++++--- docs/metrics.mdx | 8 +- docs/nextflow.mdx | 215 +++++++++++++++++++ docs/provisioning.mdx | 88 ++++++++ docs/sidebar.json | 34 ++- 17 files changed, 1036 insertions(+), 479 deletions(-) delete mode 100644 docs/cli/build-conda.mdx delete mode 100644 docs/cli/build-directory.mdx delete mode 100644 docs/cli/build-docker.mdx delete mode 100644 docs/cli/build-freeze.mdx delete mode 100644 docs/cli/build-singularity.mdx delete mode 100644 docs/cli/build-spack.mdx delete mode 100644 docs/cli/install.mdx create mode 100644 docs/cli/reference.mdx create mode 100644 docs/get-started.mdx delete mode 100644 docs/guide.mdx create mode 100644 docs/nextflow.mdx create mode 100644 docs/provisioning.mdx diff --git a/docs/api.mdx b/docs/api.mdx index 019a95b39..0d646c0b3 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -4,6 +4,20 @@ title: API reference This page summarizes the API provided by the Wave container service. +**API limits** + +The Wave service implements API rate limits for API calls. Authenticated users have higher rate limits than anonymous users. + +If an access token is provided, the following rate limits apply: + +- 100 container builds per hour +- 1,000 container pulls per minute + +If an access token isn't provided, the following rate limits apply: + +- 25 container builds per day +- 250 container pulls per hour + ## POST `/container-token` Deprecated endpoint allows you to submit a request to access a private container registry via Wave, or build a container image on-the-fly with a Dockerfile or Conda recipe file. diff --git a/docs/cli/build-conda.mdx b/docs/cli/build-conda.mdx deleted file mode 100644 index be13cb2dd..000000000 --- a/docs/cli/build-conda.mdx +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Build a container from Conda packages ---- - -The Wave CLI supports building a container from a list of [Conda] packages. - -## Related CLI arguments - -Conda builds support the following arguments: - -- `--conda-base-image`: A base image for installing Conda packages. The default value is `mambaorg/micromamba:1.5.1`. -- `--conda-channels`: One or more comma separated channels. The default value is ` seqera,bioconda,conda-forge,defaults`. -- `--conda-file`: A [Conda lock file][conda-lock]. Can be a local file or a URL to a remote file. -- `--conda-package`: A Conda package to install. Can be specified multiple times. Expression are supported, such as `bioconda::samtools=1.17` or `samtools>=1.0,<1.17`. -- ` --conda-run-command`: A Docker `RUN` command used when the container is built. Can be specified multiple times. - -## Example usage - -In the following example, a container with the `samtools` and `bamtools` packages is built: - -``` -wave \ - --conda-package bamtools=2.5.2 \ - --conda-package samtools=1.17 -``` - -[Conda]: https://anaconda.org/anaconda/repo -[conda-lock]: https://github.com/conda/conda-lock diff --git a/docs/cli/build-directory.mdx b/docs/cli/build-directory.mdx deleted file mode 100644 index c627d05ab..000000000 --- a/docs/cli/build-directory.mdx +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Augment a container with a directory ---- - -The Wave CLI supports container augmentation with a specified directory. You can use container augmentation to dynamically add a layer to your container, so you can inject scripts or configuration files as a new layer. - -The following limitations apply: - -- A file must be no larger than 1 MB each. -- A directory must be no larger than 10 MB, inclusive of all files. -- A base image must be specified. - -## Related CLI arguments - -The following arguments are used for a directory build: - -- `--layer`: A directory that contains layer content. -- `--image` or `-i`: An existing container image. The default image registry is `docker.io`. Specify an image name such as `alpine:latest` or an image URL such as `public.ecr.aws/docker/library/busybox`. - -## Example usage - -Create a new context directory: - -``` -mkdir -p new-layer/usr/local/bin -printf 'echo Hello world!' > new-layer/usr/local/bin/hello.sh -chmod +x new-layer/usr/local/bin/hello.sh -``` - -Use the CLI to build the image and run the result with Docker: - -``` -docker run $(wave -i alpine --layer new-layer) sh -c hello.sh -``` diff --git a/docs/cli/build-docker.mdx b/docs/cli/build-docker.mdx deleted file mode 100644 index 6399abcf5..000000000 --- a/docs/cli/build-docker.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: Build a container from a Dockerfile ---- - -The Wave CLI supports building a container from a `Dockerfile`. Specifying an optional build context allows the use of `ADD` and `COPY` commands in a Dockerfile. - -:::note -Building a Dockerfile that requires `--build-arg` for build time variables isn't currently supported. -::: - -## Related CLI arguments - -- `--containerfile` or `-f`: A Dockerfile to build. Build args aren't currently supported. -- `--context`: A directory that contains the context for the build. - -## Example usage - -In the following example `Dockerfile`, several packages are installed: - -``` -cat << EOF > ./Dockerfile -FROM alpine - -RUN apk update && apk add bash cowsay \ - --update-cache \ - --repository https://alpine.global.ssl.fastly.net/alpine/edge/community \ - --repository https://alpine.global.ssl.fastly.net/alpine/edge/main \ - --repository https://dl-3.alpinelinux.org/alpine/edge/testing -EOF -``` - -Build and run the container based on the Dockerfile in the previous example by running the following command: - -``` -container=$(wave --containerfile ./Dockerfile) -docker run --rm $container cowsay "Hello world" -``` - -In the following example `Dockerfile`, a local context is used: - -``` -cat << EOF > ./Dockerfile -FROM alpine -ADD hello.sh /usr/local/bin/ -EOF -``` - -Create the shell script referenced in the previous example by running the following commands in your terminal: - -``` -mkdir -p build-context/ -printf 'echo Hello world!' > build-context/hello.sh -chmod +x build-context/hello.sh -``` - -Build and run the container based on the Dockerfile in the previous example by running the following command: - -``` -docker run $(wave -f Dockerfile --context build-context) sh -c hello.sh -``` diff --git a/docs/cli/build-freeze.mdx b/docs/cli/build-freeze.mdx deleted file mode 100644 index 72f83482e..000000000 --- a/docs/cli/build-freeze.mdx +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Build a container and freeze to a container registry ---- - -The Wave CLI supports building a container and persisting the container to a container registry, such as DockerHub. You can refer to this frozen container image in a Dockerfile or [Nextflow] pipeline in the same way as any other container. - -To freeze a container, you must ensure the following conditions are met: - -- You created a Seqera Platform access token. -- You specified the destination container registry credentials in Seqera Platform. -- You specify the Seqera Platform access token via either the `TOWER_ACCESS_TOKEN` environment variable or the `--tower-token` Wave command line option. - -## Related CLI arguments - -The following arguments are used to freeze a container build: - -- `--build-repo`: A target repository to save the built container to. -- `--freeze`: Enable a container freeze. -- `--tower-token`: A Seqera Platform auth token so that Wave can access your private registry credentials. Not required if the `TOWER_ACCESS_TOKEN` environment variable is set. -- `--tower-workspace-id`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. - -## Example usage - -In the following example, the `alpine` container image is frozen to a private DockerHub image registry. The `--tower-token` argument is not required if the `TOWER_ACCESS_TOKEN` environment variable is defined. - -``` -wave -i alpine --freeze \ - --build-repo docker.io/user/repo --tower-token -``` - -[Nextflow]: https://www.nextflow.io/ diff --git a/docs/cli/build-singularity.mdx b/docs/cli/build-singularity.mdx deleted file mode 100644 index 54a361ecb..000000000 --- a/docs/cli/build-singularity.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Build a Singularity container ---- - -The Wave CLI supports building a [Singularity]. A target build repository, specified with the `--build-repo` argument, is required to build a Singularity container. You can build a Singularity container from several sources: - -- A [SingularityCE] def file -- A Docker container image with an optional local context directory -- Conda packages -- Spack packages - -The following limitations apply: - -- The `linux/arm64` platform is not currently supported - -## Related CLI arguments - -The following arguments are used to build a Singularity container: - -- `--build-repo`: A target repository to save the built container to. -- `--freeze`: Enable a container freeze. -- `--singularity` and `-s`: Build a Singularity container. -- `--tower-token`: A Seqera Platform auth token so that Wave can access your private registry credentials. Not required if the `TOWER_ACCESS_TOKEN` environment variable is set. -- `--tower-workspace-id`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. - -## Example usage - -In the following example, a Docker base imagine is augmented: - -``` -wave -i alpine --layer context-dir/ --build-repo docker.io/user/repo -``` - -In the following example, a SingularityCE def file is specified: - -``` -wave -f hello-world.def --singularity --freeze --build-repo docker.io/user/repo -``` - -In the following example, two Conda packages are specified: - -``` -wave --conda-package bamtools=2.5.2 --conda-package samtools=1.17 --freeze --singularity --build-repo docker.io/user/repo -``` - -[Singularity]: https://docs.sylabs.io/guides/latest/user-guide/introduction.html -[SingularityCE]: https://docs.sylabs.io/guides/latest/user-guide/definition_files.html diff --git a/docs/cli/build-spack.mdx b/docs/cli/build-spack.mdx deleted file mode 100644 index a11acbd9a..000000000 --- a/docs/cli/build-spack.mdx +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Build a container from Spack packages ---- - -**Note**: Spack support will be removed in future releases. - -The Wave CLI supports building a container from a list of [Spack] packages. - -:::caution -Support for Spack packages is currently experimental. -::: - -## Related CLI arguments - -Spack builds support following arguments: - -- `--spack-file`: A Spack YAML file that specifies packages and their dependencies. -- `--spack-package`: A Spack package to install. Can be specified multiple times. Version expression are supported, such as `curl@7.42.1`. -- `--spack-run-command`: A Docker `RUN` command used when the container is built. Can be specified multiple times. - -## Example usage - -In the following example, a container with the `curl` package is built: - -``` -wave --spack-package curl@7.42.1 -``` - -[Spack]: https://spack.io/ diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index 562d3846f..6581f628d 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -1,5 +1,5 @@ --- -title: Wave CLI +title: CLI overview --- The Wave CLI is a convenient wrapper around the Wave API. diff --git a/docs/cli/install.mdx b/docs/cli/install.mdx deleted file mode 100644 index ddf9574f1..000000000 --- a/docs/cli/install.mdx +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: Install the Wave CLI ---- - -To install the `wave` CLI for your platform, complete the following steps: - -1. Download the [latest version of the Wave CLI][download] for your platform. - -2. In a new terminal, complete the following steps: - - 1. Move the executable from your downloads folder to a location in your `PATH`, such as `~/bin`. For example: `mv wave-cli-0.8.0-macos-x86_64 ~/bin/wave` - 2. Ensure that the executable bit is set. For example: `chmod u+x ~/bin/wave` - -3. You can also use [Homebrew](https://brew.sh/) in macos and linux, you can install like this: - ```bash - brew install seqeralabs/tap/wave-cli - ``` - -4. Verify that you can build containers with Wave: - - 1. Create a basic `Dockerfile`: - - ``` - cat << EOF > ./Dockerfile - FROM busybox:latest - EOF - ``` - - 2. Use the CLI to build the container: - - ``` - wave -f Dockerfile - ``` - - Example output: - - ``` - wave.seqera.io/wt/xxxxxxxxxxxx/wave/build:xxxxxxxxxxxxxxxx - ``` - -[download]: https://github.com/seqeralabs/wave-cli/releases diff --git a/docs/cli/reference.mdx b/docs/cli/reference.mdx new file mode 100644 index 000000000..fae9c3cb1 --- /dev/null +++ b/docs/cli/reference.mdx @@ -0,0 +1,271 @@ +--- +title: CLI reference +--- + +## Install the Wave CLI + +To install the `wave` CLI for your platform, complete the following steps: + +1. Download latest version of the Wave CLI for your platform: + + - To install the latest release from GitHub: + + 1. Download the [latest version of the Wave CLI][download] for your platform. + + 1. In a new terminal, complete the following steps: + + 1. Move the executable from your downloads folder to a location in your `PATH`, such as `~/bin`. For example: `mv wave-cli-0.8.0-macos-x86_64 ~/bin/wave` + 1. Ensure that the executable bit is set. For example: `chmod u+x ~/bin/wave` + + - To install the latest version with [Homebrew]: + + ```bash + brew install seqeralabs/tap/wave-cli + ``` + +1. Verify that you can build containers with Wave: + + 1. Create a basic `Dockerfile`: + + ``` + cat << EOF > ./Dockerfile + FROM busybox:latest + EOF + ``` + + 1. Use the CLI to build the container: + + ``` + wave -f Dockerfile + ``` + + Example output: + + ``` + wave.seqera.io/wt/xxxxxxxxxxxx/wave/build:xxxxxxxxxxxxxxxx + ``` + +[download]: https://github.com/seqeralabs/wave-cli/releases +[Homebrew]: https://brew.sh/ + +## Build a container + +With the Wave CLI you can build Docker and Singularity containers from a variety of sources, including a Dockerfile, Singularity def file, file system directory, and Conda packages. + +The following sections describe several common usage cases. To get started by creating an example Nextflow pipeline that uses Wave CLI, see [Get started][start]. + +[start]: ../get-started.mdx#wave-cli + +### Augment a container with a directory + +The Wave CLI supports container augmentation with a specified directory. You can use container augmentation to dynamically add a layer to your container, so you can inject scripts or configuration files as a new layer. + +
+**Augment a container with a directory** + +The following limitations apply: + +- A file must be no larger than 1 MB each. +- A directory must be no larger than 10 MB, inclusive of all files. +- A base image must be specified. + +**Related CLI arguments** + +The following arguments are used for a directory build: + +- `--layer`: A directory that contains layer content. +- `--image` or `-i`: An existing container image. The default image registry is `docker.io`. Specify an image name such as `alpine:latest` or an image URL such as `public.ecr.aws/docker/library/busybox`. + +**Example usage** + +Create a new context directory: + +``` +mkdir -p new-layer/usr/local/bin +printf 'echo Hello world!' > new-layer/usr/local/bin/hello.sh +chmod +x new-layer/usr/local/bin/hello.sh +``` + +Use the CLI to build the image and run the result with Docker: + +``` +docker run $(wave -i alpine --layer new-layer) sh -c hello.sh +``` +
+ +### Build a container from Conda packages + +The Wave CLI supports building a container from a list of [Conda] packages. + +
+**Build a container from Conda packages** + +**Related CLI arguments** + +Conda builds support the following arguments: + +- `--conda-base-image`: A base image for installing Conda packages. The default value is `mambaorg/micromamba:1.5.1`. +- `--conda-channels`: One or more comma-separated channels. The default value is ` seqera,bioconda,conda-forge,defaults`. +- `--conda-file`: A [Conda lock file][conda-lock]. Can be a local file or a URL to a remote file. +- `--conda-package`: A Conda package to install. Can be specified multiple times. Expressions are supported, such as `bioconda::samtools=1.17` or `samtools>=1.0,<1.17`. +- ` --conda-run-command`: A Docker `RUN` command used when the container is built. Can be specified multiple times. + +**Example usage** + +In the following example, a container with the `samtools` and `bamtools` packages is built: + +``` +wave \ + --conda-package bamtools=2.5.2 \ + --conda-package samtools=1.17 +``` + +[Conda]: https://anaconda.org/anaconda/repo +[conda-lock]: https://github.com/conda/conda-lock +
+ +### Build a container from a Dockerfile + +The Wave CLI supports building a container from a `Dockerfile`. Specifying an optional build context allows the use of `ADD` and `COPY` commands in a Dockerfile. + +:::note +Building a Dockerfile that requires `--build-arg` for build time variables isn't currently supported. +::: + +
+**Build a container from a Dockerfile** + +**Related CLI arguments** + +- `--containerfile` or `-f`: A Dockerfile to build. Build args aren't currently supported. +- `--context`: A directory that contains the context for the build. + +**Example usage** + +In the following example `Dockerfile`, several packages are installed: + +``` +cat << EOF > ./Dockerfile +FROM alpine + +RUN apk update && apk add bash cowsay \ + --update-cache \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/community \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/main \ + --repository https://dl-3.alpinelinux.org/alpine/edge/testing +EOF +``` + +Build and run the container based on the Dockerfile in the previous example by running the following command: + +``` +container=$(wave --containerfile ./Dockerfile) +docker run --rm $container cowsay "Hello world" +``` + +In the following example `Dockerfile`, a local context is used: + +``` +cat << EOF > ./Dockerfile +FROM alpine +ADD hello.sh /usr/local/bin/ +EOF +``` + +Create the shell script referenced in the previous example by running the following commands in your terminal: + +``` +mkdir -p build-context/ +printf 'echo Hello world!' > build-context/hello.sh +chmod +x build-context/hello.sh +``` + +Build and run the container based on the Dockerfile in the previous example by running the following command: + +``` +docker run $(wave -f Dockerfile --context build-context) sh -c hello.sh +``` +
+ +### Build a Singularity container + +The Wave CLI supports building a [Singularity]. A target build repository, specified with the `--build-repo` argument, is required to build a Singularity container. You can build a Singularity container from several sources. + +
+**Build a Singularity container** + +- A [SingularityCE] def file +- A Docker container image with an optional local context directory +- Conda packages + +The following limitations apply: + +- The `linux/arm64` platform is not currently supported + +**Related CLI arguments** + +The following arguments are used to build a Singularity container: + +- `--build-repo`: A target repository to save the built container to. +- `--freeze`: Enable a container freeze. +- `--singularity` and `-s`: Build a Singularity container. +- `--tower-token`: A Seqera Platform auth token so that Wave can access your private registry credentials. Not required if the `TOWER_ACCESS_TOKEN` environment variable is set. +- `--tower-workspace-id`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. + +**Example usage** + +In the following example, a Docker base image is augmented: + +``` +wave -i alpine --layer context-dir/ --build-repo docker.io/user/repo +``` + +In the following example, a SingularityCE def file is specified: + +``` +wave -f hello-world.def --singularity --freeze --build-repo docker.io/user/repo +``` + +In the following example, two Conda packages are specified: + +``` +wave --conda-package bamtools=2.5.2 --conda-package samtools=1.17 --freeze --singularity --build-repo docker.io/user/repo +``` + +[Singularity]: https://docs.sylabs.io/guides/latest/user-guide/introduction.html +[SingularityCE]: https://docs.sylabs.io/guides/latest/user-guide/definition_files.html +
+ +### Build a container and freeze to a container registry + +The Wave CLI supports building a container and persisting the container to a container registry, such as DockerHub. You can refer to this frozen container image in a Dockerfile or [Nextflow] pipeline in the same way as any other container. + +
+**Build a container and freeze to a container registry** + +To freeze a container, you must ensure the following conditions are met: + +- You created a Seqera Platform access token. +- You specified the destination container registry credentials in Seqera Platform. +- You specify the Seqera Platform access token via either the `TOWER_ACCESS_TOKEN` environment variable or the `--tower-token` Wave command line option. + +**Related CLI arguments** + +The following arguments are used to freeze a container build: + +- `--build-repo`: A target repository to save the built container to. +- `--freeze`: Enable a container freeze. +- `--tower-token`: A Seqera Platform auth token so that Wave can access your private registry credentials. Not required if the `TOWER_ACCESS_TOKEN` environment variable is set. +- `--tower-workspace-id`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. + +**Example usage** + +In the following example, the `alpine` container image is frozen to a private DockerHub image registry. The `--tower-token` argument is not required if the `TOWER_ACCESS_TOKEN` environment variable is defined. + +``` +wave -i alpine --freeze \ + --build-repo docker.io/user/repo --tower-token +``` + +[Nextflow]: https://www.nextflow.io/ +
diff --git a/docs/get-started.mdx b/docs/get-started.mdx new file mode 100644 index 000000000..b704aa691 --- /dev/null +++ b/docs/get-started.mdx @@ -0,0 +1,377 @@ +--- +title: Get started +--- + +Wave is versatile and you can leverage it in your Nextflow pipelines in several ways. The following guides describe how to quickly get started with [Seqera Containers and Nextflow](#nextflow-and-seqera-containers), [Nextflow and Wave integration](#nextflow), and the [Wave CLI](#wave-cli). + +## Nextflow and Seqera Containers + +You can provision containers that include [Conda packages][conda] through [Seqera Containers][sc] and use them directly in your Nextflow pipelines. + +**Prerequisites** + +- You have either [Docker Desktop] or [Podman] installed locally. +- You have [Nextflow] 23.10.x or newer installed locally. + +In this guide, you'll request a containerized Conda package from Seqera Containers. + +### Request a Conda package as a Seqera Container + +1. Open [Seqera Containers][sc] in a browser. +1. In the search box, enter `faker`. +1. In the search results, select **Add** in the `conda-forge::faker` result, and then **Get Container** to initiate the container build. +1. From the **Fetching container** modal, copy the the durable container image URI that Seqera Containers provides. +1. Optional: Select **View build details** to watch Seqera Containers build the requested container in real time. + +### Create a Nextflow pipeline that uses the container + +Nextflow can use the container that Seqera Containers built in the previous section. Use the container URI from Seqera Containers in the `container` directive. + +1. In a terminal window, create a new directory for the Nextflow pipeline. +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + ``` + +1. Create a `main.nf` file with the following contents: + + ```groovy + process FAKER { + container '' + debug true + + """ + faker address + """ + } + + workflow { + FAKER() + } + ``` + + Substitute `` for the container URI that you received from Seqera Containers in the previous section. + +### Run the Nextflow pipeline + +To confirm that the `faker` command is available from your pipeline, run the following command: + +``` +nextflow run main.nf +``` + +The output from a successful execution is displayed in the following example: + +``` +Launching `main.nf` [jolly_edison] DSL2 - revision: 5c414bd927 + +executor > local (1) +[86/0d56e8] faker | 1 of 1 ✔ +234 Nicholas Circle +Masonport, MS 98018 +``` +## Nextflow + +You can provision containers with Wave directly from your Nextflow pipelines. + +**Prerequisites** + +- You have an account with a container registry, such as DockerHub, and an access token that provides write access to your container repository. +- You have a [Seqera Cloud](https://cloud.seqera.io/login) or Platform account, to store your container registry credentials for Wave to use. +- You have either [Docker Desktop] or [Podman] installed locally. +- You have [Nextflow] 23.10.x or newer installed locally. + +In this guide, you'll build a container from a Nextflow module and freeze that package in your private container repository. + +### Create your Seqera access token + +1. Log in to Seqera. +1. From your personal workspace: Go to the user menu and select **Settings > Your tokens**. +1. Select **Add token**. +1. Enter a unique name for your token, then select **Add**. +1. Copy and store your token securely. + + :::caution + The access token is displayed only once. Save the token value before closing the **Personal Access Token** window. + ::: + +1. In a terminal window, assign your access token to the `TOWER_ACCESS_TOKEN` environment variable: + + ``` + export TOWER_ACCESS_TOKEN= + ``` + +### Add your container registry credentials to Seqera + +When freezing a container to the build repository that you specify, Wave uses Seqera to obtain your registry access credentials. If you use Docker as your container registry, complete the following steps: + +To create your access token in Docker Hub: + +1. Log in to [Docker Hub](https://hub.docker.com/). +2. Select your username in the top right corner and select **Account Settings**. +3. Select **Security > New Access Token**. +4. Enter a token description and select **Read-only** from the Access permissions drop-down menu, then select **Generate**. +5. Copy and save the generated access token (this is only displayed once). + +To add your credentials to Seqera: + +1. Add your credentials to your organization or personal workspace: + - From an organization workspace: Go to **Credentials > Add Credentials**. + - From your personal workspace: From the user menu, go to **Your credentials > Add credentials**. + +2. Complete the following fields: + + - **Name**: Specify a unique name for the credentials using alphanumeric characters, dashes, or underscores. For example, `my-registry-creds`. + - **Provider**: Select **Container registry**. + - **User name**: Specify your Docker username. For example, `user1`. + - **Password**: Specify your personal access token (PAT). For example, `1fcd02dc-...215bc3f3`. + - **Registry server**: Specify the container registry hostname, excluding the protocol. For example, `docker.io`. + +3. After you've completed all the form fields, select **Add**. The new credential is now listed under the **Credentials** tab. + +Seqera supports other container registries, such as GitHub and Quay.io. + +### Create a Nextflow pipeline that uses Wave + +Nextflow can use Wave to seamlessly build a container directly from a Dockerfile in your pipeline. + +1. In a terminal window, create a new directory for the Nextflow pipeline. + +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + + wave { + build.repository = '' + wave.freeze = true + } + + tower { + accessToken = "$TOWER_ACCESS_TOKEN" + } + ``` + + The `` must be substituted with your private container repository. + +1. Create a `wave.nf` file with the following contents: + + ```groovy + include { HELLO } from './modules/gamma' + + workflow { + HELLO() + } + ``` + +1. Create a directory for the module: + ``` + mkdir -p modules/gamma + ``` + +1. Create the `modules/gamma/main.nf` file for the module: + + ``` + process HELLO { + debug true + + """ + cowsay Hello! + """ + } + ``` + +1. Create the `modules/gamma/Dockerfile` file for the module: + + ``` + FROM alpine + + RUN apk update && apk add bash cowsay \ + --update-cache \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/community \ + --repository https://alpine.global.ssl.fastly.net/alpine/edge/main \ + --repository https://dl-3.alpinelinux.org/alpine/edge/testing + + RUN echo hello + ``` + +### Run the Nextflow pipeline + +To run the pipeline, and initiate the Wave container build, enter the following command: + +``` +nextflow run wave.nf -with-wave +``` + +The output from a successful execution is displayed in the following example: + +``` +Launching `wave.nf` [naughty_wiles] DSL2 - revision: 3756d705d9 + +executor > local (1) +[c1/6d7d9d] HELLO | 1 of 1 ✔ + ________ +< Hello! > + -------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` + +## Wave CLI + +With the Wave CLI, you can provision containers for later use in your Nextflow pipelines. + +**Prerequisites** + +For this guide, you must satisfy the following prerequisites: + +- You have an account with a container registry, such as DockerHub, and an access token that provides write access to your container repository. +- You have a [Seqera Cloud](https://cloud.seqera.io/login) or Platform account, to store your container registry credentials for Wave to use. +- You have either [Docker Desktop] or [Podman] installed locally. +- You have [Nextflow] 23.10.x or newer installed locally. +- You have the [Wave CLI][wave-cli] installed locally. + +In this guide, you'll build a container from a [Conda package][conda] and freeze that package in your private container repository. By freezing the container, you ensure that you can always use it in your Nextflow pipelines. + +### Create your Seqera access token + +1. Log in to Seqera. +1. From your personal workspace: Go to the user menu and select **Settings > Your tokens**. +1. Select **Add token**. +1. Enter a unique name for your token, then select **Add**. +1. Copy and store your token securely. + + :::caution + The access token is displayed only once. Save the token value before closing the **Personal Access Token** window. + ::: + +1. In a terminal window, assign your access token to the `TOWER_ACCESS_TOKEN` environment variable: + + ``` + export TOWER_ACCESS_TOKEN= + ``` + +### Add your container registry credentials to Seqera + +When freezing a container to the build repository that you specify, Wave uses Seqera to obtain your registry access credentials. If you use Docker as your container registry, completing the following steps: + +To create your access token in Docker Hub: + +1. Log in to [Docker Hub](https://hub.docker.com/). +2. Select your username in the top right corner and select **Account Settings**. +3. Select **Security > New Access Token**. +4. Enter a token description and select **Read-only** from the Access permissions drop-down menu, then select **Generate**. +5. Copy and save the generated access token (this is only displayed once). + +To add your credentials to Seqera: + +1. Add your credentials to your organization or personal workspace: + - From an organization workspace: Go to **Credentials > Add Credentials**. + - From your personal workspace: From the user menu, go to **Your credentials > Add credentials**. + +2. Complete the following fields: + + - **Name**: Specify a unique name for the credentials using alphanumeric characters, dashes, or underscores. For example, `my-registry-creds`. + - **Provider**: Select **Container registry**. + - **User name**: Specify your Docker username. For example, `user1`. + - **Password**: Specify your personal access token (PAT). For example, `1fcd02dc-...215bc3f3`. + - **Registry server**: Specify the container registry hostname, excluding the protocol. For example, `docker.io`. + +3. After you've completed all the form fields, select **Add**. The new credential is now listed under the **Credentials** tab. + +Seqera supports other container registries, such as GitHub and Quay.io. + +### Create and freeze a container + +Wave lets you build a container from any conda package or set of conda packages that you specify. In this section, you'll use Wave to build a container that includes the `faker` conda package. + +In the same terminal window from the previous section, run the Wave CLI to build and freeze a container with the `faker` conda package to the repository that you specify. Specify the URI for the repository for which you added an access token to Seqera, so that Wave can push the built container. + +``` +wave --conda-package 'faker' --freeze --build-repo --await +``` + +Example output: + +``` +docker.io/example-user/repo:faker--2aa7a4d826a76301 +``` + +After the container build completes, Seqera emails you a build status report, including the build logs for the container. + +### Create a Nextflow pipeline that uses the container + +Nextflow can use the container that Wave froze to the build repository that you provided in the previous section. The Wave CLI outputs the URI for the container image and the image tag. Use these values in the `container` directive. + +1. In a terminal window, create a new directory for the Nextflow pipeline. +1. Create a `nextflow.config` file with the following contents: + + ```groovy + docker { + enabled = true + } + + tower { + accessToken = "$TOWER_ACCESS_TOKEN" + } + ``` + +1. Create a `main.nf` file with the following contents: + + ```groovy + process FAKER { + container 'docker.io/example-user/repo:faker--2aa7a4d826a76301' + debug true + + """ + faker address + """ + } + + workflow { + FAKER() + } + ``` + +### Run the Nextflow pipeline + +To confirm that the `faker` command is available from your pipeline, run the following command: + +``` +nextflow run main.nf +``` + +The output from a successful execution is displayed in the following example: + +``` +Launching `./main.nf` [happy_leavitt] DSL2 - revision: 03b4e42ba3 + +executor > local (1) +[1d/120069] FAKER | 1 of 1 ✔ +1287 Eric Grove +Reneechester, AK 75644 +``` + +## Next steps + +- Explore additional [Wave CLI][wave-build] use cases. +- Review the available [Nextflow configuration][nf-config] options for Wave. + + +[sc]: https://seqera.io/containers/ +[conda]: https://docs.conda.io/en/latest/ +[Nextflow]: https://www.nextflow.io/docs/latest/install.html +[Podman]: https://podman.io/docs/installation +[Docker Desktop]: https://www.docker.com/products/docker-desktop/ + +[wave-build]: ./cli/build.mdx +[nf-config]: ./nextflow.mdx#configuration-options +[wave-cli]: ./cli/index.mdx diff --git a/docs/guide.mdx b/docs/guide.mdx deleted file mode 100644 index 32d5dc9b2..000000000 --- a/docs/guide.mdx +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: User guide ---- - -## Getting started - -Wave containers can be used with any container runtime supporting the [Docker Registry API v2](https://docs.docker.com/registry/spec/api/) and it's smoothly integrated with [Nextflow](https://www.nextflow.io/) and [Seqera Platform](https://cloud.tower.nf/). - -This feature requires Nextflow `22.10.0` or later. - -### Nextflow installation - -If you've already installed Nextflow, update to the latest version using this command:: - -```bash -nextflow -self-update -``` - -If you don't have Nextflow already installed, install it with the command below: - -```bash -curl get.nextflow.io | bash -``` - -### Wave configuration - -Wave can be used in any Nextflow pipeline by adding the following snippet to your `nextflow.config` file: - -```groovy -wave { - enabled = true -} - -tower { - accessToken = '' -} -``` - -:::tip -The use of the Seqera access token is not mandatory, however, it's required to enable access to private repositories and it allows higher service rate limits compared to anonymous users. -::: - -## API limits - -The Wave service implements API rate limits for API calls. Authenticated users have higher rate limits than anonymous users. - -If an access token is provided, the following rate limits apply: - -- 100 container builds per hour -- 1,000 container pulls per minute - -If an access token isn't provided, the following rate limits apply: - -- 25 container builds per day -- 250 container pulls per hour - -## Known limitation - -### Use of sha256 digest in the image name - -The Wave does not support the use of sha256 digest in the image name, e.g. `ubuntu@sha256:3235...ce8f`, when using -the augmentation process to extend container images. - -In order to reference a container via sha256 digest in the image name with Wave you will need to *freeze* image mode -that will force the creation of a new container image using the container you have specified as base image. - -In your pipeline configuration, ensure that you specify the following settings: - -```groovy -wave.enabled = true -wave.freeze = true -wave.strategy = ['dockerfile'] -wave.build.repository = 'docker.io//' -``` - -## Tutorials - -### Authenticate private repositories - -Wave allows the use of private repositories in your Nextflow pipelines. The repository access keys must be provided via [Seqera credentials](https://help.tower.nf/23.1/credentials/overview/). - -When you have created the credentials, you only need to specify your [Seqera access token](https://help.tower.nf/23.1/api/overview/#authentication) in your pipeline configuration file. If the credentials were created in a Seqera organization workspace, specify the workspace ID as well in the config file as shown below: - -```groovy -tower { - accessToken = '' - workspaceId = '' -} -``` - -That's it. When launching the pipeline execution, Wave will allow Nextflow to access the private container repositories defined in your pipeline configuration, using the credentials stored in the Seqera Platform credentials manager. - -### Build Nextflow module containers - -Wave can build and provision container images on-demand for your Nextflow pipelines. - -To enable this feature, add the `Dockerfile` of the container to be built in the [module directory](https://www.nextflow.io/docs/latest/dsl2.html#module-binaries) where the pipeline process is defined. When Wave is enabled, it automatically uses the Dockerfile to build the required container, upload to the registry, and uses the container to carry out the tasks defined in the module. - -:::tip -Make sure the process does not declare a `container` directive, otherwise it will take precedence over the Dockerfile definition. -::: - -If a process uses a `container` directive and you still want to build the container using the `Dockerfile` provided in the module directory, add the following setting to the pipeline config file: - -```groovy -wave.strategy = ['dockerfile','container'] -``` - -The above line instructs Wave to give the module `Dockerfile` priority over process `container` directives. - -:::caution -Wave currently does not support `ADD`, `COPY` and other Dockerfile commands that access files in the host file system. -::: - -### Build Conda based containers - -Wave allows the provisioning of containers based on the [`conda` directive](https://www.nextflow.io/docs/latest/process.html#conda) used by the processes in your Nextflow pipeline. This is a quick alternative to building Conda packages in the local computer. Moreover, this enables the use of Conda packages in your pipeline when deploying it in cloud-native platforms such as AWS Batch and Kubernetes, which do not allow the (easy) use of the Conda package manager. - -Having Wave enabled in your pipeline, there's nothing else to do other than define the `conda` requirements in the pipeline processes provided the same process does not also specify a `container` directive or a Dockerfile. - -In the latter case, add the following setting to your pipeline configuration: - -```groovy -wave.strategy = ['conda'] -``` - -The above setting instructs Wave to only use the `conda` directive to provision the pipeline containers, ignoring the use of the `container` directive and any `Dockerfile`(s). - -### Store container images into a private repository - -Containers built by Wave are uploaded to the Wave default repository hosted on AWS ECR with name `195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build`. The images in this repository are automatically deleted 1 week from the date of their push. - -If you want to store Wave containers in your own container repository use the following settings in the Nextflow configuration file: - -```groovy -wave.build.repository = 'example.com/your/build-repo' -wave.build.cacheRepository = 'example.com/your/cache-repo' -``` - -The first repository is used to store the built container images. The second one is used to store the individual image layers for caching purposes. - -The repository access keys need to be specified using the Seqera Platform credentials manager as specified in the [Authenticate private repositories](#Authenticate private repositories) section. - -## Advanced settings - -The following configuration options are available: - -| Method | Description | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `wave.enabled` | Enable/disable the execution of Wave containers | -| `wave.endpoint` | The Wave service endpoint (default: `https://wave.seqera.io`) | -| `wave.build.repository` | The container repository where image built by Wave needs to be uploaded (note: the corresponding credentials need to be provided in your Seqera Platform account). | -| `wave.build.cacheRepository` | The container repository used to cache image layers build by the Wave service (note: the corresponding credentials need to be provided in your Seqera Platform account). | -| `wave.conda.mambaImage` | The Mamba container image is used to build Conda based container. This is expected to be [micromamba-docker](https://github.com/mamba-org/micromamba-docker) image. | -| `wave.conda.commands` | One or more commands to be added to the Dockerfile used by build a Conda based image. | -| `wave.strategy` | The strategy to be used when resolving ambiguous Wave container requirement (default: `'container,dockerfile,conda'`) | -| `wave.freeze` | When `freeze` mode is enabled containers provisioned by Wave are stored permanently in the repository specified via the setting `wave.build.repository`. | - -## More examples - -Check out the [Wave showcase repository](https://github.com/seqeralabs/wave-showcase) for more examples how to use Wave containers. diff --git a/docs/index.mdx b/docs/index.mdx index 8695b60f1..a9dc1017f 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -1,52 +1,77 @@ --- -title: Wave containers +title: Wave overview --- -Containers are an essential part of data analysis in the cloud. Building and delivering optimized, context-aware container images slows down development. +Containers are an essential part of modern data analysis pipelines in bioinformatics. They encapsulate applications and dependencies in portable, self-contained packages that can be easily distributed across diverse computing environments. Containers are also key to enabling predictable and reproducible scientific results. However, workflows can comprise dozens of distinct container images. Pipeline developers must manage and maintain these container images and ensure that their functionality precisely aligns with the requirements of every pipeline task, creating unnecessary friction in the maintenance and deployment of data pipelines. -Wave is a container provisioning service designed for use with data analysis applications such as Nextflow. +Wave solves this problem by provisioning containers on-demand during the pipeline execution. This allows the delivery of container images that are defined precisely depending on the requirements of each pipeline task in terms of dependencies and platform architecture. This process is completely transparent and fully automated, removing all the plumbing and friction commonly needed to create, upload, and maintain dozens of container images that might be required by a pipeline execution. -It allows for the on-demand assembly, augmentation, and deployment of containerized images based on task requirements. +To get started with Wave: -The Wave container service itself is not a container registry. All containers builds are stored in a Seqera-hosted image registry for a limited time or frozen to a user-specified container registry. +1. See the [Get started][started] guide. +1. Learn about [Nextflow integration][nf]. +1. Learn about the [Wave CLI][cli]. :::note Wave is also available as hosted service on [Seqera Platform](https://cloud.seqera.io/). For Seqera Enterprise customers, a licensed self-hosted Wave solution is also available. Contact us [contact us](https://seqera.io/contact-us/) for more information. ::: -## Features +[started]: ./get-started.mdx +[nf]: ./nextflow.mdx +[cli]: ./cli/index.mdx -### Private container registries +## Wave features -Container registry authentication is the new norm. Yet when it comes to authenticating against cloud-specific container registries, the process is hardly hassle free. -Wave integrates with Seqera Platform credentials management enabling seamless access and publishing to private registries. +### Container registries -### Augment existing containers +#### Private container registries + +Wave integrates with [Seqera Platform credentials management][private] enabling seamless access and publishing to private registries. + +[private]: ./nextflow.mdx#access-private-container-repositories + +#### Seqera Containers - The community container registry -Regulatory and security requirements sometimes dictate specific container images, but additional context is often needed. -Wave enables any existing container to be extended without rebuilding it. Developers can add user-provided content such as custom scripts and logging agents, providing greater flexibility in the container’s configuration. +[Seqera Containers] is a free to use service operated for the community by Seqera. -Wave offers a flexible approach to container image management. It allows you to dynamically add custom layers to existing docker images, creating new images tailored to your specific needs. +It uses Wave to build images from Conda / PyPI packages on demand, either through the [web interface](https://seqera.io/containers/) or using the [Wave CLI](./cli/index.mdx) / [Nextflow integration](./nextflow.mdx). -#### An example of Wave augmentation +These images are cached and hosted permanently, being served through a [Docker Distribution][docker] registry and hosted on AWS infrastructure. Images are cached and served via Cloudflare CDN. -Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your Docker client and the registry. When you request an augmented image, Wave intercepts the process. +Images are publicly accessible to anyone for free and will be stored for at least 5 years. They can be pulled using any infrastructure (local, HPC, cloud) as Docker or native Singularity images. Images can be built for both `linux/aarch64` and `linux/arm64` architectures. -1. Base image layers download: The Docker client downloads the standard Ubuntu layers from the registry. -2. Custom layer injection: Wave injects your custom layer, denoted by "ω", which could represent application code, libraries, configurations etc. -3. New image creation: Wave combines the downloaded Ubuntu layers with your custom layer, effectively creating a new image on the fly. +:::note +Seqera Containers does not work with custom container files, augmentation, or authorization. It provides only Conda based containers. +::: -![](_images/wave_container_augmentation.png) +[docker]: https://github.com/distribution/distribution +[Seqera Containers]: https://seqera.io/containers/ + +### Augment existing containers -#### Benefits of Wave augmentation +Wave offers a flexible approach to container image management. It allows you to [dynamically add custom layers][augment] to existing docker images, creating new images tailored to your specific needs. +Any existing container can be extended without rebuilding it. You can add user-provided content such as custom scripts and logging agents, providing greater flexibility in the container’s configuration. -1. Streamlined workflows: Wave simplifies your workflow by eliminating the need to manually build and manage custom images. -2. Flexibility: You can easily modify the custom layer for different use cases, allowing for greater adaptability. +[augment]: ./provisioning.mdx#container-augmentation ### Conda-based containers -Package management systems such as Conda and Bioconda simplify the installation of scientific software. However, there’s considerable friction when it comes to using those tools to deploy pipelines in cloud environments. -Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just declare the Conda packages in your Nextflow pipeline and Wave will assemble the required container. +Package management systems such as Conda and Bioconda simplify the installation of scientific software. +Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just [declare the Conda packages][conda] in your Nextflow pipeline and Wave will assemble the required container. + +[conda]: ./nextflow.mdx#build-conda-based-containers + +### Singularity containers + +Singularity and Apptainer use a proprietary format called _Singularity Image Format_ (SIF). The Wave service can [provision containers based on the Singularity image format][singularity] either by using a `Singularityfile` file or Conda packages. The resulting Singularity image file is stored as an ORAS artifact in an OCI-compliant container registry of your choice or the Wave Community registry. + +The advantage of this approach is that Singularity and Apptainer engines can pull and execute those container images natively without requiring extra conversion steps, as needed when using Docker images with those two engines. + +:::note +Considering the Singularity image format's peculiarities, Wave's freeze mode is mandatory when provisioning Singularity images. +::: + +[singularity]: ./nextflow.mdx#build-singularity-containers ### Deploying containers across multi-clouds @@ -60,7 +85,7 @@ Builds for OCI-compliant container images are automatically scanned for known se ### Optimize workloads for specific architectures -Modern data pipelines can be deployed across different data centers having different hardware architectures. e.g., amd64, arm64, and others. This requires curating different collections of containers for each architecture. +Modern data pipelines can be deployed across different data centers having different hardware architectures such as amd64, arm64, and others. This requires curating different collections of containers for each architecture. Wave allows for the on-demand provisioning of containers, depending on the target execution platform (in development). ### Near caching diff --git a/docs/metrics.mdx b/docs/metrics.mdx index cd504f33a..1d3104818 100644 --- a/docs/metrics.mdx +++ b/docs/metrics.mdx @@ -1,5 +1,5 @@ --- -title: Wave usage metrics +title: Usage metrics --- Wave uses Redis to store its usage metrics for a specific date and/or a specific organization. @@ -50,7 +50,7 @@ Then, if the pulled container uses fusion, it increments the values of following ## How keys are created - When a request is made to wave, first it increments the key with current date. e.g. `builds/d/2024-04-23`. -- Keys with organisation are only incremented if the user is authenticated means there is Seqera platform token in the request. +- Keys with organization are only incremented if the user is authenticated means there is Seqera platform token in the request. - Wave extract the domain from the user email id (For example: `test_metrics@seqera.io`), which it gets from Seqera platform using the access token. -- In this case, The organisation value will be `seqera.io`. -- Then it increments the key with organisation. For example: `builds/o/seqera.io/d/2024-04-23` and `builds/o/seqera.io`. +- In this case, The organization value will be `seqera.io`. +- Then it increments the key with organization. For example: `builds/o/seqera.io/d/2024-04-23` and `builds/o/seqera.io`. diff --git a/docs/nextflow.mdx b/docs/nextflow.mdx new file mode 100644 index 000000000..ec9d246e8 --- /dev/null +++ b/docs/nextflow.mdx @@ -0,0 +1,215 @@ +--- +title: Nextflow integration +--- + +You can use Wave directly from your Nextflow pipelines, with full support for private repositories, container freezing, and conda packages. + +This feature requires Nextflow `22.10.0` or later. + +You can use Wave can in any Nextflow pipeline by adding the following snippet to your `nextflow.config` file: + +```groovy +wave { + enabled = true +} + +tower { + accessToken = '' +} +``` + +The use of the Seqera access token is not mandatory. However, it grants the following additional capabilities: + +- Access to private repositories +- More API requests than is permitted for anonymous users + +For the entire list of configuration options, see [Configuration options](#configuration-options). + +## Use Wave with Nextflow + +The following sections describe several common usage cases. To get started by creating an example pipeline that uses Wave, see [Get started][start]. + +[start]: ./get-started.mdx#nextflow + +### Access private container repositories + +Wave allows the use of private repositories in your Nextflow pipelines. The repository access keys must be provided in the form of [Seqera Platform credentials][credentials]. + +
+**Access private container repositories** + +After creating the credentials, specify your [personal access token][pat] in your pipeline configuration file. If the credentials were created in a Seqera Platform organization workspace, specify the workspace ID as well in the config file as shown below: + +```groovy +tower { + accessToken = '' + workspaceId = '' +} +``` + +Containers built by Wave are uploaded to the Wave default repository hosted on AWS ECR with name `195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build`. The images in this repository are automatically deleted 1 week from the date of their push. + +If you want to store Wave containers in your own container repository use the following settings in the Nextflow configuration file: + +```groovy +wave.build.repository = 'example.com/your/build-repo' +wave.build.cacheRepository = 'example.com/your/cache-repo' +``` + +The first repository is used to store the built container images. The second one is used to store the individual image layers for caching purposes. + +When launching the pipeline execution, Wave allows Nextflow to access the private container repositories defined in your pipeline configuration, by using the credentials stored in the Seqera Platform credentials manager. + + +[credentials]: /platform_versioned_docs/version-23.4.0/credentials/overview +[pat]: /platform_versioned_docs/version-23.4.0/api/overview#authentication +
+ +### Build Nextflow modules containers + +Wave can build and provision container images on-demand for your Nextflow pipelines. + +
+**Build Nextflow modules containers** + +To enable this feature, add the Dockerfile of the container to be built in the [module directory][module-directory] where the pipeline process is defined. When Wave is enabled, it automatically uses the Dockerfile to build the required container, upload to the registry, and it uses the container to execute the script defined in the process. + +Make sure the process does not declare a `container` directive, otherwise it will take precedence over the Dockerfile definition. + +If a process uses a `container` directive and you still want to build the container using the Dockerfile provided in the module directory, add the following setting to the pipeline config file: + +```groovy +wave.strategy = ['dockerfile','container'] +``` + +This setting instructs Wave to prioritize the module Dockerfile over process `container` directives. + +:::warning +When building containers, Wave currently does not support `ADD`, `COPY`, or any other Dockerfile commands that access files in the host file system. +::: + +[module-directory]: https://www.nextflow.io/docs/latest/module.html#module-directory +
+ +### Build Conda-based containers + +Wave allows the provisioning of containers based on the `process-conda` [directive][process-conda] used by the processes in your pipeline. This is a quick alternative to building Conda packages in the local computer. Moreover, this enables the use of Conda packages in your pipeline when deploying in cloud-native platforms such as AWS Batch and Kubernetes, which do not allow the convenient use of the Conda package manager. + +
+**Build Conda-based containers** + +With Wave enabled in your pipeline, simply define the `conda` requirements in the pipeline processes, provided the same process does not also specify a `container` directive or a Dockerfile. + +In the latter case, add the following setting to your pipeline configuration: + +```groovy +wave.strategy = ['conda'] +``` + +The above setting instructs Wave to use the `conda` directive to provision the pipeline containers and ignore the `container` directive and any Dockerfile(s). + +For versions of Nextflow 23.10.x or newer, when a container is provisioned, the `conda-forge::procps-ng` package is included automatically. This package includes the `ps` command. + +Some configuration options in the `conda` scope are used when Wave is used to build Conda-based containers. +For example, the Conda channels and their priority can be set with `conda.channels`: + +```groovy +wave.strategy = ['conda'] +conda.channels = 'seqera,conda-forge,bioconda,defaults' +``` + +[process-conda]: https://www.nextflow.io/docs/latest/process.html#conda +
+ +### Build Singularity containers + +Nextflow can build Singularity native images on-demand using `Singularityfile`, +Conda packages. The Singularity images are automatically uploaded in a container registry OCI compliant +of your choice and stored as a [ORAS artifact](https://oras.land/). + +:::note +Available as of Nextflow version 23.09.0-edge. +::: + +
+**Build Singularity containers** + +:::note +This feature requires a version of Singularity (or Apptainer) that supports pulling images using the `oras:` pseudo-protocol. +::: + +For example to enable the provisioning of Singularity images in your pipeline use the following configuration snippet: + +```groovy +singularity.enabled = true +wave.enabled = true +wave.freeze = true +wave.strategy = ['conda'] +wave.build.repository = 'docker.io/user/repo' +``` + +In the above configuration replace `docker.io/user/repo` with a repository of your choice where Singularity image files +should be uploaded. + +When using a private repository, the repository access keys must be provided via the Seqera Platform credentials manager. For more information, see [Authenticate private repositories][private]. + +Moreover the access to the repository must be granted in the compute nodes by using the command `singularity remote login `. +Please see Singularity documentation for further details. + +In order to build Singularity native images, both `singularity.ociAutoPull` and `singularity.ociMode` must be disabled in the configuration. For more information, see the Nextflow [configuration][config] documentation. + +[private]: https://docs.seqera.io/platform/24.1/credentials/overview +[config]: https://www.nextflow.io/docs/latest/config.html#config-singularity +
+ +### Use Wave with Fusion + +Wave containers allows you to run your containerized workflow with the Fusion file system. + +
+**Use Wave with Fusion** + +This enables the use of an object storage bucket such as AWS S3 or Google Cloud Storage as your pipeline work directory, simplifying and speeding up many operations on local, AWS Batch, Google Batch or Kubernetes executions. + +For more information, refer to the following documentation pages: + +- [Fusion documentation][fusion] +- [Nextflow Fusion integration documentation][nextflow-fusion] + +[fusion]: https://docs.seqera.io/fusion +[nextflow-fusion]: https://www.nextflow.io/docs/latest/fusion.html +
+ +## Configuration options + +The following configuration options are available: + +| Method | Description | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `wave.enabled` | Enable/disable the execution of Wave containers | +| `wave.endpoint` | The Wave service endpoint (default: `https://wave.seqera.io`) | +| `wave.build.repository` | The container repository where image built by Wave needs to be uploaded (note: the corresponding credentials need to be provided in your Seqera Platform account). | +| `wave.build.cacheRepository` | The container repository used to cache image layers build by the Wave service (note: the corresponding credentials need to be provided in your Seqera Platform account). | +| `wave.conda.mambaImage` | The Mamba container image is used to build Conda based container. This is expected to be [micromamba-docker](https://github.com/mamba-org/micromamba-docker) image. | +| `wave.conda.commands` | One or more commands to be added to the Dockerfile used by build a Conda based image. | +| `wave.strategy` | The strategy to be used when resolving ambiguous Wave container requirement (default: `'container,dockerfile,conda'`) | +| `wave.freeze` | When `freeze` mode is enabled containers provisioned by Wave are stored permanently in the repository specified via the setting `wave.build.repository`. | + +## Limitations + +### Use of sha256 digest in the image name + +The Wave does not support the use of sha256 digest in the image name, e.g. `ubuntu@sha256:3235...ce8f`, when using +the augmentation process to extend container images. + +In order to reference a container via sha256 digest in the image name with Wave you will need to *freeze* image mode +that will force the creation of a new container image using the container you have specified as base image. + +In your pipeline configuration, ensure that you specify the following settings: + +```groovy +wave.enabled = true +wave.freeze = true +wave.strategy = ['dockerfile'] +wave.build.repository = 'docker.io//' +``` diff --git a/docs/provisioning.mdx b/docs/provisioning.mdx new file mode 100644 index 000000000..caa975874 --- /dev/null +++ b/docs/provisioning.mdx @@ -0,0 +1,88 @@ +--- +title: How container provisioning works +--- + +In the container lifecycle, images are generally created (*built*) and uploaded (*pushed*) to a container registry, and then these images are downloaded (*pulled*) for the execution of a specific pipeline. + +The Wave container provisioning process streamlines the container lifecycle by delivering container images on-demand during the pipeline execution and making sure each container includes the dependencies required by the requesting actor, such as a pipeline task or a user. + +Wave provides the following container provisioning capabilities: + +- Container augmentation +- Container freezing + +## Container augmentation + +The container augmentation provisioning mode allows _extending_ the content of a container image without rebuilding it. Instead, this mechanism modifies a container image during the pull phase made by a Docker-compatible client. Augmented containers are ephemeral: they are not stored in a container repository, and they can only be accessed for a short period of time. The extended content added by Wave is served from a CDN. The augmentation process does not perform any _build_ operation behind the scenes. + +This approach supports use cases such as the following: + +- Authenticate access to your private repositories with Seqera Platform credentials +- Extend existing containers by adding infrastructure and pipeline dependencies on the fly, without rebuilding and maintaining additional container images + +Container augmentation works as follows: + +1. The client, either Nextflow or Wave CLI, submits a container request specifying: + 1. The Seqera Platform user identity + 1. The container image to be augmented + 1. The container extension configuration, which can be either a custom payload, one or more extension layers, or container images. +1. The Wave service validates the request and authorizes the user submitting a request to the Platform service. +1. The Wave service responds with an ephemeral container image name e.g. `wave.seqera.io/wt//library/alpine:latest`. The `` is uniquely assigned and is used to identify and authorize the following container request. +1. The Docker client uses the returned image name to pull the container binary content of the upstream image directly from the target registry. +1. The content added by Wave as one or more layer extensions is shipped by the Wave service. + +Notable parts of this workflow include: + +- Wave acts as a proxy between the Docker client and the target registry that hosts the container image. +- Wave modifies, if needed, the container manifest to add the new content as specified by the request, but it does not (and cannot) alter the container layer blob files that have a unique checksum, which is preserved. +- Image blobs are downloaded directly from the target registry, not from Wave. + +## Container freezes + +The container _freeze mode_ allows the provisioning of non-ephemeral containers that are stored permanently in a container registry of your choice. When using the freeze mode, the Wave service transparently carries out a regular container build. + +This approach supports use cases such as the following: + +- Create container images on-demand from Conda packages +- Deliver multi-architecture (AMD64 and ARM64) and multi-format (Docker and Singularity) container collections +- Deliver container images in the same region where compute is performed + +Wave freeze mode works as follows: + +1. The client, either Nextflow or the Wave CLI, submits a container request specifying: + 1. The Seqera Platform user identity + 1. The container image to augment + 1. The container extension configuration, which can be either a custom payload, one or more extension layers, or container images + 1. The target repository where the built container should be uploaded +1. The Wave service validates the request and authorizes the user via a request to the Platform service. +1. The Wave service checks if the container image already exists in the target registry. +1. If the image does not exist, Wave launches a container build job and pushes the resulting image to the target registry. +1. The Wave service responds with the container image name e.g. `example.com/some/image/build:1234567`. + +Notable parts of this workflow include: + +- Container images provisioned with freeze mode are regular container builds. +- Each container image is associated with a unique ID that is obtained by hashing the following elements: + - The Container file + - Any package dependencies + - The target platform, which is either AMD64 or ARM64 + - The target repository name +- When a request for the same container is made, the same ID is assigned to it and therefore, the build is skipped. +- The resulting images are hosted in your selected repository and not cached locally, unless a cache repository is specified. +- The container images are stored permanently unless the repository owner deletes them. + +## Container provisioning capability matrix + +Wave supports the following types of container builds: + +|Type|Provisioning mode|Source|Freeze|Build repo|Accessibility|Format| +|--- |--- |--- |--- |--- |--- |--- | +|Ephemeral|Augmentation|Container image|No|n/a|Temporary token|Docker| +|Ephemeral|Build|Container file|No|Default|Temporary token|Docker| +|Ephemeral|Build|Conda package|No|Default|Temporary token|Docker| +|Ephemeral|Build|Container file|No|Custom|Temporary token|Docker| +|Ephemeral|Build|Conda package|No|Custom|Temporary token|Docker| +|Durable|Build|Container file|Yes|Custom|Docker auth|Docker /Singularity| +|Durable|Build|Conda package|Yes|Custom|Docker auth|Docker /Singularity| +|Community (durable)|Build|Container file|Yes|Default|Public|Docker /Singularity| +|Community (durable)|Build|Conda package|Yes|Default|Public|Docker /Singularity| diff --git a/docs/sidebar.json b/docs/sidebar.json index cb6df70fa..e41da8226 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -1,28 +1,26 @@ { "sidebar": [ - "index", { "type": "category", - "label": "Wave CLI", + "label": "Wave", "collapsed": false, - "link": { - "type": "doc", - "id": "cli/index" - }, "items": [ - "cli/install", - "cli/build-directory", - "cli/build-docker", - "cli/build-conda", - "cli/build-spack", - "cli/build-singularity", - "cli/build-freeze" + "index", + "get-started", + "provisioning" ] }, - "guide", - "api", - "metrics", - "faq", - "troubleshoot" + "nextflow", + { + "type": "category", + "label": "Developer tools", + "collapsed": false, + "items": [ + "cli/index", + "cli/reference", + "api" + ] + }, + "faq" ] } From 463d0ebf655b6131f4c7e63ddf1509b96d24b53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20=CE=92oxman?= <141646877+jason-seqera@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:51:41 -0400 Subject: [PATCH 39/50] Fix docs link (#611) [ci skip] --- docs/get-started.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/get-started.mdx b/docs/get-started.mdx index b704aa691..b779c1e6c 100644 --- a/docs/get-started.mdx +++ b/docs/get-started.mdx @@ -372,6 +372,6 @@ Reneechester, AK 75644 [Podman]: https://podman.io/docs/installation [Docker Desktop]: https://www.docker.com/products/docker-desktop/ -[wave-build]: ./cli/build.mdx +[wave-build]: ./cli/reference.mdx#build-a-container [nf-config]: ./nextflow.mdx#configuration-options [wave-cli]: ./cli/index.mdx From fe9b42732bd16697e7aaefc915496f31d2163ba0 Mon Sep 17 00:00:00 2001 From: Llewellyn vd Berg <113503285+llewellyn-sl@users.noreply.github.com> Date: Thu, 22 Aug 2024 06:52:28 +0200 Subject: [PATCH 40/50] Update nextflow.mdx (#612) [ci skip] --- docs/nextflow.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/nextflow.mdx b/docs/nextflow.mdx index ec9d246e8..030fcac68 100644 --- a/docs/nextflow.mdx +++ b/docs/nextflow.mdx @@ -61,8 +61,8 @@ The first repository is used to store the built container images. The second one When launching the pipeline execution, Wave allows Nextflow to access the private container repositories defined in your pipeline configuration, by using the credentials stored in the Seqera Platform credentials manager. -[credentials]: /platform_versioned_docs/version-23.4.0/credentials/overview -[pat]: /platform_versioned_docs/version-23.4.0/api/overview#authentication +[credentials]: /platform_versioned_docs/version-24.1/credentials/overview +[pat]: /platform_versioned_docs/version-24.1/api/overview#authentication ### Build Nextflow modules containers From 62e42e829f745d402ceb0a71479991d1cceb324d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20=CE=92oxman?= <141646877+jason-seqera@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:53:51 -0400 Subject: [PATCH 41/50] Links to Platform docs must be external to resolve (#613) --- docs/nextflow.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/nextflow.mdx b/docs/nextflow.mdx index 030fcac68..ef2bec509 100644 --- a/docs/nextflow.mdx +++ b/docs/nextflow.mdx @@ -61,8 +61,8 @@ The first repository is used to store the built container images. The second one When launching the pipeline execution, Wave allows Nextflow to access the private container repositories defined in your pipeline configuration, by using the credentials stored in the Seqera Platform credentials manager. -[credentials]: /platform_versioned_docs/version-24.1/credentials/overview -[pat]: /platform_versioned_docs/version-24.1/api/overview#authentication +[credentials]: https://docs.seqera.io/platform/latest/credentials/overview +[pat]: https://docs.seqera.io/platform/latest/api/overview#authentication ### Build Nextflow modules containers From ec43fa0da2403a93ca548f02bd88136fb2eb2684 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 22 Aug 2024 20:47:17 +0200 Subject: [PATCH 42/50] Fix too many requests error code (#610) Signed-off-by: Paolo Di Tommaso Co-authored-by: Munish Chouhan --- .../groovy/io/seqera/wave/ErrorHandler.groovy | 2 +- .../RegistryControllerPullLimitTest.groovy | 98 +++++++++++++++++++ .../SpillwayRegistryControllerTest.groovy | 2 +- 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy diff --git a/src/main/groovy/io/seqera/wave/ErrorHandler.groovy b/src/main/groovy/io/seqera/wave/ErrorHandler.groovy index ad356e963..602385f33 100644 --- a/src/main/groovy/io/seqera/wave/ErrorHandler.groovy +++ b/src/main/groovy/io/seqera/wave/ErrorHandler.groovy @@ -95,7 +95,7 @@ class ErrorHandler { } if( t instanceof SlowDownException ) { - final resp = responseFactory.apply(msg, 'DENIED') + final resp = responseFactory.apply(msg, 'TOOMANYREQUESTS') return HttpResponseFactory.INSTANCE.status(HttpStatus.TOO_MANY_REQUESTS).body(resp) } diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy new file mode 100644 index 000000000..33b52493d --- /dev/null +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerPullLimitTest.groovy @@ -0,0 +1,98 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.controller + +import spock.lang.Shared +import spock.lang.Specification + +import io.micronaut.context.ApplicationContext +import io.micronaut.context.annotation.Requires +import io.micronaut.http.HttpRequest +import io.micronaut.http.HttpStatus +import io.micronaut.http.MediaType +import io.micronaut.http.client.HttpClient +import io.micronaut.http.client.annotation.Client +import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.exception.SlowDownException +import io.seqera.wave.model.ContentType +import io.seqera.wave.ratelimit.AcquireRequest +import io.seqera.wave.ratelimit.RateLimiterService +import io.seqera.wave.storage.ManifestCacheStore +import io.seqera.wave.test.DockerRegistryContainer +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test-rate-limit']) +class RegistryControllerPullLimitTest extends Specification implements DockerRegistryContainer{ + + @Requires(env = 'test-rate-limit') + @Singleton + static class Limit implements RateLimiterService { + + @Override + void acquireBuild(AcquireRequest request) throws SlowDownException { + throw new SlowDownException("Request exceeded expected build rate limit") + } + + @Override + void acquirePull(AcquireRequest request) throws SlowDownException { + throw new SlowDownException("Request exceeded expected pull rate limit") + } + + @Override + boolean acquireTimeoutCounter(String endpoint) { + return false + } + } + + @Inject + @Client("/") + HttpClient client; + + @Inject + @Shared + ApplicationContext applicationContext + + @Inject + ManifestCacheStore storage + + def setupSpec() { + initRegistryContainer(applicationContext) + } + + void 'should get manifest'() { + when: + HttpRequest request = HttpRequest.GET("/v2/library/hello-world/manifests/sha256:53641cd209a4fecfc68e21a99871ce8c6920b2e7502df0a20671c6fccc73a7c6").headers({h-> + h.add('Accept', ContentType.DOCKER_MANIFEST_V2_TYPE) + h.add('Accept', ContentType.DOCKER_MANIFEST_V1_JWS_TYPE) + h.add('Accept', MediaType.APPLICATION_JSON) + }) + client.toBlocking().exchange(request,String) + + then: + HttpClientResponseException e = thrown() + e.status == HttpStatus.TOO_MANY_REQUESTS + e.response.body() == '{"errors":[{"code":"TOOMANYREQUESTS","message":"Request exceeded expected pull rate limit"}]}' + } + +} diff --git a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy index c84164ece..f4ece7b3b 100644 --- a/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/ratelimit/SpillwayRegistryControllerTest.groovy @@ -90,7 +90,7 @@ class SpillwayRegistryControllerTest extends Specification implements DockerRegi e.message == "Client '/': Too Many Requests" def b = new JsonSlurper().parseText( e.response.body.get() as String) b.errors.size() - b.errors.first().code == 'DENIED' + b.errors.first().code == 'TOOMANYREQUESTS' b.errors.first().message.contains('Request exceeded pull rate limit for IP') } From d9b8cab8f1db0b139074b98f3bc403b451179381 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 23 Aug 2024 10:04:00 +0200 Subject: [PATCH 43/50] Improve contaiener view page (#615) Signed-off-by: Paolo Di Tommaso --- .../wave/controller/ViewController.groovy | 5 ++++- .../io/seqera/wave/container-view.hbs | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 5a41979c9..6234e8d2e 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -133,7 +133,10 @@ class ViewController { binding.build_conda_file = data.condaFile ?: '-' binding.build_repository = data.buildRepository ?: '-' binding.build_cache_repository = data.cacheRepository ?: '-' - + binding.build_id = data.buildId ?: '-' + binding.build_cached = data.buildId ? !data.buildNew : '-' + binding.build_freeze = data.buildId ? data.freeze : '-' + binding.fusion_version = data.fusionVersion ?: '-' return HttpResponse.>ok(binding) } diff --git a/src/main/resources/io/seqera/wave/container-view.hbs b/src/main/resources/io/seqera/wave/container-view.hbs index 1751a70f9..317d1ccc8 100644 --- a/src/main/resources/io/seqera/wave/container-view.hbs +++ b/src/main/resources/io/seqera/wave/container-view.hbs @@ -89,6 +89,10 @@ Wave digest {{wave_container_digest}} + + Fusion version + {{fusion_version}} + {{#if request_container_config}} @@ -105,6 +109,18 @@ 
{{build_conda_file}}
+ + + + + + + + + + + + @@ -116,8 +132,9 @@
Build ID{{build_id}}
Cached{{build_cached}}
Freeze mode{{build_freeze}}
Build repository {{build_repository}}
{{/if}} + - + {{#if tower_user_id}}

Tower & User info

From 6c05498c0506f95225c6a703f364861abde00cc0 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Fri, 23 Aug 2024 10:29:41 +0200 Subject: [PATCH 44/50] added /v1alpha2/container/{containerId} (#609) Signed-off-by: munishchouhan --- .../controller/ContainerController.groovy | 8 +++++++ .../controller/ContainerControllerTest.groovy | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index be89c905a..441234ece 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -470,4 +470,12 @@ class ContainerController { .header(WWW_AUTHENTICATE, "Basic realm=Wave Authentication") } + @Get('/v1alpha2/container/{containerId}') + HttpResponse getContainerDetails(String containerId) { + final data = persistenceService.loadContainerRequest(containerId) + if( !data ) + return HttpResponse.notFound() + return HttpResponse.ok(data) + } + } diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 5d3e1ef0e..0d707d001 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -53,6 +53,7 @@ import io.seqera.wave.service.inspect.ContainerInspectServiceImpl import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.token.ContainerTokenService import io.seqera.wave.service.token.TokenData import io.seqera.wave.service.validation.ValidationServiceImpl @@ -611,4 +612,27 @@ class ContainerControllerTest extends Specification { then: noExceptionThrown() } + + def '/v1alpha2/container/{containerId} should return a container record' () { + given: + def body = new SubmitContainerTokenRequest(containerImage: 'hello-world') + def req1 = HttpRequest.POST("/container-token", body) + def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) + and: + resp1.status() == HttpStatus.OK + and: + def containerId = resp1.body().containerToken + + when: + def req2 = HttpRequest.GET("/v1alpha2/container/${containerId}") + def resp2 = client.toBlocking().exchange(req2, WaveContainerRecord) + then: + resp2.status() == HttpStatus.OK + and: + def result = resp2.body() + and: + result.containerImage == 'hello-world' + result.sourceImage == 'docker.io/library/hello-world:latest' + result.waveImage == resp1.body().targetImage + } } From eff30a9bce138ba08e8829e0d087d96022a7de55 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 23 Aug 2024 10:51:00 +0200 Subject: [PATCH 45/50] [release] bump 1.11.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 18b311420..1cac385c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.4 +1.11.0 diff --git a/changelog.txt b/changelog.txt index 07c8a2cc6..2042d0315 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,11 @@ # Wave changelog +1.11.0 - 23 Aug 2024 +- added /v1alpha2/container/{containerId} (#609) [5221b5a0] +- Improve contaiener view page (#615) [9e15b455] +- Fix too many requests error code (#610) [cc9fa23f] +- Add trusted builds timeout (#600) [7500fc5f] +- Change k8s Job deletion to foreground pods propagation (#595) [816bd6b4] + 1.10.4 - 6 Aug 2024 - Fix Prevent NPE when creds are empty [14f0476d] - Remove ContentReader and ContentReaderFactory classes (#584) [f05623a2] From 5712996092598e0f2fb5f43fdffc7eafbf6f679b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 23 Aug 2024 11:56:30 +0200 Subject: [PATCH 46/50] Add link to build ID Signed-off-by: Paolo Di Tommaso --- src/main/groovy/io/seqera/wave/controller/ViewController.groovy | 1 + src/main/resources/io/seqera/wave/container-view.hbs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 6234e8d2e..60ba05537 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -136,6 +136,7 @@ class ViewController { binding.build_id = data.buildId ?: '-' binding.build_cached = data.buildId ? !data.buildNew : '-' binding.build_freeze = data.buildId ? data.freeze : '-' + binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : '#' binding.fusion_version = data.fusionVersion ?: '-' return HttpResponse.>ok(binding) diff --git a/src/main/resources/io/seqera/wave/container-view.hbs b/src/main/resources/io/seqera/wave/container-view.hbs index 317d1ccc8..78f18e99b 100644 --- a/src/main/resources/io/seqera/wave/container-view.hbs +++ b/src/main/resources/io/seqera/wave/container-view.hbs @@ -111,7 +111,7 @@
- + From 5cbd67a86e11bccef68e9b6b2c26c41874cd21aa Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Fri, 23 Aug 2024 12:16:22 +0200 Subject: [PATCH 47/50] added GET /v1alpha2/container/{token} in typespec (#618) added GET /v1alpha2/container/{token} in typespec --- typespec/models/User.tsp | 6 ++++++ typespec/models/WaveContainerRecord.tsp | 28 +++++++++++++++++++++++++ typespec/models/models.tsp | 3 ++- typespec/routes.tsp | 9 +++++++- 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 typespec/models/User.tsp create mode 100644 typespec/models/WaveContainerRecord.tsp diff --git a/typespec/models/User.tsp b/typespec/models/User.tsp new file mode 100644 index 000000000..883494f13 --- /dev/null +++ b/typespec/models/User.tsp @@ -0,0 +1,6 @@ +@doc("Wave USer details") +model User { + id: int64; + userName: string; + email: string; +} diff --git a/typespec/models/WaveContainerRecord.tsp b/typespec/models/WaveContainerRecord.tsp new file mode 100644 index 000000000..be2717b31 --- /dev/null +++ b/typespec/models/WaveContainerRecord.tsp @@ -0,0 +1,28 @@ +import "./User.tsp"; + +@doc("Wave container details") +model WaveContainerRecord { + user: User; + workspaceId: int64; + containerImage?: string; + containerFile?: string; + containerConfig: ContainerConfig; + condaFile?: string; + platform?: string; + towerEndpoint?: string; + buildRepository?: string; + cacheRepository?: string; + fingerprint?: string; + timestamp: string; + zoneId?: string; + ipAddress?: string; + sourceImage?: string; + sourceDigest?: string; + waveImage?: string; + waveDigest?: string; + expiration: string; + buildId?: string; + buildNew?: boolean; + freeze?: boolean; + fusionVersion?: string; +} diff --git a/typespec/models/models.tsp b/typespec/models/models.tsp index c63c2e6f3..e010857b1 100644 --- a/typespec/models/models.tsp +++ b/typespec/models/models.tsp @@ -6,4 +6,5 @@ import "./ContainerInspectResponse.tsp"; import "./MetricsResponse.tsp"; import "./WaveScanRecord.tsp"; import "./WaveBuildRecord.tsp"; -import "./ValidateRegistryCredsRequest.tsp"; \ No newline at end of file +import "./ValidateRegistryCredsRequest.tsp"; +import "./WaveContainerRecord.tsp"; diff --git a/typespec/routes.tsp b/typespec/routes.tsp index d3d1c7623..061e3e287 100644 --- a/typespec/routes.tsp +++ b/typespec/routes.tsp @@ -11,11 +11,18 @@ namespace wave { @route("/v1alpha2/container") interface ContainerService { - @post op createV1Alpha2Container(@body requestBody: ContainerRequest): { + @post op createContainer(@body requestBody: ContainerRequest): { @body response: ContainerResponse; @statusCode statusCode: 200; }; + @route("/{token}") + @get op getContainerDetails(@path token: string): { + @body response: WaveContainerRecord; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; } @route("/v1alpha1/builds/{buildId}") From 00daf9191048f67c7066c89db773ea1d788c9b3a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 27 Aug 2024 09:21:04 +0200 Subject: [PATCH 48/50] Add Job manager (#605) Signed-off-by: Paolo Di Tommaso Signed-off-by: munishchouhan Co-authored-by: munishchouhan --- README.md | 1 + .../wave/configuration/BlobCacheConfig.groovy | 3 - .../wave/service/blob/BlobCacheInfo.groovy | 24 +-- .../blob/impl/BlobCacheServiceImpl.groovy | 96 ++++++++-- .../blob/impl/DockerTransferStrategy.groovy | 38 ++-- .../blob/impl/KubeTransferStrategy.groovy | 34 ++-- .../blob/transfer/TransferManager.groovy | 163 ---------------- .../data/stream/AbstractMessageStream.groovy | 178 ++++++++++++++++++ .../service/data/stream/MessageStream.groovy | 52 +++++ .../stream/impl/LocalMessageStream.groovy | 76 ++++++++ .../stream/impl/RedisMessageStream.groovy | 154 +++++++++++++++ .../JobConfig.groovy} | 26 +-- .../wave/service/job/JobDispatcher.groovy | 85 +++++++++ .../JobHandler.groovy} | 25 +-- .../io/seqera/wave/service/job/JobId.groovy | 70 +++++++ .../seqera/wave/service/job/JobManager.groovy | 100 ++++++++++ .../seqera/wave/service/job/JobQueue.groovy | 69 +++++++ .../seqera/wave/service/job/JobService.groovy | 32 ++++ .../wave/service/job/JobServiceImpl.groovy | 52 +++++ .../Transfer.groovy => job/JobState.groovy} | 28 +-- .../JobStrategy.groovy} | 13 +- .../seqera/wave/service/k8s/K8sService.groovy | 2 +- .../wave/service/k8s/K8sServiceImpl.groovy | 2 +- src/main/resources/application-dev.yml | 7 +- .../encoder/MoshiEncodingStrategyTest.groovy | 17 ++ .../service/blob/BlobCacheInfoTest.groovy | 1 - ...st.groovy => DockerJobStrategyTest.groovy} | 2 +- ...Test.groovy => KubeJobStrategyTest.groovy} | 79 ++++---- .../blob/transfer/TransferManagerTest.groovy | 143 -------------- .../AbstractMessageStreamLocalTest.groovy | 59 ++++++ .../AbstractMessageStreamRedisTest.groovy | 68 +++++++ .../data/stream/LocalMessageStreamTest.groovy | 84 +++++++++ .../data/stream/RedisMessageStreamTest.groovy | 109 +++++++++++ .../service/data/stream/TestMessage.groovy | 31 +++ .../service/data/stream/TestStream.groovy | 42 +++++ .../seqera/wave/service/job/JobIdTest.groovy | 72 +++++++ .../wave/service/job/JobManagerTest.groovy | 128 +++++++++++++ .../transfer => job}/TransferTest.groovy | 29 +-- 38 files changed, 1701 insertions(+), 493 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy rename src/main/groovy/io/seqera/wave/service/{blob/transfer/TransferQueue.groovy => job/JobConfig.groovy} (64%) create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy rename src/main/groovy/io/seqera/wave/service/{blob/transfer/TransferTimeoutException.groovy => job/JobHandler.groovy} (66%) create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobId.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobManager.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobService.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy rename src/main/groovy/io/seqera/wave/service/{blob/transfer/Transfer.groovy => job/JobState.groovy} (69%) rename src/main/groovy/io/seqera/wave/service/{blob/transfer/TransferStrategy.groovy => job/JobStrategy.groovy} (77%) rename src/test/groovy/io/seqera/wave/service/blob/impl/{DockerTransferStrategyTest.groovy => DockerJobStrategyTest.groovy} (98%) rename src/test/groovy/io/seqera/wave/service/blob/impl/{KubeTransferStrategyTest.groovy => KubeJobStrategyTest.groovy} (65%) delete mode 100644 src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/TestMessage.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/data/stream/TestStream.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy rename src/test/groovy/io/seqera/wave/service/{blob/transfer => job}/TransferTest.groovy (65%) diff --git a/README.md b/README.md index 27558e7ee..cbbe3e5f6 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ container registry where the image is stored, while the instrumented layers are * Java 19 or later * Linux or macOS +* Redis 6.2 (or later) * Docker engine (for development) * Kubernetes cluster (for production) diff --git a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy index 8640206d6..2dd816d41 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy @@ -44,9 +44,6 @@ class BlobCacheConfig { @Value('${wave.blobCache.failure.duration:4s}') Duration failureDuration - @Value('${wave.blobCache.grace.duration:20s}') - Duration graceDuration - @Value('${wave.blobCache.timeout:10m}') Duration transferTimeout diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy index 29757a856..8511cc4ce 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobCacheInfo.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.service.blob import java.time.Duration import java.time.Instant -import com.google.common.hash.Hashing import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString @@ -31,7 +30,7 @@ import groovy.util.logging.Slf4j * @author Paolo Di Tommaso */ @Slf4j -@ToString(includePackage = false, includeNames = true) +@ToString(includePackage = false, includeNames = true, excludes = ['headers','logs']) @Canonical @CompileStatic class BlobCacheInfo { @@ -46,11 +45,6 @@ class BlobCacheInfo { */ final String objectUri - /** - * it is the name of k8s job or docker container depends on the transfer strategy - */ - final String jobName - /** * The request http headers */ @@ -119,7 +113,7 @@ class BlobCacheInfo { final type = headerString0(response, 'Content-Type') final cache = headerString0(response, 'Cache-Control') final creationTime = Instant.now() - return new BlobCacheInfo(locationUri, objectUri, generateJobName(locationUri, creationTime), headers0, length, type, cache, creationTime, null, null, null) + return new BlobCacheInfo(locationUri, objectUri, headers0, length, type, cache, creationTime, null, null, null) } static String headerString0(Map> headers, String name) { @@ -140,7 +134,6 @@ class BlobCacheInfo { new BlobCacheInfo( locationUri, objectUri, - jobName, headers, contentLength, contentType, @@ -156,7 +149,6 @@ class BlobCacheInfo { new BlobCacheInfo( locationUri, objectUri, - jobName, headers, contentLength, contentType, @@ -172,7 +164,6 @@ class BlobCacheInfo { new BlobCacheInfo( locationUri, objectUri, - jobName, headers, contentLength, contentType, @@ -188,7 +179,6 @@ class BlobCacheInfo { new BlobCacheInfo( location, objectUri, - jobName, headers, contentLength, contentType, @@ -201,7 +191,7 @@ class BlobCacheInfo { } static BlobCacheInfo unknown(String logs) { - new BlobCacheInfo(null, null, null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null, logs) { + new BlobCacheInfo(null, null, null, null, null, null, Instant.ofEpochMilli(0), Instant.ofEpochMilli(0), null, logs) { @Override BlobCacheInfo withLocation(String uri) { // prevent the change of location for unknown status @@ -210,12 +200,4 @@ class BlobCacheInfo { } } - static private String generateJobName(String locationUri, Instant creationTime) { - return 'transfer-' + Hashing - .sipHash24() - .newHasher() - .putUnencodedChars(locationUri) - .putUnencodedChars(creationTime.toString()) - .hash() - } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy index 4dc8d8978..a6c8fbeda 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.blob.impl import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -34,13 +35,16 @@ import io.seqera.wave.service.blob.BlobCacheInfo import io.seqera.wave.service.blob.BlobCacheService import io.seqera.wave.service.blob.BlobSigningService import io.seqera.wave.service.blob.BlobStore -import io.seqera.wave.service.blob.transfer.TransferQueue -import io.seqera.wave.service.blob.transfer.TransferStrategy +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobId +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobState import io.seqera.wave.util.Escape import io.seqera.wave.util.Retryable import io.seqera.wave.util.StringUtils import jakarta.annotation.PostConstruct import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS /** @@ -49,10 +53,11 @@ import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS * @author Paolo Di Tommaso */ @Slf4j +@Named('Transfer') @Singleton @Requires(property = 'wave.blobCache.enabled', value = 'true') @CompileStatic -class BlobCacheServiceImpl implements BlobCacheService { +class BlobCacheServiceImpl implements BlobCacheService, JobHandler { @Value('${wave.debug:false}') private Boolean debug @@ -67,10 +72,7 @@ class BlobCacheServiceImpl implements BlobCacheService { private RegistryProxyService proxyService @Inject - private TransferStrategy transferStrategy - - @Inject - private TransferQueue transferQueue + private JobService jobService @Inject private BlobSigningService signingService @@ -178,21 +180,18 @@ class BlobCacheServiceImpl implements BlobCacheService { return command } - protected void store(RoutePath route, BlobCacheInfo info) { - log.debug "== Blob cache begin for object '${info.locationUri}'" + protected void store(RoutePath route, BlobCacheInfo blob) { + log.debug "== Blob cache begin for object '${blob.locationUri}'" try { // the transfer command to be executed - final cli = transferCommand(route, info) - transferStrategy.transfer(info, cli) - // signal the transfer to be started - // note: both `transferQueue` and `blobStore` use the same object `id` - transferQueue.offer(info.id()) + final cli = transferCommand(route, blob) + jobService.launchTransfer(blob, cli) } catch (Throwable t) { - log.warn "== Blob cache failed for object '${info.objectUri}' - cause: ${t.message}", t - final result = info.failed(t.message) + log.warn "== Blob cache failed for object '${blob.objectUri}' - cause: ${t.message}", t + final result = blob.failed(t.message) // update the blob status - blobStore.storeBlob(info.id(), result, blobConfig.failureDuration) + blobStore.storeBlob(blob.id(), result, blobConfig.failureDuration) } } @@ -266,4 +265,67 @@ class BlobCacheServiceImpl implements BlobCacheService { } } + + // ============ handles transfer job events ============ + + @Override + Duration jobMaxDuration(JobId job) { + return blobConfig.transferTimeout + } + + @Override + void onJobCompletion(JobId job, JobState state) { + final blob = blobStore.getBlob(job.id) + if( !blob ) { + log.error "== Blob cache entry unknown for job=$job [1]" + return + } + if( blob.done() ) { + log.warn "== Blob cache entry already marked as completed for job=$job [1] - entry=$blob; state=$state" + return + } + // use a short time-to-live for failed downloads + // this is needed to allow re-try caching of failure transfers + final ttl = state.succeeded() + ? blobConfig.statusDuration + : blobConfig.failureDuration + // update the blob status + final result = state.succeeded() + ? blob.completed(state.exitCode, state.stdout) + : blob.failed(state.stdout) + blobStore.storeBlob(blob.id(), result, ttl) + log.debug "== Blob cache completed for object '${blob.objectUri}'; id=${blob.objectUri}; status=${result.exitStatus}; duration=${result.duration()}" + } + + @Override + void onJobException(JobId job, Throwable error) { + final blob = blobStore.getBlob(job.id) + if( !blob ) { + log.error "== Blob cache entry unknown for job=$job [2]" + return + } + if( blob.done() ) { + log.warn "== Blob cache entry already marked as completed for job=$job [2] - entry=$blob; error=${error.message}" + return + } + final result = blob.failed("Unexpected error caching blob '${blob.locationUri}' - job name '${job.schedulerId}'") + log.error("== Blob cache exception for object '${blob.objectUri}'; job name=${job.schedulerId}; cause=${error.message}", error) + blobStore.storeBlob(job.id, result, blobConfig.failureDuration) + } + + @Override + void onJobTimeout(JobId job) { + final blob = blobStore.getBlob(job.id) + if( !blob ) { + log.error "== Blob cache entry unknown for job=$job [3]" + return + } + if( blob.done() ) { + log.warn "== Blob cache entry already marked as completed for job=$job [3] - entry=$blob; duration=${blob.duration()}" + return + } + final result = blob.failed("Blob cache transfer timed out ${blob.objectUri}") + log.warn "== Blob cache completed for object '${blob.objectUri}'; job name=${job.schedulerId}; duration=${result.duration()}" + blobStore.storeBlob(blob.id(), result, blobConfig.failureDuration) + } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy index 06d87e147..0f97ebdc1 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategy.groovy @@ -18,21 +18,19 @@ package io.seqera.wave.service.blob.impl - import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.transfer.Transfer -import io.seqera.wave.service.blob.transfer.TransferStrategy +import io.seqera.wave.service.job.JobId +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.job.JobStrategy import jakarta.inject.Inject import jakarta.inject.Singleton - /** - * Implements {@link TransferStrategy} that runs s5cmd using a docker + * Implements {@link JobStrategy} that runs s5cmd using a docker * container. Meant for development purposes * * @author Paolo Di Tommaso @@ -42,15 +40,15 @@ import jakarta.inject.Singleton @Singleton @Requires(missingProperty = 'wave.build.k8s') @Requires(property = 'wave.blobCache.enabled', value = 'true') -class DockerTransferStrategy implements TransferStrategy { +class DockerTransferStrategy implements JobStrategy { @Inject private BlobCacheConfig blobConfig @Override - void transfer(BlobCacheInfo info, List command) { + void launchJob(String jobName, List command) { // create a unique name for the container - createProcess(command, info.jobName, blobConfig.transferTimeout.toSeconds()) + createProcess(command, jobName, blobConfig.transferTimeout.toSeconds()) .start() } @@ -80,32 +78,32 @@ class DockerTransferStrategy implements TransferStrategy { } @Override - Transfer status(BlobCacheInfo blob) { - final state = getDockerContainerState(blob.jobName) - log.trace "Docker transfer status name=$blob.jobName; state=$state" + JobState status(JobId job) { + final state = getDockerContainerState(job.schedulerId) + log.trace "Docker transfer status name=$job.schedulerId; state=$state" if (state.status == 'running') { - return Transfer.running() + return JobState.running() } else if (state.status == 'exited') { - final logs = getDockerContainerLogs(blob.jobName) - return Transfer.completed(state.exitCode, logs) + final logs = getDockerContainerLogs(job.schedulerId) + return JobState.completed(state.exitCode, logs) } else if (state.status == 'created' || state.status == 'paused') { - return Transfer.pending() + return JobState.pending() } else { - final logs = getDockerContainerLogs(blob.jobName) - return Transfer.unknown(logs) + final logs = getDockerContainerLogs(job.schedulerId) + return JobState.unknown(logs) } } @Override - void cleanup(BlobCacheInfo blob) { + void cleanup(JobId jobId, Integer exitStatus) { final cli = new ArrayList() cli.add('docker') cli.add('rm') - cli.add(blob.jobName) + cli.add(jobId.schedulerId) final builder = new ProcessBuilder(cli) builder.redirectErrorStream(true) diff --git a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy index 24da1496b..3fbe68112 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategy.groovy @@ -26,17 +26,17 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.transfer.Transfer -import io.seqera.wave.service.blob.transfer.Transfer.Status -import io.seqera.wave.service.blob.transfer.TransferStrategy import io.seqera.wave.service.cleanup.CleanupStrategy +import io.seqera.wave.service.job.JobId +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.job.JobState.Status +import io.seqera.wave.service.job.JobStrategy import io.seqera.wave.service.k8s.K8sService import io.seqera.wave.service.k8s.K8sService.JobStatus import jakarta.inject.Inject import jakarta.inject.Named /** - * Implements {@link TransferStrategy} that runs s5cmd using a + * Implements {@link JobStrategy} that runs s5cmd using a * Kubernetes job * * @author Paolo Di Tommaso @@ -45,7 +45,7 @@ import jakarta.inject.Named @CompileStatic @Requires(property = 'wave.build.k8s') @Requires(property = 'wave.blobCache.enabled', value = 'true') -class KubeTransferStrategy implements TransferStrategy { +class KubeTransferStrategy implements JobStrategy { @Inject private BlobCacheConfig blobConfig @@ -61,29 +61,29 @@ class KubeTransferStrategy implements TransferStrategy { private ExecutorService executor @Override - void transfer(BlobCacheInfo info, List command) { + void launchJob(String jobName, List command) { // run the transfer job - k8sService.transferJob(info.jobName, blobConfig.s5Image, command, blobConfig) + k8sService.launchJob(jobName, blobConfig.s5Image, command, blobConfig) } @Override - void cleanup(BlobCacheInfo blob) { - if( cleanup.shouldCleanup(blob.exitStatus) ) { - CompletableFuture.supplyAsync (() -> k8sService.deleteJob(blob.jobName), executor) + void cleanup(JobId job, Integer exitStatus) { + if( cleanup.shouldCleanup(exitStatus) ) { + CompletableFuture.supplyAsync (() -> k8sService.deleteJob(job.schedulerId), executor) } } @Override - Transfer status(BlobCacheInfo info) { - final status = k8sService.getJobStatus(info.jobName) + JobState status(JobId job) { + final status = k8sService.getJobStatus(job.schedulerId) if( !status || !status.completed() ) { - return new Transfer(mapToStatus(status)) + return new JobState(mapToStatus(status)) } // Find the latest created pod among the pods associated with the job - final pod = k8sService.getLatestPodForJob(info.jobName) + final pod = k8sService.getLatestPodForJob(job.schedulerId) if( !pod ) - throw new IllegalStateException("Missing carried pod for job: ${info.jobName}") + throw new IllegalStateException("Missing carried pod for job: ${job.schedulerId}") // determine exit code and logs final exitCode = pod @@ -94,7 +94,7 @@ class KubeTransferStrategy implements TransferStrategy { ?.terminated ?.exitCode final stdout = k8sService.logsPod(pod) - return new Transfer(mapToStatus(status), exitCode, stdout) + return new JobState(mapToStatus(status), exitCode, stdout) } /** diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy b/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy deleted file mode 100644 index acae180e0..000000000 --- a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferManager.groovy +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.blob.transfer - -import java.time.Duration -import java.time.Instant -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Context -import io.micronaut.context.annotation.Requires -import io.micronaut.scheduling.TaskExecutors -import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.impl.BlobCacheStore -import io.seqera.wave.util.ExponentialAttempt -import jakarta.annotation.PostConstruct -import jakarta.inject.Inject -import jakarta.inject.Named -/** - * Implement the logic to handle Blob cache transfer (uploads) - * - * @author Paolo Di Tommaso - */ -@Slf4j -@Context -@CompileStatic -@Requires(property = 'wave.blobCache.enabled', value = 'true') -class TransferManager { - - @Inject - private TransferStrategy transferStrategy - - @Inject - private BlobCacheStore blobStore - - @Inject - private BlobCacheConfig blobConfig - - @Inject - private TransferQueue queue - - @Inject - @Named(TaskExecutors.IO) - private ExecutorService executor - - private final ExponentialAttempt attempt = new ExponentialAttempt() - - @PostConstruct - private init() { - CompletableFuture.supplyAsync(()->run(), executor) - } - - void run() { - log.info "+ Starting Blob cache transfer manager" - while( !Thread.currentThread().isInterrupted() ) { - try { - final transferId = queue.poll(blobConfig.statusDelay) - - if( transferId ) { - handle(transferId) - attempt.reset() - } - } - catch (InterruptedException e) { - log.debug "Interrupting transfer manager watcher thread" - break - } - catch (Throwable e) { - final d0 = attempt.delay() - log.error("Transfer manager unexpected error (await: ${d0}) - cause: ${e.message}", e) - sleep(d0.toMillis()) - } - } - } - - /** - * Handles the blob transfer operation i.e. check and update the current upload status - * - * @param blobId the blob cache id i.e. {@link BlobCacheInfo#id()} - */ - protected void handle(String blobId) { - try { - final blob = blobStore.get(blobId) - if( !blob ) { - log.error "Unknown blob transfer with id: $blobId" - return - } - try { - handle0(blob) - } - catch (Throwable t) { - log.error("Unexpected error caching blob '${blob.objectUri}' - job name '${blob.jobName}", t) - blobStore.put(blobId, blob.failed("Unexpected error caching blob '${blob.locationUri}' - job name '${blob.jobName}'")) - } - } - catch (InterruptedException e) { - // re-queue the transfer to not lose it - queue.offer(blobId) - // re-throw the exception - throw e - } - } - - protected void handle0(BlobCacheInfo info) { - final duration = Duration.between(info.creationTime, Instant.now()) - final transfer = transferStrategy.status(info) - log.trace "Blob cache transfer name=${info.jobName}; state=${transfer}; object=${info.objectUri}" - final done = - transfer.completed() || - // considered failed when remain in unknown status too long - (transfer.status==Transfer.Status.UNKNOWN && duration>blobConfig.graceDuration) - if( done ) { - // use a short time-to-live for failed downloads - // this is needed to allow re-try caching of failure transfers - final ttl = transfer.succeeded() - ? blobConfig.statusDuration - : blobConfig.failureDuration - // update the blob status - final result = transfer.succeeded() - ? info.completed(transfer.exitCode, transfer.stdout) - : info.failed(transfer.stdout) - blobStore.storeBlob(info.id(), result, ttl) - log.debug "== Blob cache completed for object '${info.objectUri}'; id=${info.objectUri}; status=${result.exitStatus}; duration=${result.duration()}" - // finally cleanup the job - transferStrategy.cleanup(result) - return - } - // set the await timeout nearly double as the blob transfer timeout, this because the - // transfer pod can spend `timeout` time in pending status awaiting to be scheduled - // and the same `timeout` time amount carrying out the transfer (upload) operation - final max = (blobConfig.transferTimeout.toMillis() * 2.10) as long - if( duration.toMillis()>max ) { - final result = info.failed("Blob cache transfer timed out - id: ${info.objectUri}; object: ${info.objectUri}") - log.warn "== Blob cache completed for object '${info.objectUri}'; id=${info.objectUri}; duration=${result.duration()}" - blobStore.storeBlob(info.id(), result, blobConfig.failureDuration) - } - else { - log.trace "== Blob cache pending for completion $info" - // re-schedule for a new check - queue.offer(info.id()) - } - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy new file mode 100644 index 000000000..54331bcb6 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy @@ -0,0 +1,178 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.Predicate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.wave.encoder.EncodingStrategy +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.util.ExponentialAttempt +import io.seqera.wave.util.TypeHelper +/** + * Implement an abstract stream that allows that consumes messages asynchronously + * as soon as they are available. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +abstract class AbstractMessageStream implements Closeable { + + static final private AtomicInteger count = new AtomicInteger() + + final private Map> listeners = new ConcurrentHashMap<>() + + final private ExponentialAttempt attempt = new ExponentialAttempt() + + final private EncodingStrategy encoder + + final private MessageStream stream + + private Thread thread + + private String name0 + + AbstractMessageStream(MessageStream target) { + final type = TypeHelper.getGenericType(this, 0) + this.encoder = new MoshiEncodeStrategy(type) {} + this.stream = target + this.name0 = name() + '-thread-' + count.getAndIncrement() + this.thread = new Thread(()-> processMessages(), name0) + this.thread.setDaemon(true) + this.thread.start() + } + + /** + * @return The name of the message queue implementation + */ + protected abstract String name() + + /** + * @return + * The time interval to await before trying to read again the stream + * when no more entries are available. + */ + protected abstract Duration pollInterval() + + /** + * Add a message to the stream with the specified identifier + * + * @param streamId + * The stream unique ID + * @param message + * The message object to be added to the stream + */ + void offer(String streamId, M message) { + final msg = encoder.encode(message) + stream.offer(streamId, msg) + } + + /** + * Define the consumer {@link Predicate} to be applied when a message is available + * for reading in the stream. + * + * @param streamId + * The stream unique ID + * @param consumer + * The {@link Predicate} to be invoked when a stream message is consumed (read from) the stream. + */ + void consume(String streamId, Predicate consumer) { + final value = listeners.put(streamId, consumer) + if( value!=null ) + throw new IllegalStateException("Only one consumer can be defined for each stream - offending streamId=$streamId; consumer=$consumer") + } + + /** + * Deserialize the message as string into the target message object and process it by applying + * the given consumer {@link Predicate}. + * + * @param msg + * The message serialised as a string value + * @param consumer + * The consumer {@link Predicate} that will handle the message as a object + * @param count + * An {@link AtomicInteger} counter incremented by one when this method is invoked, + * irrespective if the consumer is successful or not. + * @return + * The result of the consumer {@link Predicate} operation. + */ + protected boolean processMessage(String msg, Predicate consumer, AtomicInteger count) { + count.incrementAndGet() + final decoded = encoder.decode(msg) + log.trace "Message streaming - receiving message=$msg; decoded=$decoded" + return consumer.test(decoded) + } + + /** + * Process the messages as they are available from the underlying stream + */ + protected void processMessages() { + log.trace "Message stream - starting listener thread" + while( !thread.interrupted() ) { + try { + final count=new AtomicInteger() + for( Map.Entry> entry : listeners.entrySet() ) { + final streamId = entry.key + final consumer = entry.value + stream.consume(streamId, (String msg)-> processMessage(msg, consumer, count)) + } + // reset the attempt count because no error has been thrown + attempt.reset() + // if no message was sent, sleep for a while before retrying + if( count==0 ) { + sleep(pollInterval().toMillis()) + } + } + catch (InterruptedException e) { + log.debug "Message streaming interrupt exception - cause: ${e.message}" + Thread.currentThread().interrupt() + break + } + catch (Throwable e) { + final d0 = attempt.delay() + log.error("Unexpected error on message stream ${name0} (await: ${d0}) - cause: ${e.message}", e) + sleep(d0.toMillis()) + } + } + log.trace "Message stream - exiting listener thread" + } + + /** + * Shutdown orderly the stream + */ + @Override + void close() { + if( !thread ) + return + // interrupt the thread + thread.interrupt() + // wait for the termination + try { + thread.join(1_000) + } + catch (Exception e) { + log.debug "Unexpected error while terminating ${name0} - cause: ${e.message}" + } + } +} diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy new file mode 100644 index 000000000..279183991 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/data/stream/MessageStream.groovy @@ -0,0 +1,52 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + + +import java.util.function.Predicate +/** + * Define the contract for a generic message stream + * able to add message and consume them asynchronously. + * + * @author Paolo Di Tommaso + */ +interface MessageStream { + + /** + * Offer a message to the stream. + * + * @param message + * The message that should be offered to the queue + */ + void offer(String streamId, M message) + + /** + * Consume a message from the stream and invoke the specified predicate + * + * @param streamId + * The target stream ID + * @param consumer + * The {@link Predicate} instance to be invoked to consume the message + * @return + * {code true} when the message has been processed successfully, + * otherwise {@code false} when the message needs to be further processed + */ + boolean consume(String streamId, Predicate consumer) + +} diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy new file mode 100644 index 000000000..01d971080 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/LocalMessageStream.groovy @@ -0,0 +1,76 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream.impl + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.LinkedBlockingQueue +import java.util.function.Predicate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires +import io.seqera.wave.service.data.stream.MessageStream +import jakarta.inject.Singleton +/** + * Implement a {@link MessageStream} using a Java {@link java.util.concurrent.BlockingQueue}. + * This is only meant for developing purpose. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Requires(notEnv = 'redis') +@Singleton +@CompileStatic +class LocalMessageStream implements MessageStream { + + private ConcurrentHashMap> delegate = new ConcurrentHashMap<>() + + @Override + void offer(String streamId, String message) { + delegate + .computeIfAbsent(streamId, (it)-> new LinkedBlockingQueue<>()) + .offer(message) + } + + @Override + boolean consume(String streamId, Predicate consumer) { + final message = delegate + .computeIfAbsent(streamId, (it)-> new LinkedBlockingQueue<>()) + .poll() + if( message==null ) { + return false + } + + def result = false + try { + result = consumer.test(message) + } + catch (Throwable e) { + result = false + throw e + } + finally { + if( !result ) { + offer(streamId, message) + } + return result + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy new file mode 100644 index 000000000..e9402e8ec --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy @@ -0,0 +1,154 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream.impl + +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Predicate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires +import io.micronaut.context.annotation.Value +import io.seqera.wave.service.data.stream.MessageStream +import io.seqera.wave.util.LongRndKey +import jakarta.annotation.PostConstruct +import jakarta.inject.Inject +import jakarta.inject.Singleton +import redis.clients.jedis.Jedis +import redis.clients.jedis.JedisPool +import redis.clients.jedis.StreamEntryID +import redis.clients.jedis.exceptions.JedisDataException +import redis.clients.jedis.params.XAutoClaimParams +import redis.clients.jedis.params.XReadGroupParams +import redis.clients.jedis.resps.StreamEntry +/** + * Implement a distributed {@link MessageStream} backed by a Redis stream. + * This implementation allows multiple concurrent consumers and guarantee consistency + * across replicas restart. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Requires(env = 'redis') +@Singleton +@CompileStatic +class RedisMessageStream implements MessageStream { + + private static final StreamEntryID STREAM_ENTRY_ZERO = new StreamEntryID("0-0") + + private static final String CONSUMER_GROUP_NAME = "wave-message-stream" + + private static final String DATA_FIELD = 'data' + + private final ConcurrentHashMap group0 = new ConcurrentHashMap<>() + + @Inject + private JedisPool pool + + @Value('${wave.message-stream.claim-timeout:10s}') + private Duration claimTimeout + + private String consumerName + + @PostConstruct + private void init() { + consumerName = "consumer-${LongRndKey.rndLong()}" + log.info "Creating Redis message stream - consumer=${consumerName}; claim-timeout=${claimTimeout}" + } + + protected void createGroup(Jedis jedis, String stream, String group) { + // use a concurrent hash map to create it only the very first time + group0.computeIfAbsent("$stream/$group".toString(),(it)-> createGroup0(jedis,stream,group)) + } + + protected boolean createGroup0(Jedis jedis, String stream, String group) { + try { + jedis.xgroupCreate(stream, group, STREAM_ENTRY_ZERO, true) + return true + } + catch (JedisDataException e) { + if (e.message.contains("BUSYGROUP")) { + // The group already exists, so we can safely ignore this exception + log.debug "Redis message stream - consume group=$group already exists" + return true + } + throw e + } + } + + @Override + void offer(String streamId, String message) { + try (Jedis jedis = pool.getResource()) { + jedis.xadd(streamId, StreamEntryID.NEW_ENTRY, Map.of(DATA_FIELD, message)) + } + } + + @Override + boolean consume(String streamId, Predicate consumer) { + try (Jedis jedis = pool.getResource()) { + createGroup(jedis, streamId, CONSUMER_GROUP_NAME) + final entry = claimMessage(jedis,streamId) ?: readMessage(jedis, streamId) + if( entry && consumer.test(entry.getFields().get(DATA_FIELD)) ) { + // Acknowledge the job after processing + jedis.xack(streamId, CONSUMER_GROUP_NAME, entry.getID()) + return true + } + else + return false + } + } + + protected StreamEntry readMessage(Jedis jedis, String streamId) { + // Create parameters for reading with a group + final params = new XReadGroupParams() + // Read one message at a time + .count(1) + + // Read new messages from the stream using the correct xreadGroup signature + List>> messages = jedis.xreadGroup( + CONSUMER_GROUP_NAME, + consumerName, + params, + Map.of(streamId, StreamEntryID.UNRECEIVED_ENTRY) ) + + final entry = messages?.first()?.value?.first() + log.trace "Redis stream id=$streamId; read entry=$entry" + return entry + } + + protected StreamEntry claimMessage(Jedis jedis, String streamId) { + // Attempt to claim any pending messages that are idle for more than the threshold + final params = new XAutoClaimParams() + // claim one entry at time + .count(1) + final messages = jedis.xautoclaim( + streamId, + CONSUMER_GROUP_NAME, + consumerName, + claimTimeout.toMillis(), + STREAM_ENTRY_ZERO, + params + ) + final entry = messages?.getValue()?[0] + log.trace "Redis stream id=$streamId; claimed entry=$entry" + return entry + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy b/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy similarity index 64% rename from src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy rename to src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy index 183a0cbaf..504ba9506 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobConfig.groovy @@ -16,33 +16,27 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob.transfer +package io.seqera.wave.service.job import java.time.Duration -import io.seqera.wave.service.data.queue.MessageQueue -import jakarta.inject.Inject +import groovy.transform.ToString +import io.micronaut.context.annotation.Value import jakarta.inject.Singleton /** - * Implements a simple persistent FIFO queue + * Model Job manager configuration settings * * @author Paolo Di Tommaso */ +@ToString(includeNames = true, includePackage = false) @Singleton -class TransferQueue { +class JobConfig { - final private static String QUEUE_NAME = 'transfer-queue/v1' + @Value('${wave.job-manager.grace-interval:20s}') + Duration graceInterval - @Inject - private MessageQueue transferQueue - - void offer(String transferId) { - transferQueue.offer(QUEUE_NAME, transferId) - } - - String poll(Duration timeout) { - transferQueue.poll(QUEUE_NAME, timeout) - } + @Value('${wave.job-manager.poll-interval:200ms}') + Duration pollInterval } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy b/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy new file mode 100644 index 000000000..44b8ae519 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobDispatcher.groovy @@ -0,0 +1,85 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.time.Duration + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.ApplicationContext +import io.micronaut.inject.qualifiers.Qualifiers +import jakarta.annotation.PostConstruct +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * Concrete implementation of {@link JobHandler} that dispatcher event invocations + * to the target implementation based on the job {@link JobId#type} + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class JobDispatcher implements JobHandler { + + @Inject + private ApplicationContext context + + private Map dispatch = new HashMap<>() + + @PostConstruct + void init() { + // implementation should be added here + add(JobId.Type.Transfer, dispatch, false) + } + + protected void add(JobId.Type type, Map map, boolean required) { + final handler = context.findBean(JobHandler.class, Qualifiers.byName(type.toString())) + if( handler.isPresent() ) { + log.debug "Adding job handler for type: $type; handler=$handler" + map.put(type, handler.get()) + } + else if( required ) { + throw new IllegalStateException("Unable to find Job handler for type: $type") + } + else { + log.debug "Disabled job handler for type: $type" + } + } + + @Override + Duration jobMaxDuration(JobId job) { + return dispatch.get(job.type).jobMaxDuration(job) + } + + @Override + void onJobCompletion(JobId job, JobState state) { + dispatch.get(job.type).onJobCompletion(job, state) + } + + @Override + void onJobException(JobId job, Throwable error) { + dispatch.get(job.type).onJobException(job, error) + } + + @Override + void onJobTimeout(JobId job) { + dispatch.get(job.type).onJobTimeout(job) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy similarity index 66% rename from src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy rename to src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy index d16412a2e..33898e01d 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferTimeoutException.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobHandler.groovy @@ -16,22 +16,23 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob.transfer - -import groovy.transform.CompileStatic -import io.seqera.wave.exception.WaveException +package io.seqera.wave.service.job +import java.time.Duration /** - * Exception fired when the time to blob download takes too long - * - * @author: Paolo Di Tommaso + * Define events and properties for jobs managed via {@link JobManager} * + * @author Paolo Di Tommaso */ -@CompileStatic -class TransferTimeoutException extends WaveException{ +interface JobHandler { + + Duration jobMaxDuration(JobId job) + + void onJobCompletion(JobId job, JobState state) + + void onJobException(JobId job, Throwable error) + + void onJobTimeout(JobId job) - TransferTimeoutException(String message) { - super(message) - } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobId.groovy b/src/main/groovy/io/seqera/wave/service/job/JobId.groovy new file mode 100644 index 000000000..b0f40c3c5 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobId.groovy @@ -0,0 +1,70 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.time.Instant + +import com.google.common.hash.Hashing +import groovy.transform.Canonical + +/** + * Model a unique job to be managed + * + * @author Paolo Di Tommaso + */ +@Canonical +class JobId { + enum Type { Transfer, Build, Scan } + + final Type type + final String id + final Instant creationTime + final String schedulerId + + JobId( Type type, String id, Instant creationTime ) { + this.type = type + this.id = id + this.creationTime = creationTime + schedulerId = generate(type, id, creationTime) + } + + static JobId transfer(String id) { + new JobId(Type.Transfer, id, Instant.now()) + } + + static JobId build(String id) { + new JobId(Type.Build, id, Instant.now()) + } + + static JobId scan(String id) { + new JobId(Type.Scan, id, Instant.now()) + } + + static private String generate(Type type, String id, Instant creationTime) { + final prefix = type.toString().toLowerCase() + return prefix + '-' + Hashing + .sipHash24() + .newHasher() + .putUnencodedChars(id) + .putUnencodedChars(type.toString()) + .putUnencodedChars(creationTime.toString()) + .hash() + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy new file mode 100644 index 000000000..2970e6aa5 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy @@ -0,0 +1,100 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.time.Duration +import java.time.Instant + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Context +import io.micronaut.context.annotation.Requires +import jakarta.annotation.PostConstruct +import jakarta.inject.Inject +/** + * Implement the logic to handle Blob cache transfer (uploads) + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Context +@CompileStatic +@Requires(property = 'wave.blobCache.enabled', value = 'true') +class JobManager { + + @Inject + private JobStrategy jobStrategy + + @Inject + private JobQueue queue + + @Inject + private JobDispatcher dispatcher + + @Inject + private JobConfig config + + @PostConstruct + void init() { + log.info "Creating job manager - config=$config" + queue.consume((job)-> processJob(job)) + } + + protected boolean processJob(JobId jobId) { + try { + return processJob0(jobId) + } + catch (Throwable err) { + // in the case of an expected exception report the error condition by using `onJobException` + dispatcher.onJobException(jobId, err) + // finally return `true` to signal the job should not be processed anymore + return true + } + } + + protected boolean processJob0(JobId jobId) { + final duration = Duration.between(jobId.creationTime, Instant.now()) + final state = jobStrategy.status(jobId) + log.trace "Job status id=${jobId.schedulerId}; state=${state}" + final done = + state.completed() || + // considered failed when remain in unknown status too long + (state.status==JobState.Status.UNKNOWN && duration>config.graceInterval) + if( done ) { + // publish the completion event + dispatcher.onJobCompletion(jobId, state) + // cleanup the job + jobStrategy.cleanup(jobId, state.exitCode) + return true + } + // set the await timeout nearly double as the blob transfer timeout, this because the + // transfer pod can spend `timeout` time in pending status awaiting to be scheduled + // and the same `timeout` time amount carrying out the transfer (upload) operation + final max = (dispatcher.jobMaxDuration(jobId).toMillis() * 2.10) as long + if( duration.toMillis()>max ) { + dispatcher.onJobTimeout(jobId) + return true + } + else { + log.trace "== Job pending for completion $jobId" + return false + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy new file mode 100644 index 000000000..9eba8ecc4 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy @@ -0,0 +1,69 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import java.time.Duration +import java.util.function.Predicate + +import groovy.transform.CompileStatic +import io.seqera.wave.service.data.stream.AbstractMessageStream +import io.seqera.wave.service.data.stream.MessageStream +import jakarta.annotation.PreDestroy +import jakarta.inject.Singleton +/** + * Implements a simple persistent FIFO queue + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class JobQueue extends AbstractMessageStream { + + final private static String STREAM_NAME = 'jobs-queue/v1' + + private volatile JobConfig config + + JobQueue(MessageStream target, JobConfig config) { + super(target) + this.config = config + } + + @Override + protected String name() { + return 'jobs-queue' + } + + @Override + protected Duration pollInterval() { + return config.pollInterval + } + + final void offer(JobId job) { + super.offer(STREAM_NAME, job) + } + + final void consume(Predicate consumer) { + super.consume(STREAM_NAME, consumer) + } + + @PreDestroy + void destroy() { + this.close() + } +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy new file mode 100644 index 000000000..9c1c6f2e1 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy @@ -0,0 +1,32 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import io.seqera.wave.service.blob.BlobCacheInfo + +/** + * Define the contract for submitting and monitoring jobs + * + * @author Paolo Di Tommaso + */ +interface JobService { + + JobId launchTransfer(BlobCacheInfo blob, List command) + +} diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy new file mode 100644 index 000000000..902e17c7b --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -0,0 +1,52 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import groovy.transform.CompileStatic +import io.seqera.wave.service.blob.BlobCacheInfo +import jakarta.inject.Inject +import jakarta.inject.Singleton + +/** + * Implement a service for job creation and execution + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class JobServiceImpl implements JobService { + + @Inject + private JobStrategy jobStrategy + + @Inject + private JobQueue jobQueue + + @Override + JobId launchTransfer(BlobCacheInfo blob, List command) { + // create the ID for the job transfer + final job = JobId.transfer(blob.id()) + // submit the job execution + jobStrategy.launchJob(job.schedulerId, command) + // signal the transfer to be started + jobQueue.offer(job) + return job + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy b/src/main/groovy/io/seqera/wave/service/job/JobState.groovy similarity index 69% rename from src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy rename to src/main/groovy/io/seqera/wave/service/job/JobState.groovy index 600aca519..bd24bbf2e 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/transfer/Transfer.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobState.groovy @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob.transfer +package io.seqera.wave.service.job import groovy.transform.Canonical import groovy.transform.ToString @@ -29,7 +29,7 @@ import groovy.transform.ToString @ToString(includePackage = false, includeNames = true) @Canonical -class Transfer { +class JobState { enum Status { PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN } @@ -45,28 +45,28 @@ class Transfer { status==Status.SUCCEEDED && exitCode==0 } - static Transfer pending() { - return new Transfer(Status.PENDING) + static JobState pending() { + return new JobState(Status.PENDING) } - static Transfer running() { - return new Transfer(Status.RUNNING) + static JobState running() { + return new JobState(Status.RUNNING) } - static Transfer failed(Integer exit, String logs) { - return new Transfer(Status.FAILED, exit, logs) + static JobState failed(Integer exit, String logs) { + return new JobState(Status.FAILED, exit, logs) } - static Transfer succeeded(String logs) { - return new Transfer(Status.SUCCEEDED, 0, logs) + static JobState succeeded(String logs) { + return new JobState(Status.SUCCEEDED, 0, logs) } - static Transfer completed(Integer exit, String logs) { + static JobState completed(Integer exit, String logs) { final st = exit==0 ? Status.SUCCEEDED : Status.FAILED - return new Transfer(st, exit, logs) + return new JobState(st, exit, logs) } - static Transfer unknown(String logs) { - return new Transfer(Status.UNKNOWN,null,logs) + static JobState unknown(String logs) { + return new JobState(Status.UNKNOWN,null,logs) } } diff --git a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy b/src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy similarity index 77% rename from src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy rename to src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy index 08452b64f..618c54fd4 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/transfer/TransferStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobStrategy.groovy @@ -16,21 +16,18 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob.transfer - - -import io.seqera.wave.service.blob.BlobCacheInfo +package io.seqera.wave.service.job /** * Defines the contract to transfer a layer blob into a remote object storage * * @author Paolo Di Tommaso */ -interface TransferStrategy { +interface JobStrategy { - void transfer(BlobCacheInfo blob, List command) + void launchJob(String jobName, List command) - Transfer status(BlobCacheInfo blob) + JobState status(JobId jobId) - void cleanup(BlobCacheInfo blob) + void cleanup(JobId jobId, Integer exitStatus) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index d7005ab7b..7a292dad1 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -58,7 +58,7 @@ interface K8sService { void deleteJob(String name) - V1Job transferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) + V1Job launchJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) V1PodList waitJob(V1Job job, Long timeout) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index fa886a81f..7872cc097 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -585,7 +585,7 @@ class K8sServiceImpl implements K8sService { * The {@link V1Job} description the submitted job */ @Override - V1Job transferJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) { + V1Job launchJob(String name, String containerImage, List args, BlobCacheConfig blobConfig) { final spec = createTransferJobSpec(name, containerImage, args, blobConfig) return k8sClient diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index e60f1558d..aee70b476 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -37,8 +37,9 @@ datasources: --- logger: levels: - io.seqera.wave.service.data: 'TRACE' - io.seqera.wave.service.pairing: 'TRACE' - io.seqera.wave.tower.client.connector: 'TRACE' + io.seqera.wave.service.data: TRACE + io.seqera.wave.service.pairing: TRACE + io.seqera.wave.tower.client.connector: TRACE +# io.seqera.wave.service.data.stream: TRACE # io.seqera.wave.tower.auth: 'TRACE' ... diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index c8950916a..afe0fb149 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -32,6 +32,7 @@ import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult +import io.seqera.wave.service.job.JobId import io.seqera.wave.service.pairing.socket.msg.PairingHeartbeat import io.seqera.wave.service.pairing.socket.msg.PairingResponse import io.seqera.wave.service.pairing.socket.msg.ProxyHttpRequest @@ -370,4 +371,20 @@ class MoshiEncodingStrategyTest extends Specification { copy == auth } + + def 'should encode and decode job request' () { + given: + def encoder = new MoshiEncodeStrategy() { } + and: + def job = new JobId(JobId.Type.Transfer, '123-abc', Instant.now()) + + when: + def json = encoder.encode(job) + and: + def copy = encoder.decode(json) + then: + copy.getClass() == JobId + and: + copy == job + } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy index fb54a105b..845f15fec 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobCacheInfoTest.groovy @@ -249,7 +249,6 @@ class BlobCacheInfoTest extends Specification { null, null, null, - null, CREATE, COMPLETE ) diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy similarity index 98% rename from src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy rename to src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy index b361fcde2..b245061d5 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/DockerTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/DockerJobStrategyTest.groovy @@ -26,7 +26,7 @@ import io.seqera.wave.configuration.BlobCacheConfig * * @author Paolo Di Tommaso */ -class DockerTransferStrategyTest extends Specification { +class DockerJobStrategyTest extends Specification { def 'should create transfer cli' () { given: diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy similarity index 65% rename from src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy rename to src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy index 54fff307a..56ec8795d 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeJobStrategyTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.blob.impl + import spock.lang.Specification import spock.lang.Unroll @@ -35,7 +36,8 @@ import io.kubernetes.client.openapi.models.V1PodStatus import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.transfer.Transfer +import io.seqera.wave.service.job.JobId +import io.seqera.wave.service.job.JobState import io.seqera.wave.service.k8s.K8sService.JobStatus import io.seqera.wave.service.cleanup.CleanupStrategy import io.seqera.wave.service.k8s.K8sService @@ -43,7 +45,7 @@ import io.seqera.wave.service.k8s.K8sService * * @author Munish Chouhan */ -class KubeTransferStrategyTest extends Specification { +class KubeJobStrategyTest extends Specification { K8sService k8sService = Mock(K8sService) BlobCacheConfig blobConfig = new BlobCacheConfig(s5Image: 's5cmd', transferTimeout: Duration.ofSeconds(10), retryAttempts: 3) @@ -55,79 +57,80 @@ class KubeTransferStrategyTest extends Specification { def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) def command = ["transfer", "blob"] final jobName = "job-123" - def podName = "$jobName-abc" + def podName = "$jobName-abc".toString() def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) pod.status = new V1PodStatus(phase: "Succeeded") def podList = new V1PodList(items: [pod]) - k8sService.transferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) + k8sService.launchJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) k8sService.waitJob(_, _) >> podList k8sService.getPod(_) >> pod k8sService.waitPodCompletion(_, _) >> 0 k8sService.logsPod(_) >> "transfer successful" when: - strategy.transfer(info, command) + strategy.launchJob(podName, command) then: - 1 * k8sService.transferJob(info.jobName, blobConfig.s5Image, command, blobConfig) + 1 * k8sService.launchJob(podName, blobConfig.s5Image, command, blobConfig) } def 'status should return correct status when job is not completed'() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) - k8sService.getJobStatus(info.jobName) >> K8sService.JobStatus.Running + def job = JobId.transfer('foo') + and: + k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Running when: - def result = strategy.status(info) - + def result = strategy.status(job) then: - result.status == Transfer.Status.RUNNING + result.status == JobState.Status.RUNNING } - void 'status should return correct transfer status when pods are created'() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + def job = JobId.transfer('foo') + and: def status = new V1PodStatus(phase: "Succeeded", containerStatuses: [new V1ContainerStatus( state: new V1ContainerState(terminated: new V1ContainerStateTerminated(exitCode: 0)))]) - def pod = new V1Pod(metadata: [name: "pod-123"], status: status) - k8sService.getJobStatus(_) >> K8sService.JobStatus.Succeeded - k8sService.logsPod(_) >> "transfer successful" - k8sService.getLatestPodForJob(info.jobName) >> pod + def pod = new V1Pod(metadata: [name: job.schedulerId], status: status) + and: + k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Succeeded + k8sService.getLatestPodForJob(job.schedulerId) >> pod + k8sService.logsPod(pod) >> "transfer successful" when: - def result = strategy.status(info) - + def result = strategy.status(job) then: - result.status == Transfer.Status.SUCCEEDED + result.status == JobState.Status.SUCCEEDED result.exitCode == 0 result.stdout == "transfer successful" } def 'status should return failed transfer when no pods are created'() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) + def job = JobId.transfer('foo') + and: def status = new V1PodStatus(phase: "Failed") - def pod = new V1Pod(metadata: [name: "pod-123"], status: status) - k8sService.getJobStatus(info.jobName) >> K8sService.JobStatus.Failed - k8sService.getLatestPodForJob(info.jobName) >> pod + def pod = new V1Pod(metadata: [name: job.schedulerId], status: status) + and: + k8sService.getLatestPodForJob(job.schedulerId) >> pod + k8sService.getJobStatus(job.schedulerId) >> K8sService.JobStatus.Failed when: - def result = strategy.status(info) - + def result = strategy.status(job) then: - result.status == Transfer.Status.FAILED + result.status == JobState.Status.FAILED } def 'status should handle null job status'() { given: - def info = BlobCacheInfo.create("https://test.com/blobs", "https://test.com/bucket/blobs", null, null) - k8sService.getJobStatus(info.id()) >> null - + def job = JobId.transfer('foo') + and: + k8sService.getJobStatus(job.schedulerId) >> null + when: - def result = strategy.status(info) - + def result = strategy.status(job) then: - result.status == Transfer.Status.UNKNOWN + result.status == JobState.Status.UNKNOWN } @Unroll @@ -137,10 +140,10 @@ class KubeTransferStrategyTest extends Specification { where: JOB_STATUS | TRANSFER_STATUS - JobStatus.Pending | Transfer.Status.PENDING - JobStatus.Running | Transfer.Status.RUNNING - JobStatus.Succeeded | Transfer.Status.SUCCEEDED - JobStatus.Failed | Transfer.Status.FAILED - null | Transfer.Status.UNKNOWN + JobStatus.Pending | JobState.Status.PENDING + JobStatus.Running | JobState.Status.RUNNING + JobStatus.Succeeded | JobState.Status.SUCCEEDED + JobStatus.Failed | JobState.Status.FAILED + null | JobState.Status.UNKNOWN } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy deleted file mode 100644 index 71681a23e..000000000 --- a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferManagerTest.groovy +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.blob.transfer - -import spock.lang.Specification - -import java.time.Duration - -import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.service.blob.BlobCacheInfo -import io.seqera.wave.service.blob.impl.BlobCacheStore -/** - * - * @author Munish Chouhan - */ -class TransferManagerTest extends Specification { - - def "handle should process valid transferId"() { - given: - def blobStore = Mock(BlobCacheStore) - def queue = Mock(TransferQueue) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = Mock(BlobCacheConfig) - def manager = new TransferManager(blobStore: blobStore, queue: queue, transferStrategy: transferStrategy, blobConfig: blobConfig) - - and: - def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) - - when: - manager.handle(blob.id()) - - then: - 1 * blobStore.get(blob.id()) >> blob - 1 * transferStrategy.status(blob) >> Transfer.completed(0, 'logs') - } - - def "handle should log error for unknown transferId"() { - given: - def transferId = 'unknown' - def blobStore = Mock(BlobCacheStore) - def queue = Mock(TransferQueue) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = Mock(BlobCacheConfig) - def manager = new TransferManager(blobStore: blobStore, queue: queue, transferStrategy: transferStrategy, blobConfig: blobConfig) - - when: - manager.handle(transferId) - - then: - 1 * blobStore.get(transferId) >> null - 0 * manager.handle0(_) - } - - def "handle0 should complete transfer when status is completed"() { - given: - def blobStore = Mock(BlobCacheStore) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = Mock(BlobCacheConfig) - def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) - def blob = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) - def transfer = Transfer.succeeded('logs') - blobConfig.statusDuration >> Duration.ofMinutes(5) - transferStrategy.status(blob) >> transfer - - when: - manager.handle0(blob) - - then: - 1 * blobStore.storeBlob(blob.id(), _, blobConfig.statusDuration) - 1 * transferStrategy.cleanup(_) - } - - def "handle0 should fail transfer when status is unknown and duration exceeds grace period"() { - given: - def blobStore = Mock(BlobCacheStore) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1), graceDuration: Duration.ofSeconds(1)) - def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) - def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) - def transfer = Transfer.unknown('logs') - transferStrategy.status(info) >> transfer - - when: - sleep 1_000 //sleep for grace period - manager.handle0(info) - - then: - 1 * blobStore.storeBlob(info.id(), _, blobConfig.failureDuration) - 1 * transferStrategy.cleanup(_) - } - - def "handle0 should requeue transfer when duration is within limits"() { - given: - def blobStore = Mock(BlobCacheStore) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1)) - def queue = Mock(TransferQueue) - def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig, queue: queue) - def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) - def transfer = Transfer.running() - transferStrategy.status(info) >> transfer - - when: - manager.handle0(info) - - then: - 1 * queue.offer(info.id()) - } - - def "handle0 should timeout transfer when duration exceeds max limit"() { - given: - def blobStore = Mock(BlobCacheStore) - def transferStrategy = Mock(TransferStrategy) - def blobConfig = new BlobCacheConfig(transferTimeout: Duration.ofSeconds(1)) - def manager = new TransferManager(blobStore: blobStore, transferStrategy: transferStrategy, blobConfig: blobConfig) - def info = BlobCacheInfo.create('http://foo.com', 's3://foo/com', [:], [:]) - def transfer = Transfer.running() - transferStrategy.status(info) >> transfer - - when: - sleep 1_100 * 2 //await timeout - manager.handle0(info) - - then: - 1 * blobStore.storeBlob(info.id(), _, blobConfig.failureDuration) - } -} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy new file mode 100644 index 000000000..8531af73f --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamLocalTest.groovy @@ -0,0 +1,59 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import spock.lang.Specification + +import java.util.concurrent.ArrayBlockingQueue + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.data.stream.impl.LocalMessageStream +import io.seqera.wave.util.LongRndKey +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class AbstractMessageStreamLocalTest extends Specification { + + @Inject + LocalMessageStream target + + def 'should offer and consume some messages' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + and: + def stream = new TestStream(target) + def queue = new ArrayBlockingQueue(10) + + when: + stream.offer(id1, new TestMessage('one','two')) + stream.offer(id1, new TestMessage('alpha','omega')) + then: + stream.consume(id1, { it-> queue.add(it) }) + and: + queue.take()==new TestMessage('one','two') + queue.take()==new TestMessage('alpha','omega') + + cleanup: + stream.close() + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy new file mode 100644 index 000000000..185754655 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/AbstractMessageStreamRedisTest.groovy @@ -0,0 +1,68 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import spock.lang.Shared +import spock.lang.Specification + +import java.util.concurrent.ArrayBlockingQueue + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.service.data.stream.impl.RedisMessageStream +import io.seqera.wave.test.RedisTestContainer +import io.seqera.wave.util.LongRndKey + +/** + * + * @author Paolo Di Tommaso + */ +class AbstractMessageStreamRedisTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext context + + def setup() { + context = ApplicationContext.run([ + REDIS_HOST: redisHostName, + REDIS_PORT: redisPort + ], 'test', 'redis') + } + + def 'should offer and consume some messages' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + and: + def target = context.getBean(RedisMessageStream) + def stream = new TestStream(target) + def queue = new ArrayBlockingQueue(10) + + when: + stream.offer(id1, new TestMessage('one','two')) + stream.offer(id1, new TestMessage('alpha','omega')) + then: + stream.consume(id1, { it-> queue.add(it) }) + and: + queue.take()==new TestMessage('one','two') + queue.take()==new TestMessage('alpha','omega') + + cleanup: + stream.close() + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy new file mode 100644 index 000000000..eb2a25969 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/LocalMessageStreamTest.groovy @@ -0,0 +1,84 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.data.stream.impl.LocalMessageStream +import io.seqera.wave.util.LongRndKey +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class LocalMessageStreamTest extends Specification { + + def 'should offer and consume a value' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + def id2 = "stream-${LongRndKey.rndHex()}" + and: + def stream = new LocalMessageStream() + when: + stream.offer(id1, 'one') + and: + stream.offer(id2, 'alpha') + stream.offer(id2, 'delta') + stream.offer(id2, 'gamma') + + then: + stream.consume(id1, { it-> it=='one'}) + and: + stream.consume(id2, { it-> it=='alpha'}) + stream.consume(id2, { it-> it=='delta'}) + stream.consume(id2, { it-> it=='gamma'}) + and: + !stream.consume(id2, { it-> assert false /* <-- this should not be invoked */ }) + } + + def 'should offer and consume a value with a failure' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalMessageStream() + when: + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') + + then: + stream.consume(id1, { it-> it=='alpha'}) + and: + !stream.consume(id1, { it-> throw new RuntimeException("Oops")}) + and: + // next message is 'gamma' as expected + stream.consume(id1, { it-> it=='gamma'}) + and: + // now the errored message is available again + stream.consume(id1, { it-> it=='delta'}) + and: + !stream.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + + when: + stream.offer(id1, 'something') + then: + stream.consume(id1, { it-> it=='something'}) + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy new file mode 100644 index 000000000..b680b0929 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/RedisMessageStreamTest.groovy @@ -0,0 +1,109 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import spock.lang.Shared +import spock.lang.Specification + +import io.micronaut.context.ApplicationContext +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.data.stream.impl.RedisMessageStream +import io.seqera.wave.test.RedisTestContainer +import io.seqera.wave.util.LongRndKey + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class RedisMessageStreamTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext context + + def setup() { + context = ApplicationContext.run([ + 'wave.message-stream.claim-timeout': '1s', + REDIS_HOST: redisHostName, + REDIS_PORT: redisPort, + ], 'test', 'redis') + } + + def 'should offer and consume a value' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + def id2 = "stream-${LongRndKey.rndHex()}" + and: + def stream = context.getBean(RedisMessageStream) + when: + stream.offer(id1, 'one') + and: + stream.offer(id2, 'alpha') + stream.offer(id2, 'delta') + stream.offer(id2, 'gamma') + + then: + stream.consume(id1, { it-> it=='one'}) + and: + stream.consume(id2, { it-> it=='alpha'}) + stream.consume(id2, { it-> it=='delta'}) + stream.consume(id2, { it-> it=='gamma'}) + and: + !stream.consume(id2, { it-> assert false /* <-- this should not be invoked */ }) + } + + def 'should offer and consume a value with a failure' () { + given: + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = context.getBean(RedisMessageStream) + when: + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') + + then: + stream.consume(id1, { it-> it=='alpha'}) + and: + try { + stream.consume(id1, { it-> throw new RuntimeException("Oops")}) + } + catch (RuntimeException e) { + assert e.message == 'Oops' + } + and: + // next message is 'gamma' as expected + stream.consume(id1, { it-> it=='gamma'}) + and: + // still nothing + !stream.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + and: + // wait 2 seconds (claim timeout is 1 sec) + sleep 2_000 + // now the errored message is available + stream.consume(id1, { it-> it=='delta'}) + and: + !stream.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + + when: + stream.offer(id1, 'something') + then: + stream.consume(id1, { it-> it=='something'}) + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/TestMessage.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/TestMessage.groovy new file mode 100644 index 000000000..e98fec98d --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/TestMessage.groovy @@ -0,0 +1,31 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import groovy.transform.Canonical + +/** + * + * @author Paolo Di Tommaso + */ +@Canonical +class TestMessage { + String x + String y +} diff --git a/src/test/groovy/io/seqera/wave/service/data/stream/TestStream.groovy b/src/test/groovy/io/seqera/wave/service/data/stream/TestStream.groovy new file mode 100644 index 000000000..297ddc24d --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/data/stream/TestStream.groovy @@ -0,0 +1,42 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.data.stream + +import java.time.Duration + +/** + * + * @author Paolo Di Tommaso + */ +class TestStream extends AbstractMessageStream { + + TestStream(MessageStream target) { + super(target) + } + + @Override + protected String name() { + return 'test-stream' + } + + @Override + protected Duration pollInterval() { + return Duration.ofSeconds(1) + } +} diff --git a/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy new file mode 100644 index 000000000..e0dc8a3fd --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/JobIdTest.groovy @@ -0,0 +1,72 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import spock.lang.Specification + +import java.time.Instant + +/** + * + * @author Paolo Di Tommaso + */ +class JobIdTest extends Specification { + + def 'should create job id' () { + given: + def ts = Instant.parse('2024-08-18T19:23:33.650722Z') + + when: + def job = new JobId(JobId.Type.Transfer, 'foo', ts) + then: + job.id == 'foo' + job.schedulerId == 'transfer-8e5e0d3b81e48cac' + job.creationTime == ts + job.type == JobId.Type.Transfer + + } + + def 'should create transfer job' () { + when: + def job = JobId.transfer('abc-123') + then: + job.id == 'abc-123' + job.schedulerId =~ /transfer-.+/ + job.type == JobId.Type.Transfer + } + + def 'should create build job' () { + when: + def job = JobId.build('abc-123') + then: + job.id == 'abc-123' + job.schedulerId =~ /build-.+/ + job.type == JobId.Type.Build + } + + def 'should create scan job' () { + when: + def job = JobId.scan('abc-123') + then: + job.id == 'abc-123' + job.schedulerId =~ /scan-.+/ + job.type == JobId.Type.Scan + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy new file mode 100644 index 000000000..2ad51b6c8 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy @@ -0,0 +1,128 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.job + +import spock.lang.Specification + +import java.time.Duration +/** + * + * @author Munish Chouhan + */ +class JobManagerTest extends Specification { + + def "handle should process valid transferId"() { + given: + def jobStrategy = Mock(JobStrategy) + def jobDispatcher = Mock(JobDispatcher) + def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + and: + def job = JobId.transfer('foo') + + when: + def done = manager.processJob(job) + then: + 1 * jobStrategy.status(job) >> JobState.completed(0, 'My job logs') + and: + 1 * jobDispatcher.onJobCompletion(job, _) >> { JobId id, JobState state -> + assert state.exitCode == 0 + assert state.stdout == 'My job logs' + } + and: + 1 * jobStrategy.cleanup(job,0) + and: + done + } + + def "handle should log error for unknown transferId"() { + given: + def jobStrategy = Mock(JobStrategy) + def jobDispatcher = Mock(JobDispatcher) + def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + and: + def job = JobId.transfer('unknown') + + when: + def done = manager.processJob(job) + then: + 1 * jobStrategy.status(job) >> null + and: + 1 * jobDispatcher.onJobException(job,_) >> null + and: + done + } + + def "handle0 should fail transfer when status is unknown and duration exceeds grace period"() { + given: + def jobStrategy = Mock(JobStrategy) + def jobDispatcher = Mock(JobDispatcher) + def config = new JobConfig(graceInterval: Duration.ofMillis(500)) + def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher, config:config) + and: + def job = JobId.transfer('foo') + + when: + sleep 1_000 //sleep longer than grace period + def done = manager.processJob(job) + + then: + 1 * jobStrategy.status(job) >> JobState.unknown('logs') + 1 * jobDispatcher.onJobCompletion(job, _) + 1 * jobStrategy.cleanup(job, _) + and: + done + } + + def "should requeue transfer when duration is within limits"() { + given: + def jobStrategy = Mock(JobStrategy) + def jobDispatcher = Mock(JobDispatcher) + def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + and: + def job = JobId.transfer('foo') + + when: + def done = manager.processJob(job) + then: + 1 * jobStrategy.status(job) >> JobState.running() + 1 * jobDispatcher.jobMaxDuration(job) >> Duration.ofSeconds(10) + and: + !done + } + + def "handle0 should timeout transfer when duration exceeds max limit"() { + given: + def jobStrategy = Mock(JobStrategy) + def jobDispatcher = Mock(JobDispatcher) + def manager = new JobManager(jobStrategy: jobStrategy, dispatcher: jobDispatcher) + and: + def job = JobId.transfer('foo') + + when: + sleep 1_000 //await timeout + def done = manager.processJob(job) + then: + 1 * jobStrategy.status(job) >> JobState.running() + 1 * jobDispatcher.jobMaxDuration(job) >> Duration.ofMillis(100) + and: + 1 * jobDispatcher.onJobTimeout(job) + and: + done + } +} diff --git a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy b/src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy similarity index 65% rename from src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy rename to src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy index f8b3e615f..76a29cf80 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/transfer/TransferTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/TransferTest.groovy @@ -16,7 +16,8 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.blob.transfer +package io.seqera.wave.service.job + import spock.lang.Specification import spock.lang.Unroll @@ -30,31 +31,31 @@ class TransferTest extends Specification { @Unroll def 'should validate completed status' () { expect: - new Transfer(STATUS).completed() == EXPECTED + new JobState(STATUS).completed() == EXPECTED where: STATUS | EXPECTED - Transfer.Status.PENDING | false - Transfer.Status.RUNNING | false - Transfer.Status.UNKNOWN | false + JobState.Status.PENDING | false + JobState.Status.RUNNING | false + JobState.Status.UNKNOWN | false and: - Transfer.Status.SUCCEEDED | true - Transfer.Status.FAILED | true + JobState.Status.SUCCEEDED | true + JobState.Status.FAILED | true } @Unroll def 'should validate succeeded status' () { expect: - new Transfer(STATUS, EXIT).succeeded() == EXPECTED + new JobState(STATUS, EXIT).succeeded() == EXPECTED where: STATUS | EXIT | EXPECTED - Transfer.Status.PENDING | null | false - Transfer.Status.RUNNING | null | false - Transfer.Status.UNKNOWN | null | false - Transfer.Status.FAILED | null | false - Transfer.Status.SUCCEEDED | 1 | false - Transfer.Status.SUCCEEDED | 0 | true + JobState.Status.PENDING | null | false + JobState.Status.RUNNING | null | false + JobState.Status.UNKNOWN | null | false + JobState.Status.FAILED | null | false + JobState.Status.SUCCEEDED | 1 | false + JobState.Status.SUCCEEDED | 0 | true } From 3d940e889927b360076101114d0a037d99b12d3b Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Thu, 29 Aug 2024 08:51:29 +0200 Subject: [PATCH 49/50] Add build in progress status in build page (#607) Signed-off-by: munishchouhan Co-authored-by: Paolo Di Tommaso --- .../wave/controller/ViewController.groovy | 2 + .../resources/io/seqera/wave/build-view.hbs | 89 ++++++++++++------ .../wave/controller/ViewControllerTest.groovy | 90 +++++++++++++++++++ 3 files changed, 154 insertions(+), 27 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 60ba05537..1a4d9ba95 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -75,6 +75,8 @@ class ViewController { final binding = new HashMap(20) binding.build_id = result.buildId binding.build_success = result.succeeded() + binding.build_failed = result.exitStatus && result.exitStatus != 0 + binding.build_in_progress = result.exitStatus == null binding.build_exit_status = result.exitStatus binding.build_user = (result.userName ?: '-') + " (ip: ${result.requestIp})" binding.build_time = formatTimestamp(result.startTime, result.offsetId) ?: '-' diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 5cb33235f..acceacc55 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -1,6 +1,9 @@ + {{#if build_in_progress}} + + {{/if}} @@ -16,17 +19,50 @@ {{#if build_success}} -
-

- Container build completed successfully! -

-
- {{else}} -
-

- Container build failed -

-
+
+

+ Container build completed successfully! +

+
+ {{else build_failed}} +
+

+ Container build failed +

+
+ {{else build_in_progress}} + {{! build is not completed, show a spinning icon }} + +
+

+
Container build in progress +

+
{{/if}}

Summary

@@ -75,36 +111,35 @@ - {{#if scan_url}} - - - - - {{/if}} + {{#if scan_url}} + + + + + {{/if}}
Build ID{{build_id}}{{build_id}}
CachedExit status {{build_exit_status}}
SecurityImage scan report
SecurityImage scan report

Container file

{{build_containerfile}}
{{#if build_condafile}} -

Conda file

- 
{{build_condafile}}
+

Conda file

+ 
{{build_condafile}}
{{/if}} {{#if build_spackfile}} -

Spack file

- 
{{build_spackfile}}
+

Spack file

+ 
{{build_spackfile}}
{{/if}} {{#if build_log_data}} -

Build logs

- 
{{build_log_data}}
- {{#if build_log_truncated}} - Click here to download the complete build log - {{/if}} +

Build logs

+ 
{{build_log_data}}
+ {{#if build_log_truncated}} + Click here to download the complete build log + {{/if}} {{/if}} -
diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index 772d61151..e4447cbc1 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -102,6 +102,9 @@ class ViewControllerTest extends Specification { binding.build_log_data == 'log content' binding.build_log_truncated == false binding.build_log_url == 'http://foo.com/v1alpha1/builds/12345/logs' + binding.build_success == true + binding.build_in_progress == false + binding.build_failed == false } def 'should render a build page' () { @@ -228,4 +231,91 @@ class ViewControllerTest extends Specification { then: response.body().contains(token) } + + def 'should render in progress build page' () { + given: + def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService) + and: + def record = new WaveBuildRecord( + buildId: '12345', + dockerFile: 'FROM foo', + condaFile: 'conda::foo', + spackFile: 'some-spack-recipe', + targetImage: 'docker.io/some:image', + userName: 'paolo', + userEmail: 'paolo@seqera.io', + userId: 100, + requestIp: '10.20.30.40', + startTime: Instant.now(), + offsetId: '+02:00', + duration: Duration.ofMinutes(1), + platform: 'linux/amd64' ) + when: + def binding = controller.renderBuildView(record) + then: + 1 * buildLogService.fetchLogString('12345') >> new BuildLogService.BuildLog('log content', false) + and: + binding.build_id == '12345' + binding.build_containerfile == 'FROM foo' + binding.build_condafile == 'conda::foo' + binding.build_image == 'docker.io/some:image' + binding.build_user == 'paolo (ip: 10.20.30.40)' + binding.build_platform == 'linux/amd64' + binding.build_exit_status == null + binding.build_platform == 'linux/amd64' + binding.build_containerfile == 'FROM foo' + binding.build_condafile == 'conda::foo' + binding.build_spackfile == 'some-spack-recipe' + binding.build_format == 'Docker' + binding.build_log_data == 'log content' + binding.build_log_truncated == false + binding.build_log_url == 'http://foo.com/v1alpha1/builds/12345/logs' + binding.build_success == false + binding.build_in_progress == true + binding.build_failed == false + } + + def 'should render in progress build page' () { + given: + def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService) + and: + def record = new WaveBuildRecord( + buildId: '12345', + dockerFile: 'FROM foo', + condaFile: 'conda::foo', + spackFile: 'some-spack-recipe', + targetImage: 'docker.io/some:image', + userName: 'paolo', + userEmail: 'paolo@seqera.io', + userId: 100, + requestIp: '10.20.30.40', + startTime: Instant.now(), + offsetId: '+02:00', + duration: Duration.ofMinutes(1), + exitStatus: 1, + platform: 'linux/amd64' ) + when: + def binding = controller.renderBuildView(record) + then: + 1 * buildLogService.fetchLogString('12345') >> new BuildLogService.BuildLog('log content', false) + and: + binding.build_id == '12345' + binding.build_containerfile == 'FROM foo' + binding.build_condafile == 'conda::foo' + binding.build_image == 'docker.io/some:image' + binding.build_user == 'paolo (ip: 10.20.30.40)' + binding.build_platform == 'linux/amd64' + binding.build_exit_status == 1 + binding.build_platform == 'linux/amd64' + binding.build_containerfile == 'FROM foo' + binding.build_condafile == 'conda::foo' + binding.build_spackfile == 'some-spack-recipe' + binding.build_format == 'Docker' + binding.build_log_data == 'log content' + binding.build_log_truncated == false + binding.build_log_url == 'http://foo.com/v1alpha1/builds/12345/logs' + binding.build_success == false + binding.build_in_progress == false + binding.build_failed == true + } } From 6b588f4c63b89d64a33fa4ec05ee492cc0f32e9c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 29 Aug 2024 08:52:16 +0200 Subject: [PATCH 50/50] Update dev default logs [ci skip] Signed-off-by: Paolo Di Tommaso --- src/main/resources/application-dev.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index aee70b476..b47e4812b 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -37,9 +37,8 @@ datasources: --- logger: levels: - io.seqera.wave.service.data: TRACE + io.seqera.wave.service.data: DEBUG io.seqera.wave.service.pairing: TRACE io.seqera.wave.tower.client.connector: TRACE -# io.seqera.wave.service.data.stream: TRACE # io.seqera.wave.tower.auth: 'TRACE' ...