Merge branch 'master' into 475-use-k8s-job-instead-of-pod-for-blob-ca…

…che-transfer

README.md

@@ -14,7 +14,7 @@ images.
 * Augment container images i.e. dynamically add one or more container layers to existing images;
 * Build container images on-demand for a given container file (aka Dockerfile);
 * Build container images on-demand based on one or more Conda packages;
-* Build container images on-demand based on one or more Spack packages;
+* Build container images on-demand based on one or more Spack packages, Spack support will be removed in future releases;
 * Build container images for a specified target platform (currently linux/amd64 and linux/arm64);
 * Push and cache built containers to a user-provided container repository;
 * Build Singularity native containers both using a Singularity spec file, Conda package(s) and Spack package(s);

VERSION

@@ -1 +1 @@
-1.8.3
+1.9.0

build.gradle

@@ -35,7 +35,7 @@ dependencies {
     implementation("jakarta.persistence:jakarta.persistence-api:3.0.0")
     api 'io.seqera:lib-mail:1.0.0'
     api 'io.seqera:wave-api:0.10.0'
-    api 'io.seqera:wave-utils:0.12.0'
+    api 'io.seqera:wave-utils:0.13.1'
 
     implementation("io.micronaut:micronaut-http-client")
     implementation("io.micronaut:micronaut-jackson-databind")

changelog.txt

@@ -1,4 +1,26 @@
 # Wave changelog
+1.9.1 - 11 Jul 2024
+- Prevent hard error when launch credentials cannot be accessed [a318a855]
+
+1.9.0 - 11 Jul 2024
+- Add Typespec API definitions (#537) [32f7dd16]
+- Add cache record-stats (#534) [229926e2]
+- Add http 429 error to auth service retry condition [8282a492]
+- Check and delete corrupted blobs cache uploads (#533) [b0c775a3]
+- Deprecate the support for Spack and remove the support for it in the codebase (#550) [85a05196]
+- Enable ECR authentication via AWS compute env credentials (#303) [ec895222]
+- Fix Blob cache config for local dev [0d5413bf]
+- Fix client cache deadlock (#547) [cc6012ff]
+- Fix multiple s3clients in wave (#554) [45689500]
+- Minor change in mail notification [0aba6997]
+- Refactored metrics service (#549) [5e0d32ac]
+- Remove buildlogs aws config to not initialize AwsS3Operations (#558) [d50e5b7a]
+- Update metrics response (#536) [b6b36a97]
+- Bump buildkit 0.14.0 (#528) [c54172d1]
+- Bump buildkit 0.14.1 (#548) [37afb782]
+- Bump trivy 0.53.0 [f27ff527]
+- Bump version 1.8.3 [55b473e1]
+
 1.8.3 - 29 Jun 2024
 - Fix client cache deadlock (#547) [cc6012ff]
 - Fix failing test [263b44d3]

configuration.md

@@ -109,6 +109,8 @@ Below are the standard format for known registries, but you can change registry
 
 ### Spack configuration for wave build process
 
+**Note**: Spack support will be removed in future releases.
+
 Spack configuration consists of the path of its secret file, the mount path for the secret file in the spack container, and the optional S3 bucket name for the spack binary cache.
 
 **Note**: these configuration are mandatory to support Spack in a wave installation.

docs/api.mdx

@@ -67,7 +67,7 @@ This API endpoint is deprecated in current versions of Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | Spack recipe file encoded as base64 string.                                                                                                        |
+| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                        |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |
@@ -168,7 +168,7 @@ The endpoint returns the name of the container request made available by Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | Spack recipe file encoded as base64 string.                                                                                                        |
+| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                           |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |
@@ -178,9 +178,9 @@ The endpoint returns the name of the container request made available by Wave.
 | `towerEndpoint`                     | Seqera Platform service endpoint from where container registry credentials are retrieved (optional). Default `https://api.cloud.seqera.io`.                       |
 | `towerAccessToken`                  | Access token of the user account granting access to the Seqera Platform service specified via `towerEndpoint` (optional).                      |
 | `towerWorkspaceId`                  | ID of the Seqera Platform workspace from where the container registry credentials are retrieved (optional). When omitted the personal workspace is used. |
-| `packages`                          | This object specifies Conda or Spack packages environment information.                                                                                         |
+| `packages`                          | This object specifies Conda packages environment information.                                                                                         |
 | `environment`                       | The package environment file encoded as a base64 string.                                                                                                       |
-| `type`                              | This represents the type of package builder. Use `SPACK` or `CONDA`.                                                                                   |
+| `type`                              | This represents the type of package builder. Use `CONDA`.                                                                                   |
 | `entries`                           | List of the packages names.                                                                                                                                    |
 | `channels`                          | List of Conda channels, which will be used to download packages.                                                                                               |
 | `mambaImage`                        | Name of the docker image used to build Conda containers.                                                                                              |

docs/cli/build-spack.mdx

@@ -2,6 +2,8 @@
 title: Build a container from Spack packages
 ---
 
+**Note**: Spack support will be removed in future releases.
+
 The Wave CLI supports building a container from a list of [Spack] packages.
 
 :::caution

docs/index.mdx

@@ -10,6 +10,10 @@ It allows for the on-demand assembly, augmentation, and deployment of containeri
 
 The Wave container service itself is not a container registry. All containers builds are stored in a Seqera-hosted image registry for a limited time or frozen to a user-specified container registry.
 
+:::note
+Wave is available for free as part of Seqera Cloud. As it is open source software, no support is provided by Seqera. For a supported, self-hosted, solution please [contact us](https://seqera.io/contact-us/).
+:::
+
 ## Features
 
 ### Private container registries
@@ -26,7 +30,7 @@ Wave offers a flexible approach to container image management. It allows you to
 
 #### An example of Wave augmentation
 
-Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your docker client and the registry. When you request an augmented image, Wave intercepts the process.
+Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your Docker client and the registry. When you request an augmented image, Wave intercepts the process.
 
 1. Base image layers download: The Docker client downloads the standard Ubuntu layers from the registry.
 2. Custom layer injection: Wave injects your custom layer, denoted by "ω", which could represent application code, libraries, configurations etc.
@@ -39,7 +43,7 @@ Imagine you have a base Ubuntu image in a container registry. Wave acts as a pro
 1. Streamlined workflows: Wave simplifies your workflow by eliminating the need to manually build and manage custom images.
 2. Flexibility: You can easily modify the custom layer for different use cases, allowing for greater adaptability.
 
-### Conda based containers
+### Conda-based containers
 
 Package management systems such as Conda and Bioconda simplify the installation of scientific software. However, there’s considerable friction when it comes to using those tools to deploy pipelines in cloud environments.
 Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just declare the Conda packages in your Nextflow pipeline and Wave will assemble the required container.

src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy

@@ -39,7 +39,7 @@ import io.seqera.wave.util.StringUtils
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
-import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS
+import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
 /**
  * Implement Docker authentication & login service
  *
@@ -116,7 +116,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> response.statusCode() in HTTP_SERVER_ERRORS)
+                .retryIf( (response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS)
                 .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event}"))
         // make the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))
@@ -203,7 +203,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_SERVER_ERRORS )
+                .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_RETRYABLE_ERRORS )
                 .onRetry((event) -> log.warn("Unable to connect '$login' - event: $event"))
         // submit http request
         final response = retryable.apply(()-> httpClient.send(req, HttpResponse.BodyHandlers.ofString()))

src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProvider.groovy

@@ -40,6 +40,15 @@ interface RegistryCredentialsProvider {
      */
     RegistryCredentials getDefaultCredentials(String registry)
 
+    /**
+     * Provides the default credentials for the specified container
+     *
+     * @param container
+     *      A container name e.g. docker.io/library/ubuntu.
+     * @return
+     *      A {@link RegistryCredentials} object holding the credentials for the specified container or {@code null}
+     *      if not credentials can be found
+     */
     RegistryCredentials getDefaultCredentials(ContainerPath container)
 
     /**
@@ -56,4 +65,21 @@ interface RegistryCredentialsProvider {
      */
     RegistryCredentials getUserCredentials(ContainerPath container, PlatformId identity)
 
+    /**
+     * Provides the credentials for the specified container. When the platform identity is provider
+     * this is equivalent to #getUserCredentials.
+     *
+     * @param container
+     *      A container name e.g. docker.io/library/ubuntu.
+     * @param identity
+     *      The platform identity of the user submitting the request
+     * @return
+     *      A {@link RegistryCredentials} object holding the credentials for the specified container or {@code null}
+     *      if not credentials can be found
+     */
+    default RegistryCredentials getCredentials(ContainerPath container, PlatformId identity) {
+        return !identity
+                ? getDefaultCredentials(container)
+                : getUserCredentials(container, identity)
+    }
 }

src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy

@@ -34,7 +34,7 @@ import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
 import static io.seqera.wave.WaveDefault.DOCKER_REGISTRY_1
-import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS
+import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
 /**
  * Lookup service for container registry. The role of this component
  * is to registry the retrieve the registry authentication realm
@@ -73,7 +73,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf((response) -> response.statusCode() in HTTP_SERVER_ERRORS)
+                .retryIf((response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS )
                 .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event"))
         // submit the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))

src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy

@@ -90,6 +90,9 @@ class BlobCacheConfig {
     @Value('${wave.blobCache.backoffLimit:3}')
     Integer backoffLimit
 
+    @Value('${wave.blobCache.k8s.pod.delete.timeout:20s}')
+    Duration podDeleteTimeout
+
     Map<String,String> getEnvironment() {
         final result = new HashMap<String,String>(10)
         if( storageRegion ) {

src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy

@@ -35,6 +35,7 @@ import jakarta.inject.Singleton
 @EqualsAndHashCode
 @Singleton
 @CompileStatic
+@Deprecated
 class SpackConfig {
 
     /**

src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy

@@ -107,9 +107,7 @@ class RegistryProxyService {
     }
 
     protected RegistryCredentials getCredentials(RoutePath route) {
-        final result = !route.identity
-                ? credentialsProvider.getDefaultCredentials(route)
-                : credentialsProvider.getUserCredentials(route, route.identity)
+        final result = credentialsProvider.getCredentials(route, route.identity)
         log.debug "Credentials for route path=${route.targetContainer}; identity=${route.identity} => ${result}"
         return result
     }

src/main/groovy/io/seqera/wave/service/ContainerRegistryKeys.groovy

@@ -20,26 +20,53 @@ package io.seqera.wave.service
 
 import groovy.json.JsonSlurper
 import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
 import io.seqera.wave.util.StringUtils
 
 /**
  * Model the container registry keys as stored in Tower
  * 
  * @author Paolo Di Tommaso <[email protected]>
  */
+@Slf4j
 @CompileStatic
 class ContainerRegistryKeys {
+    /**
+     * The registry user name
+     */
     String userName
+
+    /**
+     * The registry secret
+     */
     String password
+
+    /**
+     * The registry target host - NOTE: this can be null when the keys where obtained by AWS credentials record
+     */
     String registry
 
     static ContainerRegistryKeys fromJson(String json) {
         final root = (Map) new JsonSlurper().parseText(json)
-        return new ContainerRegistryKeys(userName: root.userName, password: root.password, registry: root.registry)
+        // parse container registry credentials
+        if( root.discriminator == 'container-reg' ) {
+            return new ContainerRegistryKeys(userName: root.userName, password: root.password, registry: root.registry)
+        }
+        // Map AWS keys to registry username and password
+        if( root.discriminator == 'aws' ) {
+            // AWS keys can have also the `assumeRoleArn`, not clear yet how to handle it
+            // https://github.com/seqeralabs/platform/blob/64d12c6f3f399f26422a746c0d97cea6d8ddebbb/tower-enterprise/src/main/groovy/io/seqera/tower/domain/aws/AwsSecurityKeys.groovy#L39-L39
+            if( root.assumeRoleArn ) {
+                log.warn "The use of AWS assumeRoleArn for container credentials is not supported - accessKey=${root.accessKey}; assumeRoleArn=${root.assumeRoleArn}"
+                return null
+            }
+            return new ContainerRegistryKeys(userName: root.accessKey, password: root.secretKey)
+        }
+        throw new IllegalArgumentException("Unsupported credentials key discriminator type: ${root.discriminator}")
     }
 
     @Override
     String toString() {
-        return "ContainerRegistryKeys[registry=$registry; userName=$userName; password=${StringUtils.redact(password)})]"
+        return "ContainerRegistryKeys[registry=${registry}; userName=${userName}; password=${StringUtils.redact(password)})]"
     }
 }