Merge branch 'master' into remove-keyid

README.md

@@ -14,10 +14,9 @@ images.
 * Augment container images i.e. dynamically add one or more container layers to existing images;
 * Build container images on-demand for a given container file (aka Dockerfile);
 * Build container images on-demand based on one or more Conda packages;
-* Build container images on-demand based on one or more Spack packages, Spack support will be removed in future releases;
 * Build container images for a specified target platform (currently linux/amd64 and linux/arm64);
 * Push and cache built containers to a user-provided container repository;
-* Build Singularity native containers both using a Singularity spec file, Conda package(s) and Spack package(s);
+* Build Singularity native containers both using a Singularity spec file, Conda package(s);
 * Push Singularity native container images to OCI-compliant registries;
 
 

VERSION

@@ -1 +1 @@
-1.11.0
+1.12.2

changelog.txt

@@ -1,4 +1,53 @@
 # Wave changelog
+1.12.2 - 18 Sep 2024 
+- Fix Remove entries permanently from stream once consumed [adfad9d6]
+- Refactor container build service [1a858c12]
+- Remove unused code [268c76ea]
+
+1.12.1 - 17 Sep 2024 
+- Fix stream check for new messages [16a7e256]
+
+1.12.0 - 17 Sep 2024 
+- Add Job manager (#605) [00daf919]
+  - Add support for build and scan operation via Job manager (#620) [0e5e5ca4]
+  - Do not retry on build failure (#632) [e6568d1e]
+  - Fix Blob cache failure duration (#643) [ebf65adc]
+  - Fix K8s job status detection (#630) [d5b45d8d] [7a9046ed] [e26811dd]
+  - Fix Retry policy delay multipler (#629) [80037565]
+  - Improve blob cache info (#644) [8b96173a]
+  - Improve blob cache logging [e4c75671]
+  - Improve blob cache reliability (#596) [dfb64bad]
+  - Improve build & scan logging [b086f3d8]
+  - Improve job dispatcher (#645) [fee3db9d]
+  - Remove unneeded timeout logic (#633) [5eabf285]
+  - Deferred resources cleanup (#636) [c6b3e9b8]
+  - Change k8s Job deletion to foreground pods propagation (#595) [b5baea03]
+  - Run Docker process in background (#647) [1fcb4c94]
+- Add build in progress status in build page (#607) [3d940e88]
+- Add container image name to container scan view  (#635) [7858b95f]
+- Add entropy to cron services (#640) [a6d1d884]
+- Add link to build Id in container request view [57129960]
+- Add rate limiter to container request [a05c1094]
+- Add trusted builds timeout (#600) [63b58088]
+- Add /v1alpha2/container/{containerId} endpoint (#609) [6c05498c]
+- Add /v1alpha2/container/{token} in typespec (#618) [5cbd67a8]
+- Fix failing type checks [bd704bea]
+- Fix too many requests error code (#610) [ec43fa0d]
+- Increase blob cache timeout to 10m and decrese status to 1h [cf4b7588]
+- Improve container view page (#615) [d9b8cab8]
+- Improve registry auth error handling (#628) [c9185730]
+- Increase cache-tower-client to 1min (#641) [df32b305]
+- Message queue name refactoring [861d0580]
+- Simplify tests (#627) [cf53cba2]
+- Update dev default logs [6b588f4c]
+- Update nextflow.mdx (#612) [fe9b4273]
+- Update scan model (#637) [94d37637]
+- Use public repo for s5cmd (#639) [c16c0959]
+- Bump Trivy 0.55 (#638) [b69d34c4]
+
+1.11.1 - 5 Sep 2024 
+- Add rate limiter to container request [a3c63525]
+
 1.11.0 - 23 Aug 2024
 - added /v1alpha2/container/{containerId} (#609) [5221b5a0]
 - Improve contaiener view page (#615) [9e15b455]

configuration.md

@@ -107,20 +107,6 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.build.force-compression`**: determines whether to force the compression for each cache layers produced by the build process. The default is `false`, enabling compression for more efficient storage. *Optional*.
 
-### Spack configuration for wave build process
-
-**Note**: Spack support will be removed in future releases.
-
-Spack configuration consists of the path of its secret file, the mount path for the secret file in the spack container, and the optional S3 bucket name for the spack binary cache.
-
-**Note**: these configuration are mandatory to support Spack in a wave installation.
-
-- **`wave.build.spack.secretKeyFile`**: the path to the file containing the PGP private key used to [sign Spack packages built by Wave](https://spack.readthedocs.io/en/latest/binary_caches.html#build-cache-signing). For example, `/efs/wave/spack/key`. *Mandatory*.
-
-- **`wave.build.spack.secretMountPath`**: sets the mount path inside the Spack Docker image for the PGP private key specified by `wave.build.spack.secretKeyFile`. For instance `/var/seqera/spack/key`. Indicating where the PGP private key should be mounted inside the Spack Docker image. *Mandatory*.
-
-- **`wave.build.spack.cacheBucket`**: specifies the S3 bucket for the Spack binary cache, for example, `s3://spack-binarycache`. *Optional*.
-
 ### Build process logs configuration
 
 This configuration specifies attributes for the persistence of the logs fetched from containers or k8s pods used for building requested images, which can be accessed later and also attached to the build completion email.

docs/api.mdx

@@ -52,7 +52,6 @@ This API endpoint is deprecated in current versions of Wave.
         ]
     },
     condaFile: string,
-    spackFile: string,
     containerPlatform: string,
     buildRepository: string,
     cacheRepository: string,
@@ -81,7 +80,6 @@ This API endpoint is deprecated in current versions of Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                        |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |
@@ -136,7 +134,6 @@ The endpoint returns the name of the container request made available by Wave.
         ]
     },
     condaFile: string,
-    spackFile: string,
     containerPlatform: string,
     buildRepository: string,
     cacheRepository: string,
@@ -157,10 +154,6 @@ The endpoint returns the name of the container request made available by Wave.
                                 commands: string[],
                                 basePackages: string
                               }
-                  spackOpts:{
-                                commands: string[],
-                                basePackages: string
-                            }
 
               },
     nameStrategy: string
@@ -182,7 +175,6 @@ The endpoint returns the name of the container request made available by Wave.
 | `containerConfig.layers.gzipSize`   | The size in bytes of the the provided layer tar gzip file.                                                                                         |
 | `containerFile`                     | Dockerfile used for building a new container encoded in base64 (optional). When provided, the attribute `containerImage` must be omitted.        |
 | `condaFile`                         | Conda environment file encoded as base64 string.                                                                                                   |
-| `spackFile`                         | `Deprecated` Spack recipe file encoded as base64 string. Spack support will be removed in future releases.                                                                                                           |
 | `containerPlatform`                 | Target container architecture of the built container, e.g., `linux/amd64` (optional). Currently only supporting amd64 and arm64.                     |
 | `buildRepository`                   | Container repository where container builds should be pushed, e.g., `docker.io/user/my-image` (optional).                                            |
 | `cacheRepository`                   | Container repository used to cache build layers `docker.io/user/my-cache` (optional).                                                              |

src/main/groovy/io/seqera/wave/controller/ContainerController.groovy

@@ -86,8 +86,6 @@ import static io.seqera.wave.util.ContainerHelper.makeResponseV1
 import static io.seqera.wave.util.ContainerHelper.makeResponseV2
 import static io.seqera.wave.util.ContainerHelper.makeTargetImage
 import static io.seqera.wave.util.ContainerHelper.patchPlatformEndpoint
-import static io.seqera.wave.util.ContainerHelper.spackFileFromRequest
-import static io.seqera.wave.util.SpackHelper.prependBuilderTemplate
 import static java.util.concurrent.CompletableFuture.completedFuture
 /**
  * Implement a controller to receive container token requests
@@ -237,6 +235,10 @@ class ContainerController {
             req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString())
         }
 
+        if( req.spackFile ) {
+            throw new BadRequestException("Spack packages are not supported any more")
+        }
+
         final ip = addressResolver.resolve(httpRequest)
         // check the rate limit before continuing
         if( rateLimiterService )
@@ -317,7 +319,6 @@ class ContainerController {
 
         final containerSpec = decodeBase64OrFail(req.containerFile, 'containerFile')
         final condaContent = condaFileFromRequest(req)
-        final spackContent = spackFileFromRequest(req)
         final format = req.formatSingularity() ? SINGULARITY : DOCKER
         final platform = ContainerPlatform.of(req.containerPlatform)
         final buildRepository = targetRepo( req.buildRepository ?: (req.freeze && buildConfig.defaultPublicRepository
@@ -328,7 +329,6 @@ class ContainerController {
         final containerConfig = req.freeze ? req.containerConfig : null
         final offset = DataTimeUtils.offsetId(req.timestamp)
         final scanId = scanEnabled && format==DOCKER ? LongRndKey.rndHex() : null
-        final containerFile = spackContent ? prependBuilderTemplate(containerSpec,format) : containerSpec
         // use 'imageSuffix' strategy by default for public repo images
         final nameStrategy = req.nameStrategy==null
                 && buildRepository
@@ -338,14 +338,13 @@ class ContainerController {
         checkContainerSpec(containerSpec)
 
         // create a unique digest to identify the build request
-        final containerId = makeContainerId(containerFile, condaContent, spackContent, platform, buildRepository, req.buildContext)
-        final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, spackContent, nameStrategy)
+        final containerId = makeContainerId(containerSpec, condaContent, platform, buildRepository, req.buildContext)
+        final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, nameStrategy)
         final maxDuration = buildConfig.buildMaxDuration(req)
         return new BuildRequest(
                 containerId,
-                containerFile,
+                containerSpec,
                 condaContent,
-                spackContent,
                 Path.of(buildConfig.buildWorkspace),
                 targetImage,
                 identity,

src/main/groovy/io/seqera/wave/controller/ViewController.groovy

@@ -18,22 +18,26 @@
 
 package io.seqera.wave.controller
 
+import groovy.json.JsonOutput
 import io.micronaut.core.annotation.Nullable
 
 import groovy.transform.CompileStatic
 import io.micronaut.context.annotation.Value
 import io.micronaut.http.HttpResponse
 import io.micronaut.http.annotation.Controller
 import io.micronaut.http.annotation.Get
+import io.micronaut.http.annotation.QueryValue
 import io.micronaut.scheduling.TaskExecutors
 import io.micronaut.scheduling.annotation.ExecuteOn
 import io.micronaut.views.View
 import io.seqera.wave.exception.NotFoundException
 import io.seqera.wave.service.builder.ContainerBuildService
+import io.seqera.wave.service.inspect.ContainerInspectService
 import io.seqera.wave.service.logs.BuildLogService
 import io.seqera.wave.service.persistence.PersistenceService
 import io.seqera.wave.service.persistence.WaveBuildRecord
 import io.seqera.wave.service.scan.ScanResult
+import io.seqera.wave.util.JacksonHelper
 import jakarta.inject.Inject
 import static io.seqera.wave.util.DataTimeUtils.formatDuration
 import static io.seqera.wave.util.DataTimeUtils.formatTimestamp
@@ -61,6 +65,9 @@ class ViewController {
     @Nullable
     private BuildLogService buildLogService
 
+    @Inject
+    private ContainerInspectService inspectService
+
     @View("build-view")
     @Get('/builds/{buildId}')
     HttpResponse<Map<String,String>> viewBuild(String buildId) {
@@ -86,7 +93,6 @@ class ViewController {
         binding.build_platform = result.platform
         binding.build_containerfile = result.dockerFile ?: '-'
         binding.build_condafile = result.condaFile
-        binding.build_spackfile = result.spackFile
         binding.build_digest = result.digest ?: '-'
         binding.put('server_url', serverUrl)
         binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null
@@ -164,6 +170,28 @@ class ViewController {
         return HttpResponse.<Map<String,Object>>ok(binding)
     }
 
+    @View("inspect-view")
+    @Get('/inspect')
+    HttpResponse<Map<String,Object>> viewInspect(@QueryValue String image) {
+        final binding = new HashMap(10)
+        try {
+            final spec = inspectService.containerSpec(image, null)
+            binding.imageName = spec.imageName
+            binding.reference = spec.reference
+            binding.digest = spec.digest
+            binding.registry = spec.registry
+            binding.hostName = spec.hostName
+            binding.config = JacksonHelper.toJson(spec.config)
+            binding.manifest = JacksonHelper.toJson(spec.manifest)
+        }catch (Exception e){
+            binding.error_message = e.getMessage()
+        }
+
+        // return the response
+        binding.put('server_url', serverUrl)
+        return HttpResponse.<Map<String,Object>>ok(binding)
+    }
+
     Map<String, Object> makeScanViewBinding(ScanResult result, Map<String,Object> binding=new HashMap(10)) {
         binding.should_refresh = !result.isCompleted()
         binding.scan_id = result.id
@@ -182,4 +210,5 @@ class ViewController {
 
         return binding
     }
+
 }

src/main/groovy/io/seqera/wave/service/builder/BuildCacheStore.groovy

@@ -60,7 +60,7 @@ class BuildCacheStore extends AbstractCacheStore<BuildStoreEntry> implements Bui
     protected Duration getDuration() {
         return buildConfig.statusDuration
     }
-    
+
     @Override
     BuildStoreEntry getBuild(String imageName) {
         return get(imageName)