Merge branch 'master' into add-e2e-tests

VERSION

@@ -1 +1 @@
-1.13.0-B2
+1.13.0

build.gradle

@@ -35,7 +35,7 @@ dependencies {
     compileOnly("io.micronaut:micronaut-http-validation")
     implementation("jakarta.persistence:jakarta.persistence-api:3.0.0")
     api 'io.seqera:lib-mail:1.0.0'
-    api 'io.seqera:wave-api:0.13.1'
+    api 'io.seqera:wave-api:0.13.3'
     api 'io.seqera:wave-utils:0.14.1'
     implementation("io.micronaut:micronaut-http-client")
     implementation("io.micronaut:micronaut-jackson-databind")

changelog.txt

@@ -1,4 +1,63 @@
 # Wave changelog
+1.13.0 - 14 Oct 2024
+- Add cleanup for timeout jobs [87898328]
+- Add code coverage report to build (#661) [8f3de2a0]
+- Add copy button and scroll bars in views (#674) [c6d24386]
+- Add download icon and updated copy icon (#678) [aaf2a431]
+- Add failure duration to mirror store [4afb975e]
+- Add inspect view (#619) [b890564b]
+- Add pool handling for Jackson mapper objects (#666) [1e5c64ab]
+- Add support for Container mirroring (#646) [2f5a1fe6]
+- Add support for Conda lock file (#642) [497185ef]
+- Added Typespec for container mirror and status (#677) [c6de5dc2]
+- Fix Include cached only with build [0a16554b]
+- Fix build status completion of submit exception [19eceb10]
+- Fix build status failure with missing exit status [900c1e45]
+- Fix build succeded & status response [9a1b7954]
+- Fix container scan on request [2a8e1c53]
+- Fix container view alignment (#686) [097f6628]
+- Fix entity too large when using conda file > 10k  (#658) [afeaf832]
+- Fix exception when there is no conda lockfile (#685) [893cca09]
+- Fix load last build succeed [5abce684]
+- Fix mirror typespec (#670) [5b5697f8]
+- Fix repository creds check (#665) [c77d49d1]
+- Fix retrieval of last succeded mirror result [23cb0564]
+- Fix scan status of cache record [9ee87b5f]
+- Fix singularity build mounts [3d1e00e4]
+- Fix wave build redirect [863fa105]
+- Improve build status handling [0317736b]
+- Improve handling build id in view page (#651) [69f82dc6]
+- Improve inspect for multi platform container images (#660) [884274d7]
+- Improve mirror auth check [2dca96c1]
+- Improve mirror request validation [f23d409e]
+- Increase cleanup succeeded to 30mins [69051374]
+- Move MirrorConfig to config package (#668) [2b20eeed]
+- Normalize state key prefix [BREAKING] [2e120abe]
+- Params refactoring [24b91bd6]
+- Prevent deserialization error [beb605f3]
+- Prevent scans on dry-run [a8b6bc2c]
+- Refactor state prefix [3658cfed]
+- Refactor state store components  (#657) [9ace76e7]
+- Remove BiCacheStore data structure (#653) [6c968dff]
+- Remove deprecated keyId (#616) [64e55009]
+- Remove deprecated tests [b1abf961]
+- Remove ip address (#680) [86b41760]
+- Remove logs from mail config [7a497914]
+- Remove redis from dev config [40484561]
+- Remove support for spack build (#601) [3dff0192]
+- Remove unneeded log [9a52e606]
+- Replace mirrorRegistry with buildRepository [9d31246b]
+- Revert mail template to embedded logo file [a118fee1]
+- Scan v2 (#662) [ab7e2801]
+- Simulate xclaim delay with local stream [42c3f260]
+- Update container request typespec model [1a27aee9]
+- Update scan state duration to 5 days [6270b6c5]
+- Use 3-state succeed attribute in contaienr response [d3ed2ba1]
+- Use gstring for lock file [e5364c21]
+
+1.12.4 - 26 Sep 2024
+- Fix build succeded & status response [8a760b39]
+
 1.12.3 - 22 Sep 2024
 - Fix build status completion of submit exception [3c3af360]
 - Fix singularity build mounts [3b338b29]

src/main/groovy/io/seqera/wave/service/mirror/MirrorConfig.groovy → src/main/groovy/io/seqera/wave/configuration/MirrorConfig.groovy

@@ -16,7 +16,7 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package io.seqera.wave.service.mirror
+package io.seqera.wave.configuration
 
 import java.time.Duration
 
@@ -42,6 +42,9 @@ class MirrorConfig {
     @Value('${wave.mirror.status.duration:1h}')
     Duration statusDuration
 
+    @Value('${wave.mirror.failure.duration:50s}')
+    Duration failureDuration
+
     @Value('${wave.mirror.skopeoImage:`quay.io/skopeo/stable`}')
     String skopeoImage
 

src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy

@@ -72,7 +72,7 @@ class ScanConfig {
     @Value('${wave.scan.retry-attempts:1}')
     int retryAttempts
 
-    @Value('${wave.scan.status.duration:1h}')
+    @Value('${wave.scan.status.duration:5d}')
     Duration statusDuration
 
     @Value('${wave.scan.id.duration:7d}')

src/main/groovy/io/seqera/wave/controller/ContainerController.groovy

@@ -73,6 +73,7 @@ import io.seqera.wave.service.request.ContainerStatusService
 import io.seqera.wave.service.request.TokenData
 import io.seqera.wave.service.scan.ContainerScanService
 import io.seqera.wave.service.validation.ValidationService
+import io.seqera.wave.service.validation.ValidationServiceImpl
 import io.seqera.wave.tower.PlatformId
 import io.seqera.wave.tower.User
 import io.seqera.wave.tower.auth.JwtAuth
@@ -240,14 +241,14 @@ class ContainerController {
             throw new BadRequestException("Attribute `nameStrategy` is not allowed by legacy container endpoint")
 
         // prevent the use of container file and freeze without a custom build repository
-        if( req.containerFile && req.freeze && !isCustomRepo0(req.buildRepository) && (!v2 || (v2 && !req.packages)))
+        if( req.containerFile && req.freeze && !validationService.isCustomRepo(req.buildRepository) && (!v2 || (v2 && !req.packages)))
             throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [1]")
 
         // prevent the use of container image and freeze without a custom build repository
-        if( req.containerImage && req.freeze && !isCustomRepo0(req.buildRepository) )
+        if( req.containerImage && req.freeze && !validationService.isCustomRepo(req.buildRepository) )
             throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [2]")
 
-        if( v2 && req.packages && req.freeze && !isCustomRepo0(req.buildRepository) && !buildConfig.defaultPublicRepository )
+        if( v2 && req.packages && req.freeze && !validationService.isCustomRepo(req.buildRepository) && !buildConfig.defaultPublicRepository )
             throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [3]")
 
         if( v2 && req.packages ) {
@@ -280,18 +281,6 @@ class ContainerController {
         return HttpResponse.ok(resp)
     }
 
-    protected boolean isCustomRepo0(String repo) {
-        if( !repo )
-            return false
-        if( buildConfig.defaultPublicRepository && repo.startsWith(buildConfig.defaultPublicRepository) )
-            return false
-        if( buildConfig.defaultBuildRepository && repo.startsWith(buildConfig.defaultBuildRepository) )
-            return false
-        if( buildConfig.defaultCacheRepository && repo.startsWith(buildConfig.defaultCacheRepository) )
-            return false
-        return true
-    }
-
     protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequest data, TokenData token, String target, String ip) {
         try {
             final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration)
@@ -394,13 +383,13 @@ class ContainerController {
             log.debug "== Dry-run build request: $build"
             final dryId = build.containerId +  BuildRequest.SEP + '0'
             final cached = digest!=null
-            return new BuildTrack(dryId, build.targetImage, cached)
+            return new BuildTrack(dryId, build.targetImage, cached, true)
         }
         // check for existing image
         if( digest ) {
             log.debug "== Found cached build for request: $build"
-            final cache = persistenceService.loadBuild(build.targetImage, digest)
-            return new BuildTrack(cache?.buildId, build.targetImage, true)
+            final cache = persistenceService.loadBuildSucceed(build.targetImage, digest)
+            return new BuildTrack(cache?.buildId, build.targetImage, true, true)
         }
         else {
             return buildService.buildImage(build)
@@ -436,14 +425,14 @@ class ContainerController {
         if( !digest && req.containerImage )
             throw new BadRequestException("Container image '${req.containerImage}' does not exist or access is not authorized")
 
+        ContainerRequest.Type type
         String targetImage
         String targetContent
         String condaContent
         String buildId
         boolean buildNew
         String scanId
-        Boolean mirrorFlag
-        Boolean scanOnRequest = false
+        Boolean succeeded
         if( req.containerFile ) {
             final build = makeBuildRequest(req, identity, ip)
             final track = checkBuild(build, req.dryRun)
@@ -453,9 +442,10 @@ class ContainerController {
             buildId = track.id
             buildNew = !track.cached
             scanId = build.scanId
-            mirrorFlag = false
+            succeeded = track.succeeded
+            type = ContainerRequest.Type.Build
         }
-        else if( req.mirrorRegistry ) {
+        else if( req.mirror ) {
             final mirror = makeMirrorRequest(req, identity, digest)
             final track = checkMirror(mirror, identity, req.dryRun)
             targetImage = track.targetImage
@@ -464,7 +454,8 @@ class ContainerController {
             buildId = track.id
             buildNew = !track.cached
             scanId = mirror.scanId
-            mirrorFlag = true
+            succeeded = track.succeeded
+            type = ContainerRequest.Type.Mirror
         }
         else if( req.containerImage ) {
             // normalize container image
@@ -475,13 +466,15 @@ class ContainerController {
             buildId = null
             buildNew = null
             scanId = scanService?.getScanId(req.containerImage, digest, req.scanMode, req.format)
-            mirrorFlag = null
-            scanOnRequest = true
+            type = ContainerRequest.Type.Container
+            // when there's a scan, return null because the scan status is not known
+            succeeded = scanId==null ? true : null
         }
         else
             throw new IllegalStateException("Specify either 'containerImage' or 'containerFile' attribute")
 
         ContainerRequest.create(
+                type,
                 identity,
                 targetImage,
                 targetContent,
@@ -491,21 +484,32 @@ class ContainerController {
                 buildId,
                 buildNew,
                 req.freeze,
-                mirrorFlag,
                 scanId,
                 req.scanMode,
                 req.scanLevels,
-                scanOnRequest,
+                req.dryRun,
+                succeeded,
                 Instant.now()
         )
     }
 
     protected MirrorRequest makeMirrorRequest(SubmitContainerTokenRequest request, PlatformId identity, String digest) {
         final coords = ContainerCoordinates.parse(request.containerImage)
-        if( coords.registry == request.mirrorRegistry )
-            throw new BadRequestException("Source and target mirror registry as the same - offending value '${request.mirrorRegistry}'")
-        final targetImage = request.mirrorRegistry + '/' + coords.imageAndTag
-        final configJson = inspectService.credentialsConfigJson(null, request.containerImage, targetImage, identity)
+        final target = ContainerCoordinates.parse(request.buildRepository)
+        if( !coords.imageAndTag )
+            throw new BadRequestException("Missing mirror source image - offending value '${request.containerImage}'")
+        if( !target.registry )
+            throw new BadRequestException("Missing mirror target registry - offending value '${request.buildRepository}'")
+        if( coords.registry == target.registry )
+            throw new BadRequestException("Source and target mirror registries are the same - offending value '${request.buildRepository}'")
+        final targetImage = target.repository
+                ? target.repository + '/' + coords.imageAndTag
+                : target.registry + '/' + coords.imageAndTag
+        // note: sourceImage is specified is provided via a dummy from-only dockerfile
+        // in order to bypass the strict credentials check made on the build and cache repo
+        // in fact, the absence of creds in the docker file is tolerated because it may be a
+        // public accessible repo. With build and cache repo, the creds needs to be available
+        final configJson = inspectService.credentialsConfigJson("FROM ${request.containerImage}", targetImage, null, identity)
         final platform = request.containerPlatform
                 ? ContainerPlatform.of(request.containerPlatform)
                 : ContainerPlatform.DEFAULT
@@ -535,13 +539,13 @@ class ContainerController {
         if( dryRun ) {
             log.debug "== Dry-run request request: $request"
             final dryId = request.mirrorId +  BuildRequest.SEP + '0'
-            return new BuildTrack(dryId, request.targetImage, cached)
+            return new BuildTrack(dryId, request.targetImage, cached, true)
         }
         // check for existing image
         if( request.digest==targetDigest ) {
             log.debug "== Found cached request for request: $request"
-            final cache = persistenceService.loadMirrorResult(request.targetImage, targetDigest)
-            return new BuildTrack(cache?.mirrorId, request.targetImage, true)
+            final cache = persistenceService.loadMirrorSucceed(request.targetImage, targetDigest)
+            return new BuildTrack(cache?.mirrorId, request.targetImage, true, true)
         }
         else {
             return mirrorService.mirrorImage(request)
@@ -582,41 +586,45 @@ class ContainerController {
         // check valid image name
         msg = validationService.checkContainerName(req.containerImage)
         if( msg ) throw new BadRequestException(msg)
+        // stop here for mirror request
+        if( req.mirror )
+            return
         // check build repo
-        msg = validationService.checkBuildRepository(req.buildRepository, false)
+        if( !req.mirror )
+            msg = validationService.checkBuildRepository(req.buildRepository, ValidationServiceImpl.RepoType.Build)
         if( msg ) throw new BadRequestException(msg)
         // check cache repository
-        msg = validationService.checkBuildRepository(req.cacheRepository, true)
+        msg = validationService.checkBuildRepository(req.cacheRepository, ValidationServiceImpl.RepoType.Cache)
         if( msg ) throw new BadRequestException(msg)
     }
 
     void validateMirrorRequest(SubmitContainerTokenRequest req, boolean v2) throws BadRequestException {
-        if( !req.mirrorRegistry )
+        if( !req.mirror )
             return
         // container mirror validation
         if( !v2 )
             throw new BadRequestException("Container mirroring requires the use of v2 API")
         if( !req.containerImage )
-            throw new BadRequestException("Attribute `containerImage` is required when specifying `mirrorRegistry`")
+            throw new BadRequestException("Attribute `containerImage` is required when specifying `mirror` mode")
         if( !req.towerAccessToken )
             throw new BadRequestException("Container mirroring requires an authenticated request - specify the tower token attribute")
         if( req.freeze )
-            throw new BadRequestException("Attribute `mirrorRegistry` and `freeze` conflict each other")
+            throw new BadRequestException("Attribute `mirror` and `freeze` conflict each other")
         if( req.containerFile )
-            throw new BadRequestException("Attribute `mirrorRegistry` and `containerFile` conflict each other")
+            throw new BadRequestException("Attribute `mirror` and `containerFile` conflict each other")
         if( req.containerIncludes )
-            throw new BadRequestException("Attribute `mirrorRegistry` and `containerIncludes` conflict each other")
+            throw new BadRequestException("Attribute `mirror` and `containerIncludes` conflict each other")
         if( req.containerConfig )
-            throw new BadRequestException("Attribute `mirrorRegistry` and `containerConfig` conflict each other")
+            throw new BadRequestException("Attribute `mirror` and `containerConfig` conflict each other")
+        if( !req.buildRepository )
+            throw new BadRequestException("Attribute `buildRepository` is required when specifying `mirror` mode")
         final coords = ContainerCoordinates.parse(req.containerImage)
-        if( coords.registry == req.mirrorRegistry )
-            throw new BadRequestException("Source and target mirror registry as the same - offending value '${req.mirrorRegistry}'")
-        def msg = validationService.checkMirrorRegistry(req.mirrorRegistry)
+        final target = ContainerCoordinates.parse(req.buildRepository)
+        if( coords.registry == target.registry )
+            throw new BadRequestException("Source and target mirror registry are the same - offending value '${req.buildRepository}'")
+        def msg = validationService.checkBuildRepository(req.buildRepository, ValidationServiceImpl.RepoType.Mirror)
         if( msg )
             throw new BadRequestException(msg)
-        if( !isCustomRepo0(req.mirrorRegistry) ) {
-            throw new BadRequestException("Not allowed mirror registry - offending value '${req.mirrorRegistry}'")
-        }
     }
 
     @Error(exception = AuthorizationException.class)

src/main/groovy/io/seqera/wave/controller/ViewController.groovy

@@ -169,11 +169,11 @@ class ViewController {
         // create template binding
         final binding = new HashMap(20)
         binding.build_id = result.buildId
-        binding.build_success = result.succeeded()
-        binding.build_failed = result.exitStatus  && result.exitStatus != 0
-        binding.build_in_progress = result.exitStatus == null
-        binding.build_exit_status = result.exitStatus
-        binding.build_user = (result.userName ?: '-') + " (ip: ${result.requestIp})"
+        binding.build_success = result.done() && result.succeeded()
+        binding.build_failed = result.done() && !result.succeeded() 
+        binding.build_in_progress = !result.done()
+        binding.build_exit_status = result.exitStatus != null ? result.exitStatus : '-'
+        binding.build_user = (result.userName ?: '-')
         binding.build_time = formatTimestamp(result.startTime, result.offsetId) ?: '-'
         binding.build_duration = formatDuration(result.duration) ?: '-'
         binding.build_image = result.targetImage
@@ -232,7 +232,7 @@ class ViewController {
         binding.tower_endpoint = data.towerEndpoint
 
         binding.build_container_file = data.containerFile
-        binding.build_conda_file = data.condaFile ?: '-'
+        binding.build_conda_file = data.condaFile
         binding.build_repository = data.buildRepository ?: '-'
         binding.build_cache_repository = data.cacheRepository  ?: '-'
         binding.build_id = data.buildId ?: '-'
@@ -247,7 +247,7 @@ class ViewController {
         binding.mirror_id = data.mirror ? data.buildId : null
         binding.mirror_url =  data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null
         binding.mirror_cached = data.mirror ? !data.buildNew : null
-
+        binding.server_url = serverUrl
         return HttpResponse.<Map<String,Object>>ok(binding)
     }
 

src/main/groovy/io/seqera/wave/service/blob/BlobStoreImpl.groovy

@@ -72,16 +72,16 @@ class BlobStoreImpl extends AbstractStateStore<BlobEntry> implements BlobStateSt
     }
 
     @Override
-    boolean storeIfAbsent(String key, BlobEntry info) {
-        return putIfAbsent(key, info)
+    boolean storeIfAbsent(String key, BlobEntry entry) {
+        return putIfAbsent(key, entry)
     }
 
     @Override
-    void storeBlob(String key, BlobEntry info) {
-        final ttl = info.state == BlobEntry.State.ERRORED
+    void storeBlob(String key, BlobEntry entry) {
+        final ttl = entry.state == BlobEntry.State.ERRORED
                 ? blobConfig.failureDuration
                 : blobConfig.statusDuration
-        put(key, info, ttl)
+        put(key, entry, ttl)
     }
 
 }