Merge branch 'master' into 675-render-api-docs-from-typespec-model

seqeralabs · Oct 28, 2024 · 0d4658b · 0d4658b
2 parents a79474b + 80c0f27
commit 0d4658b
Show file tree

Hide file tree

Showing 19 changed files with 236 additions and 85 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.13.5
+1.13.8
diff --git a/changelog.txt b/changelog.txt
@@ -1,4 +1,12 @@
 # Wave changelog
+1.13.8 - 26 Oct 2024
+
+1.13.7 - 25 Oct 2024 
+- Add ability to configure trivy environment & DBs (#720) [0f600306]
+
+1.13.6 - 25 Oct 2024 
+- Add scan color for different vuls (#719) [ab81b6dc]
+
 1.13.5 - 23 Oct 2024
 - Fix Do not render inspect url on fail [d96275a1]
 - Fix inspect view empty nodes (#706) [b3473b7e]

diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy
@@ -21,16 +21,14 @@ package io.seqera.wave.configuration
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
-
-import io.micronaut.context.annotation.Requires
-import io.micronaut.core.annotation.Nullable
 import javax.annotation.PostConstruct
 
 import groovy.transform.CompileStatic
 import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
+import io.micronaut.context.annotation.Requires
 import io.micronaut.context.annotation.Value
-import io.seqera.wave.util.StringUtils
+import io.micronaut.core.annotation.Nullable
 import jakarta.inject.Singleton
 /**
  * Container Scan service settings
@@ -83,8 +81,8 @@ class ScanConfig {
     Duration scanIdDuration
 
     @Nullable
-    @Value('${wave.scan.github-token}')
-    String githubToken
+    @Value('${wave.scan.environment}')
+    List<String> environment
 
     String getScanImage() {
         return scanImage
@@ -118,8 +116,23 @@ class ScanConfig {
         return severity
     }
 
+    List<Tuple2<String,String>> getEnvironmentAsTuples() {
+        if( !environment )
+            return List.of()
+        final result = new ArrayList<Tuple2<String,String>>()
+        for( String entry : environment ) {
+            final p=entry.indexOf('=')
+            final name = p!=-1 ? entry.substring(0,p) : entry
+            final value = p!=-1 ? entry.substring(p+1) : ''
+            if( !value )
+                log.warn "Invalid 'wave.scan.environment value' -- offending entry: '$entry'"
+            result.add(new Tuple2(name,value))
+        }
+        return result
+    }
+
     @PostConstruct
     private void init() {
-        log.info("Scanner config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; github-token=${StringUtils.redact(githubToken)}")
+        log.info("Scan config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; env=${environment}")
     }
 }
diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy
@@ -20,6 +20,7 @@ package io.seqera.wave.controller
 
 import java.util.regex.Pattern
 
+import groovy.transform.Canonical
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Value
@@ -45,6 +46,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord
 import io.seqera.wave.service.persistence.WaveScanRecord
 import io.seqera.wave.service.scan.ContainerScanService
 import io.seqera.wave.service.scan.ScanEntry
+import io.seqera.wave.service.scan.ScanVulnerability
 import io.seqera.wave.util.JacksonHelper
 import jakarta.inject.Inject
 import static io.seqera.wave.util.DataTimeUtils.formatDuration
@@ -408,7 +410,10 @@ class ViewController {
     }
 
     Map<String, Object> makeScanViewBinding(WaveScanRecord result, Map<String,Object> binding=new HashMap(10)) {
+        final color = getScanColor(result.vulnerabilities)
         binding.should_refresh = !result.done()
+        binding.scan_color_bg = color.background
+        binding.scan_color_fg = color.foreground
         binding.scan_id = result.id
         binding.scan_container_image = result.containerImage ?: '-'
         binding.scan_platform = result.platform?.toString()  ?: '-'
@@ -437,4 +442,24 @@ class ViewController {
         return binding
     }
 
+    @Canonical
+    static class Colour {
+        final background
+        final foreground
+    }
+
+    protected static Colour getScanColor(List<ScanVulnerability> vulnerabilities){
+        boolean hasMedium = vulnerabilities.stream()
+                .anyMatch(v -> v.severity.equals("MEDIUM"))
+        boolean hasHighOrCritical = vulnerabilities.stream()
+                .anyMatch(v -> v.severity.equals("HIGH") || v.severity.equals("CRITICAL"))
+        if(hasHighOrCritical){
+            return new Colour('#ffe4e2', '#e00404')
+        }
+        else if(hasMedium){
+            return new Colour('#fff8c5', "#000000")
+        }
+        return new Colour('#dff0d8', '#3c763d')
+    }
+
 }
diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy
@@ -29,7 +29,6 @@ import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.micronaut.context.event.ApplicationEventPublisher
 import io.micronaut.core.annotation.Nullable
-import io.micronaut.runtime.event.annotation.EventListener
 import io.micronaut.scheduling.TaskExecutors
 import io.seqera.wave.api.BuildContext
 import io.seqera.wave.auth.RegistryCredentialsProvider
@@ -191,8 +190,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
         catch (Throwable e) {
             log.error "== Container build unexpected exception: ${e.message} - request=$req", e
             final result = BuildResult.failed(req.buildId, e.message, req.startTime)
-            buildStore.storeBuild(req.targetImage, new BuildEntry(req, result))
-            eventPublisher.publishEvent(new BuildEvent(req, result))
+            handleBuildCompletion(new BuildEntry(req, result))
         }
     }
 
@@ -326,39 +324,42 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler<Bui
         final result = state.completed()
                 ? BuildResult.completed(buildId, state.exitCode, state.stdout, job.creationTime, digest)
                 : BuildResult.failed(buildId, state.stdout, job.creationTime)
-        buildStore.storeBuild(job.entryKey, entry.withResult(result))
-        eventPublisher.publishEvent(new BuildEvent(entry.request, result))
+        handleBuildCompletion(entry.withResult(result))
         log.info "== Container build completed '${entry.request.targetImage}' - operation=${job.operationName}; exit=${state.exitCode}; status=${state.status}; duration=${result.duration}"
     }
 
     @Override
     void onJobException(JobSpec job, BuildEntry entry, Throwable error) {
         final result= BuildResult.failed(entry.request.buildId, error.message, job.creationTime)
-        buildStore.storeBuild(job.entryKey, entry.withResult(result))
-        eventPublisher.publishEvent(new BuildEvent(entry.request, result))
+        handleBuildCompletion(entry.withResult(result))
         log.error("== Container build exception '${entry.request.targetImage}' - operation=${job.operationName}; cause=${error.message}", error)
     }
 
     @Override
     void onJobTimeout(JobSpec job, BuildEntry entry) {
         final buildId = entry.request.buildId
         final result= BuildResult.failed(buildId, "Container image build timed out '${entry.request.targetImage}'", job.creationTime)
-        buildStore.storeBuild(job.entryKey, entry.withResult(result))
-        eventPublisher.publishEvent(new BuildEvent(entry.request, result))
+        handleBuildCompletion(entry.withResult(result))
         log.warn "== Container build time out '${entry.request.targetImage}'; operation=${job.operationName}; duration=${result.duration}"
     }
 
+    protected handleBuildCompletion(BuildEntry entry) {
+        final event = new BuildEvent(entry.request, entry.result)
+        final targetImage = entry.request.targetImage
+        // since the underlying persistence is *not* transactional
+        // the scan request should be submitted *before* updating the record
+        // otherwise the scan status service can detect a complete build
+        // for which a scan is requested but not scan record exists
+        scanService?.scanOnBuild(entry)
+        buildStore.storeBuild(targetImage, entry)
+        persistenceService.saveBuild(WaveBuildRecord.fromEvent(event))
+        eventPublisher.publishEvent(event)
+    }
+
     // **************************************************************
     // **               build record implementation
     // **************************************************************
 
-    @EventListener
-    protected void onBuildEvent(BuildEvent event) {
-        final record0 = WaveBuildRecord.fromEvent(event)
-        persistenceService.saveBuild(record0)
-        scanService?.scanOnBuild(event)
-    }
-
     /**
      * Retrieve the build record for the specified id.
      *

diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy
@@ -784,8 +784,11 @@ class K8sServiceImpl implements K8sService {
                 .withVolumeMounts(mounts)
                 .withResources(requests)
 
-        if( scanConfig.githubToken ) {
-            container.withEnv(new V1EnvVar().name('GITHUB_TOKEN').value(scanConfig.githubToken))
+        final env = scanConfig.environmentAsTuples
+        for( Tuple2 entry : env ) {
+            final String k = entry.v1
+            final String v = entry.v2
+            container.withEnv(new V1EnvVar().name(k).value(v))
         }
 
         // spec section

diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy
@@ -124,9 +124,14 @@ class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler<M
     @Override
     void onJobCompletion(JobSpec jobSpec, MirrorEntry entry, JobState jobState) {
         final result = entry.result.complete(jobState.exitCode, jobState.stdout)
-        store.putEntry(entry.withResult(result))
+        final updated = entry.withResult(result)
+        // since the underlying persistence is *not* transactional
+        // the scan request should be submitted *before* updating the record
+        // otherwise the scan status service can detect a complete build
+        // for which a scan is requested but not scan record exists
+        scanService?.scanOnMirror(updated)
+        store.putEntry(updated)
         persistence.saveMirrorResult(result)
-        scanService?.scanOnMirror(entry.withResult(result))
         log.debug "Mirror container completed - job=${jobSpec.operationName}; result=${result}; state=${jobState}"
     }
 

diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy
@@ -19,11 +19,10 @@
 package io.seqera.wave.service.scan
 
 import io.seqera.wave.api.ScanMode
-import io.seqera.wave.service.builder.BuildEvent
+import io.seqera.wave.service.builder.BuildEntry
 import io.seqera.wave.service.mirror.MirrorEntry
 import io.seqera.wave.service.persistence.WaveScanRecord
 import io.seqera.wave.service.request.ContainerRequest
-
 /**
  * Declare operations to scan containers
  *
@@ -36,7 +35,7 @@ interface ContainerScanService {
 
     void scan(ScanRequest request)
 
-    void scanOnBuild(BuildEvent build)
+    void scanOnBuild(BuildEntry build)
 
     void scanOnMirror(MirrorEntry entry)
 

diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy
@@ -29,7 +29,7 @@ import io.micronaut.context.annotation.Requires
 import io.micronaut.scheduling.TaskExecutors
 import io.seqera.wave.api.ScanMode
 import io.seqera.wave.configuration.ScanConfig
-import io.seqera.wave.service.builder.BuildEvent
+import io.seqera.wave.service.builder.BuildEntry
 import io.seqera.wave.service.builder.BuildRequest
 import io.seqera.wave.service.cleanup.CleanupService
 import io.seqera.wave.service.inspect.ContainerInspectService
@@ -106,14 +106,14 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler<ScanE
     }
 
     @Override
-    void scanOnBuild(BuildEvent event) {
+    void scanOnBuild(BuildEntry entry) {
         try {
-            if( event.request.scanId && event.result.succeeded() && event.request.format == DOCKER ) {
-                scan(fromBuild(event.request))
+            if( entry.request.scanId && entry.result.succeeded() && entry.request.format == DOCKER ) {
+                scan(fromBuild(entry.request))
             }
         }
         catch (Exception e) {
-            log.warn "Unable to run the container scan - image=${event.request.targetImage}; reason=${e.message?:e}"
+            log.warn "Unable to run the container scan - image=${entry.request.targetImage}; reason=${e.message?:e}"
         }
     }
 
@@ -151,9 +151,18 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler<ScanE
 
     @Override
     void scan(ScanRequest request) {
-        //start scanning of build container
-        CompletableFuture
-                .runAsync(() -> launch(request), executor)
+        try {
+            // create a record to mark the beginning
+            final scan = ScanEntry.create(request)
+            if( scanStore.putIfAbsent(scan.scanId, scan) ) {
+                //start scanning of build container
+                CompletableFuture.runAsync(() -> launch(request), executor)
+            }
+        }
+        catch (Throwable e){
+            log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e
+            storeScanEntry(ScanEntry.failure(request))
+        }
     }
 
     @Override
@@ -181,21 +190,25 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler<ScanE
 
     protected void launch(ScanRequest request) {
         try {
-            // create a record to mark the beginning
-            final scan = ScanEntry.create(request)
-            if( scanStore.putIfAbsent(scan.scanId, scan) ) {
-                //increment metrics
-                CompletableFuture.supplyAsync(() -> metricsService.incrementScansCounter(request.identity), executor)
-                // launch container scan
-                jobService.launchScan(request)
-            }
+            incrScanMetrics(request)
+            jobService.launchScan(request)
         }
         catch (Throwable e){
             log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e
             storeScanEntry(ScanEntry.failure(request))
         }
     }
 
+    protected void incrScanMetrics(ScanRequest request) {
+        try {
+            //increment metrics
+            metricsService.incrementScansCounter(request.identity)
+        }
+        catch (Throwable e) {
+            log.warn "Enable to increase scan metrics - cause: ${e.cause}", e
+        }
+    }
+
     protected ScanRequest fromBuild(BuildRequest request) {
         final workDir = request.workDir.resolveSibling(request.scanId)
         return new ScanRequest(

diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy
@@ -53,7 +53,7 @@ class DockerScanStrategy extends ScanStrategy {
 
     @Override
     void scanContainer(String jobName, ScanRequest req) {
-        log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}")
+        log.info("Launching container scan job: $jobName for request: ${req}")
 
         // create the scan dir
         try {
@@ -73,7 +73,7 @@ class DockerScanStrategy extends ScanStrategy {
         // outfile file name
         final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME)
         // create the launch command
-        final dockerCommand = dockerWrapper(jobName, req.workDir, configFile, scanConfig.githubToken)
+        final dockerCommand = dockerWrapper(jobName, req.workDir, configFile, scanConfig.environment)
         final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, req.platform, scanConfig)
         final command = dockerCommand + trivyCommand
 
@@ -89,7 +89,7 @@ class DockerScanStrategy extends ScanStrategy {
         }
     }
 
-    protected List<String> dockerWrapper(String jobName, Path scanDir, Path credsFile, String githubToken) {
+    protected List<String> dockerWrapper(String jobName, Path scanDir, Path credsFile, List<String> env) {
 
         final wrapper = ['docker','run']
         wrapper.add('--detach')
@@ -112,9 +112,11 @@ class DockerScanStrategy extends ScanStrategy {
             wrapper.add("${credsFile}:${Trivy.CONFIG_MOUNT_PATH}:ro".toString())
         }
 
-        if( githubToken ) {
-            wrapper.add('-e')
-            wrapper.add("GITHUB_TOKEN="+githubToken)
+        if( env ) {
+            for( String it : env ) {
+                wrapper.add('-e')
+                wrapper.add(it)
+            }
         }
 
         return wrapper

diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy
@@ -65,7 +65,7 @@ class KubeScanStrategy extends ScanStrategy {
 
     @Override
     void scanContainer(String jobName, ScanRequest req) {
-        log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}")
+        log.info("Launching container scan job: $jobName for request: ${req}")
         try{
             // create the scan dir
             try {

diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml
@@ -4,7 +4,6 @@ wave:
     enabled: true
     failure:
       duration: '30s'
-    github-token: "${GITHUB_TOKEN:}"
   build:
     workspace: 'build-workspace'
   metrics: