diff --git a/build.gradle b/build.gradle index e11aee694..530d84bf3 100644 --- a/build.gradle +++ b/build.gradle @@ -35,9 +35,8 @@ dependencies { compileOnly("io.micronaut:micronaut-http-validation") implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") api 'io.seqera:lib-mail:1.0.0' - api 'io.seqera:wave-api:0.12.0' - api 'io.seqera:wave-utils:0.13.1' - + api 'io.seqera:wave-api:0.13.1' + api 'io.seqera:wave-utils:0.14.1' implementation("io.micronaut:micronaut-http-client") implementation("io.micronaut:micronaut-jackson-databind") implementation("io.micronaut.groovy:micronaut-runtime-groovy") @@ -72,6 +71,7 @@ dependencies { implementation 'software.amazon.awssdk:ses' implementation 'org.yaml:snakeyaml:2.0' implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' + implementation 'org.luaj:luaj-jse:3.0.1' //object storage dependency implementation("io.micronaut.objectstorage:micronaut-object-storage-aws") // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570 diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy index de729e340..ba1c834c4 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthStore.groovy @@ -51,7 +51,7 @@ class RegistryAuthStore extends AbstractStateStore { @Override protected String getPrefix() { - return 'registry-auth/v1:' + return 'registry-auth/v1' } @Override diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryTokenStore.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryTokenStore.groovy index 8595bcebc..f20eb2db8 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryTokenStore.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryTokenStore.groovy @@ -44,7 +44,7 @@ class RegistryTokenStore extends AbstractStateStore { @Override protected String getPrefix() { - return 'registry-token/v1:' + return 'registry-token/v1' } @Override diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index 41749304b..3d1e9cdf0 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -75,6 +75,9 @@ class ScanConfig { @Value('${wave.scan.status.duration:1h}') Duration statusDuration + @Value('${wave.scan.id.duration:7d}') + Duration scanIdDuration + String getScanImage() { return scanImage } @@ -86,6 +89,11 @@ class ScanConfig { return result } + @Memoized + Path getWorkspace() { + Path.of(buildDirectory).toAbsolutePath() + } + String getRequestsCpu() { return requestsCpu } diff --git a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy index a4c5378ca..de0a9479a 100644 --- a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.controller import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.core.annotation.Nullable +import io.micronaut.http.HttpHeaders import io.micronaut.http.HttpResponse import io.micronaut.http.MediaType import io.micronaut.http.annotation.Controller @@ -30,11 +31,8 @@ import io.micronaut.http.server.types.files.StreamedFile import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.api.BuildStatusResponse -import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.logs.BuildLogService -import io.seqera.wave.service.mirror.ContainerMirrorService -import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.persistence.WaveBuildRecord import jakarta.inject.Inject /** @@ -51,12 +49,9 @@ class BuildController { @Inject private ContainerBuildService buildService - @Inject - private ContainerMirrorService mirrorService - @Inject @Nullable - BuildLogService logService + private BuildLogService logService @Get("/v1alpha1/builds/{buildId}") HttpResponse getBuildRecord(String buildId) { @@ -79,25 +74,22 @@ class BuildController { @Get("/v1alpha1/builds/{buildId}/status") HttpResponse getBuildStatus(String buildId) { - final resp = buildResponse0(buildId) - resp != null - ? HttpResponse.ok(resp) + final build = buildService.getBuildRecord(buildId) + build != null + ? HttpResponse.ok(build.toStatusResponse()) : HttpResponse.notFound() } - protected BuildStatusResponse buildResponse0(String buildId) { - if( !buildId ) - throw new BadRequestException("Missing 'buildId' parameter") - // build IDs starting with the `mr-` prefix are interpreted as mirror requests - if( buildId.startsWith(MirrorRequest.ID_PREFIX) ) { - return mirrorService - .getMirrorEntry(buildId) - ?.toStatusResponse() - } - else { - return buildService - .getBuildRecord(buildId) - ?.toStatusResponse() - } + @Produces(MediaType.TEXT_PLAIN) + @Get(value="/v1alpha1/builds/{buildId}/condalock") + HttpResponse getCondaLock(String buildId){ + if( logService==null ) + throw new IllegalStateException("Build Logs service not configured") + final condaLock = logService.fetchCondaLockStream(buildId) + return condaLock + ? HttpResponse.ok(condaLock) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"conda-env-${buildId}.lock\"") + : HttpResponse.notFound() } + } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index f42ffc08f..cf5780ec9 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.controller import java.nio.file.Path +import java.time.Instant import java.util.concurrent.CompletableFuture import javax.annotation.PostConstruct @@ -39,7 +40,9 @@ import io.micronaut.scheduling.annotation.ExecuteOn import io.micronaut.security.annotation.Secured import io.micronaut.security.authentication.AuthorizationException import io.micronaut.security.rules.SecurityRule +import io.seqera.wave.api.ContainerStatusResponse import io.seqera.wave.api.ImageNameStrategy +import io.seqera.wave.api.ScanMode import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.api.SubmitContainerTokenResponse import io.seqera.wave.configuration.BuildConfig @@ -51,7 +54,6 @@ import io.seqera.wave.exchange.DescribeWaveContainerResponse import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService -import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.UserService import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildTrack @@ -65,15 +67,17 @@ import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.token.ContainerTokenService -import io.seqera.wave.service.token.TokenData +import io.seqera.wave.service.request.ContainerRequest +import io.seqera.wave.service.request.ContainerRequestService +import io.seqera.wave.service.request.ContainerStatusService +import io.seqera.wave.service.request.TokenData +import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User import io.seqera.wave.tower.auth.JwtAuth import io.seqera.wave.tower.auth.JwtAuthStore import io.seqera.wave.util.DataTimeUtils -import io.seqera.wave.util.LongRndKey import jakarta.inject.Inject import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE import static io.seqera.wave.service.builder.BuildFormat.DOCKER @@ -100,63 +104,74 @@ import static java.util.concurrent.CompletableFuture.completedFuture @ExecuteOn(TaskExecutors.IO) class ContainerController { - @Inject HttpClientAddressResolver addressResolver - @Inject ContainerTokenService tokenService - @Inject UserService userService - @Inject JwtAuthStore jwtAuthStore + @Inject + private HttpClientAddressResolver addressResolver + + @Inject + private ContainerRequestService containerService + + @Inject + private UserService userService + + @Inject + private JwtAuthStore jwtAuthStore @Inject @Value('${wave.allowAnonymous}') - Boolean allowAnonymous + private Boolean allowAnonymous @Inject @Value('${wave.server.url}') - String serverUrl + private String serverUrl @Inject @Value('${tower.endpoint.url:`https://api.cloud.seqera.io`}') - String towerEndpointUrl - - @Value('${wave.scan.enabled:false}') - boolean scanEnabled + private String towerEndpointUrl @Inject - BuildConfig buildConfig + private BuildConfig buildConfig @Inject - ContainerBuildService buildService + private ContainerBuildService buildService @Inject - ContainerInspectService inspectService + private ContainerInspectService inspectService @Inject - RegistryProxyService registryProxyService + private RegistryProxyService registryProxyService @Inject - PersistenceService persistenceService + private PersistenceService persistenceService @Inject - ValidationService validationService + private ValidationService validationService @Inject - PairingService pairingService + private PairingService pairingService @Inject - PairingChannel pairingChannel + private PairingChannel pairingChannel @Inject - FreezeService freezeService + private FreezeService freezeService @Inject - ContainerInclusionService inclusionService + private ContainerInclusionService inclusionService @Inject @Nullable - RateLimiterService rateLimiterService + private RateLimiterService rateLimiterService @Inject private ContainerMirrorService mirrorService + @Inject + private ContainerStatusService statusService + + @Inject + @Nullable + private ContainerScanService scanService + @PostConstruct private void init() { log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: $buildConfig.defaultBuildRepository; default-cache-repo: $buildConfig.defaultCacheRepository; default-public-repo: $buildConfig.defaultPublicRepository" @@ -251,13 +266,14 @@ class ContainerController { rateLimiterService.acquirePull(new AcquireRequest(identity.userId as String, ip)) // create request data final data = makeRequestData(req, identity, ip) - final token = tokenService.computeToken(data) + final token = containerService.computeToken(data) final target = targetImage(token.value, data.coordinates()) final resp = v2 ? makeResponseV2(data, token, target) : makeResponseV1(data, token, target) // persist request storeContainerRequest0(req, data, token, target, ip) + scanService?.scanOnRequest(data) // log the response log.debug "New container request fulfilled - token=$token.value; expiration=$token.expiration; container=$data.containerImage; build=$resp.buildId; identity=$identity" // return response @@ -276,9 +292,9 @@ class ContainerController { return true } - protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequestData data, TokenData token, String target, String ip) { + protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequest data, TokenData token, String target, String ip) { try { - final recrd = new WaveContainerRecord(req, data, token.value, target, ip, token.expiration) + final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) persistenceService.saveContainerRequest(recrd) } catch (Throwable e) { @@ -334,7 +350,6 @@ class ContainerController { final configJson = inspectService.credentialsConfigJson(containerSpec, buildRepository, cacheRepository, identity) final containerConfig = req.freeze ? req.containerConfig : null final offset = DataTimeUtils.offsetId(req.timestamp) - final scanId = scanEnabled && format==DOCKER ? LongRndKey.rndHex() : null // use 'imageSuffix' strategy by default for public repo images final nameStrategy = req.nameStrategy==null && buildRepository @@ -343,10 +358,15 @@ class ContainerController { checkContainerSpec(containerSpec) - // create a unique digest to identify the build request + // create a unique digest to identify the build req final containerId = makeContainerId(containerSpec, condaContent, platform, buildRepository, req.buildContext) final targetImage = makeTargetImage(format, buildRepository, containerId, condaContent, nameStrategy) final maxDuration = buildConfig.buildMaxDuration(req) + // default to async scan for build req for backward compatibility + final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async + // digest is not needed for builds, because they use a unique checksum in the container name + final scanId = scanService?.getScanId(targetImage, null, scanMode, req.format) + return new BuildRequest( containerId, containerSpec, @@ -387,7 +407,13 @@ class ContainerController { } } - ContainerRequestData makeRequestData(SubmitContainerTokenRequest req, PlatformId identity, String ip) { + protected String getContainerDigest(String containerImage, PlatformId identity) { + containerImage + ? registryProxyService.getImageDigest(containerImage, identity) + : null + } + + ContainerRequest makeRequestData(SubmitContainerTokenRequest req, PlatformId identity, String ip) { if( !req.containerImage && !req.containerFile ) throw new BadRequestException("Specify either 'containerImage' or 'containerFile' attribute") if( req.containerImage && req.containerFile ) @@ -406,11 +432,18 @@ class ContainerController { req = freezeService.freezeBuildRequest(req, identity) } + final digest = getContainerDigest(req.containerImage, identity) + if( !digest && req.containerImage ) + throw new BadRequestException("Container image '${req.containerImage}' does not exist or access is not authorized") + String targetImage String targetContent String condaContent String buildId boolean buildNew + String scanId + Boolean mirrorFlag + Boolean scanOnRequest = false if( req.containerFile ) { final build = makeBuildRequest(req, identity, ip) final track = checkBuild(build, req.dryRun) @@ -419,15 +452,19 @@ class ContainerController { condaContent = build.condaFile buildId = track.id buildNew = !track.cached + scanId = build.scanId + mirrorFlag = false } else if( req.mirrorRegistry ) { - final mirror = makeMirrorRequest(req, identity) + final mirror = makeMirrorRequest(req, identity, digest) final track = checkMirror(mirror, identity, req.dryRun) targetImage = track.targetImage targetContent = null condaContent = null buildId = track.id buildNew = !track.cached + scanId = mirror.scanId + mirrorFlag = true } else if( req.containerImage ) { // normalize container image @@ -437,11 +474,14 @@ class ContainerController { condaContent = null buildId = null buildNew = null + scanId = scanService?.getScanId(req.containerImage, digest, req.scanMode, req.format) + mirrorFlag = null + scanOnRequest = true } else throw new IllegalStateException("Specify either 'containerImage' or 'containerFile' attribute") - new ContainerRequestData( + ContainerRequest.create( identity, targetImage, targetContent, @@ -451,11 +491,16 @@ class ContainerController { buildId, buildNew, req.freeze, - req.mirrorRegistry!=null + mirrorFlag, + scanId, + req.scanMode, + req.scanLevels, + scanOnRequest, + Instant.now() ) } - protected MirrorRequest makeMirrorRequest(SubmitContainerTokenRequest request, PlatformId identity) { + protected MirrorRequest makeMirrorRequest(SubmitContainerTokenRequest request, PlatformId identity, String digest) { final coords = ContainerCoordinates.parse(request.containerImage) if( coords.registry == request.mirrorRegistry ) throw new BadRequestException("Source and target mirror registry as the same - offending value '${request.mirrorRegistry}'") @@ -464,16 +509,22 @@ class ContainerController { final platform = request.containerPlatform ? ContainerPlatform.of(request.containerPlatform) : ContainerPlatform.DEFAULT - final digest = registryProxyService.getImageDigest(request.containerImage, identity) - if( !digest ) - throw new BadRequestException("Container image '$request.containerImage' does not exist") + + final offset = DataTimeUtils.offsetId(request.timestamp) + final scanId = scanService?.getScanId(targetImage, digest, request.scanMode, request.format) + return MirrorRequest.create( request.containerImage, targetImage, digest, platform, Path.of(buildConfig.buildWorkspace).toAbsolutePath(), - configJson ) + configJson, + scanId, + Instant.now(), + offset, + identity + ) } protected BuildTrack checkMirror(MirrorRequest request, PlatformId identity, boolean dryRun) { @@ -489,7 +540,7 @@ class ContainerController { // check for existing image if( request.digest==targetDigest ) { log.debug "== Found cached request for request: $request" - final cache = persistenceService.loadMirrorEntry(request.targetImage, targetDigest) + final cache = persistenceService.loadMirrorResult(request.targetImage, targetDigest) return new BuildTrack(cache?.mirrorId, request.targetImage, true) } else { @@ -503,7 +554,7 @@ class ContainerController { @Get('/container-token/{token}') HttpResponse describeContainerRequest(String token) { - final data = persistenceService.loadContainerRequest(token) + final data = containerService.loadContainerRecord(token) if( !data ) throw new NotFoundException("Missing container record for token: $token") // return the response @@ -513,7 +564,7 @@ class ContainerController { @Secured(SecurityRule.IS_AUTHENTICATED) @Delete('/container-token/{token}') HttpResponse deleteContainerRequest(String token) { - final record = tokenService.evictRequest(token) + final record = containerService.evictRequest(token) if( !record ){ throw new NotFoundException("Missing container record for token: $token") } @@ -571,12 +622,20 @@ class ContainerController { .header(WWW_AUTHENTICATE, "Basic realm=Wave Authentication") } - @Get('/v1alpha2/container/{containerId}') - HttpResponse getContainerDetails(String containerId) { - final data = persistenceService.loadContainerRequest(containerId) + @Get('/v1alpha2/container/{requestId}') + HttpResponse getContainerDetails(String requestId) { + final data = containerService.loadContainerRecord(requestId) if( !data ) return HttpResponse.notFound() return HttpResponse.ok(data) } + @Get('/v1alpha2/container/{requestId}/status') + HttpResponse getContainerStatus(String requestId) { + final ContainerStatusResponse resp = statusService.getContainerStatus(requestId) + if( !resp ) + return HttpResponse.notFound() + return HttpResponse.ok(resp) + } + } diff --git a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy index 75253b2d4..8aaf87a51 100644 --- a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy @@ -26,7 +26,7 @@ import io.micronaut.http.annotation.Get import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.service.mirror.ContainerMirrorService -import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult import jakarta.inject.Inject /** * Implements a controller for container mirror apis @@ -43,11 +43,11 @@ class MirrorController { private ContainerMirrorService mirrorService @Get("/v1alpha1/mirrors/{mirrorId}") - HttpResponse getMirrorRecord(String mirrorId) { - final result = mirrorService.getMirrorEntry(mirrorId) + HttpResponse getMirrorRecord(String mirrorId) { + final result = mirrorService.getMirrorResult(mirrorId) return result ? HttpResponse.ok(result) - : HttpResponse.notFound() + : HttpResponse.notFound() } } diff --git a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy index 830c60eb9..230a8b4a9 100644 --- a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy @@ -47,7 +47,7 @@ class ScanController { @Get("/v1alpha1/scans/{scanId}") HttpResponse scanImage(String scanId){ - final record = containerScanService.getScanResult(scanId) + final record = containerScanService.getScanRecord(scanId) return record ? HttpResponse.ok(record) : HttpResponse.notFound() diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 5b8c16aff..bbe9bcb1b 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -25,18 +25,23 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpResponse +import io.micronaut.http.HttpStatus import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Get import io.micronaut.http.annotation.QueryValue import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.micronaut.views.View +import io.seqera.wave.exception.HttpResponseException import io.seqera.wave.exception.NotFoundException import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.logs.BuildLogService +import io.seqera.wave.service.mirror.ContainerMirrorService +import io.seqera.wave.service.mirror.MirrorResult import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.util.JacksonHelper @@ -72,8 +77,43 @@ class ViewController { private ContainerInspectService inspectService @Inject + @Nullable private ContainerScanService scanService + @Inject + private ContainerMirrorService mirrorService + + @View("mirror-view") + @Get('/mirrors/{mirrorId}') + HttpResponse viewMirror(String mirrorId) { + final result = mirrorService.getMirrorResult(mirrorId) + if( !result ) + throw new NotFoundException("Unknown container mirror id '$mirrorId'") + return HttpResponse.ok(renderMirrorView(result)) + } + + protected Map renderMirrorView(MirrorResult result) { + // create template binding + final binding = new HashMap(20) + binding.mirror_id = result.mirrorId + binding.mirror_success = result.succeeded() + binding.mirror_failed = result.exitCode && result.exitCode != 0 + binding.mirror_in_progress = result.exitCode == null + binding.mirror_exitcode = result.exitCode ?: null + binding.mirror_logs = result.exitCode ? result.logs : null + binding.mirror_time = formatTimestamp(result.creationTime, result.offsetId) ?: '-' + binding.mirror_duration = formatDuration(result.duration) ?: '-' + binding.mirror_source_image = result.sourceImage + binding.mirror_target_image = result.targetImage + binding.mirror_platform = result.platform + binding.mirror_digest = result.digest ?: '-' + binding.mirror_user = result.userName ?: '-' + binding.put('server_url', serverUrl) + binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null + binding.scan_id = result.scanId + return binding + } + @View("build-view") @Get('/builds/{buildId}') HttpResponse viewBuild(String buildId) { @@ -92,7 +132,7 @@ class ViewController { // go ahead with proper handling final record = buildService.getBuildRecord(buildId) if( !record ) - throw new NotFoundException("Unknown build id '$buildId'") + throw new NotFoundException("Unknown container build id '$buildId'") return HttpResponse.ok(renderBuildView(record)) } @@ -117,7 +157,7 @@ class ViewController { final rec = buildService.getLatestBuild(buildId) if( !rec ) return null - if( !rec.buildId.startsWith(buildId) ) + if( !rec.buildId.contains(buildId) ) return null if( rec.buildId==buildId ) return null @@ -152,6 +192,10 @@ class ViewController { binding.build_log_truncated = buildLog?.truncated binding.build_log_url = "$serverUrl/v1alpha1/builds/${result.buildId}/logs" } + //add conda lock file when available + if( buildLogService && result.condaFile ) { + binding.build_conda_lock_data = buildLogService.fetchCondaLockString(result.buildId) + } // result the main object return binding } @@ -175,6 +219,7 @@ class ViewController { binding.source_container_image = data.sourceImage ?: '-' binding.source_container_digest = data.sourceDigest ?: '-' + binding.wave_container_show = !data.freeze && !data.mirror ? true : null binding.wave_container_image = data.waveImage ?: '-' binding.wave_container_digest = data.waveDigest ?: '-' @@ -192,9 +237,16 @@ class ViewController { binding.build_id = data.buildId ?: '-' binding.build_cached = data.buildId ? !data.buildNew : '-' binding.build_freeze = data.buildId ? data.freeze : '-' - binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : '#' + binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null binding.fusion_version = data.fusionVersion ?: '-' + binding.scan_id = data.scanId + binding.scan_url = data.scanId ? "$serverUrl/view/scans/${data.scanId}" : null + + binding.mirror_id = data.mirror ? data.buildId : null + binding.mirror_url = data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null + binding.mirror_cached = data.mirror ? !data.buildNew : null + return HttpResponse.>ok(binding) } @@ -203,7 +255,8 @@ class ViewController { HttpResponse> viewScan(String scanId) { final binding = new HashMap(10) try { - final result = loadScanResult(scanId) + final result = loadScanRecord(scanId) + log.debug "Render scan record=$result" makeScanViewBinding(result, binding) } catch (NotFoundException e){ @@ -225,19 +278,13 @@ class ViewController { * @return The {@link ScanEntry} object associated with the specified build ID or throws the exception {@link NotFoundException} otherwise * @throws NotFoundException If the a record for the specified build ID cannot be found */ - protected ScanEntry loadScanResult(String scanId) { - final scanRecord = scanService.getScanResult(scanId) + protected WaveScanRecord loadScanRecord(String scanId) { + if( !scanService ) + throw new HttpResponseException(HttpStatus.SERVICE_UNAVAILABLE, "Scan service is not enabled - Check Wave configuration setting 'wave.scan.enabled'") + final scanRecord = scanService.getScanRecord(scanId) if( !scanRecord ) throw new NotFoundException("No scan report exists with id: ${scanId}") - - return ScanEntry.create( - scanRecord.id, - scanRecord.buildId, - scanRecord.containerImage, - scanRecord.startTime, - scanRecord.duration, - scanRecord.status, - scanRecord.vulnerabilities ) + return scanRecord } @View("inspect-view") @@ -262,9 +309,9 @@ class ViewController { return HttpResponse.>ok(binding) } - Map makeScanViewBinding(ScanEntry result, Map binding=new HashMap(10)) { + Map makeScanViewBinding(WaveScanRecord result, Map binding=new HashMap(10)) { binding.should_refresh = !result.done() - binding.scan_id = result.scanId + binding.scan_id = result.id binding.scan_container_image = result.containerImage ?: '-' binding.scan_exist = true binding.scan_completed = result.done() @@ -273,9 +320,16 @@ class ViewController { binding.scan_succeeded = result.status == ScanEntry.SUCCEEDED binding.scan_exitcode = result.exitCode binding.scan_logs = result.logs - + // build info binding.build_id = result.buildId - binding.build_url = "$serverUrl/view/builds/${result.buildId}" + binding.build_url = result.buildId ? "$serverUrl/view/builds/${result.buildId}" : null + // mirror info + binding.mirror_id = result.mirrorId + binding.mirror_url = result.mirrorId ? "$serverUrl/view/mirrors/${result.mirrorId}" : null + // container info + binding.request_id = result.requestId + binding.request_url = result.requestId ? "$serverUrl/view/containers/${result.requestId}" : null + binding.scan_time = formatTimestamp(result.startTime) ?: '-' binding.scan_duration = formatDuration(result.duration) ?: '-' if ( result.vulnerabilities ) diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index e8a8fae6a..1d9bf61de 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -203,7 +203,7 @@ class RegistryProxyService { static private List RETRY_ON_NOT_FOUND = HTTP_RETRYABLE_ERRORS + 404 - @Cacheable(value = 'cache-registry-proxy', atomic = true) + @Cacheable(value = 'cache-registry-proxy', atomic = true, parameters = ['image']) protected String getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { final coords = ContainerCoordinates.parse(image) final route = RoutePath.v2manifestPath(coords, identity) diff --git a/src/main/groovy/io/seqera/wave/core/RouteHandler.groovy b/src/main/groovy/io/seqera/wave/core/RouteHandler.groovy index e5a055565..632af69b1 100644 --- a/src/main/groovy/io/seqera/wave/core/RouteHandler.groovy +++ b/src/main/groovy/io/seqera/wave/core/RouteHandler.groovy @@ -24,7 +24,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.exception.NotFoundException -import io.seqera.wave.service.token.ContainerTokenService +import io.seqera.wave.service.request.ContainerRequestService import jakarta.inject.Singleton /** * Helper service to decode container request paths @@ -38,9 +38,9 @@ class RouteHandler { final public static Pattern ROUTE_PATHS = ~'/v2(?:/wt)?/([a-z0-9][a-z0-9_.-]+(?:/[a-z0-9][a-z0-9_.-]+)?(?:/[a-zA-Z0-9][a-zA-Z0-9_.-]+)*)/(manifests|blobs|tags)/(.+)' - private ContainerTokenService tokenService + private ContainerRequestService tokenService - RouteHandler(ContainerTokenService tokenService) { + RouteHandler(ContainerRequestService tokenService) { this.tokenService = tokenService } diff --git a/src/main/groovy/io/seqera/wave/core/RoutePath.groovy b/src/main/groovy/io/seqera/wave/core/RoutePath.groovy index bfe1bffe1..caeb8b448 100644 --- a/src/main/groovy/io/seqera/wave/core/RoutePath.groovy +++ b/src/main/groovy/io/seqera/wave/core/RoutePath.groovy @@ -25,7 +25,7 @@ import groovy.transform.CompileStatic import groovy.transform.ToString import io.micronaut.core.annotation.Nullable import io.seqera.wave.model.ContainerCoordinates -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.PlatformId import static io.seqera.wave.WaveDefault.DOCKER_IO /** @@ -68,9 +68,9 @@ class RoutePath implements ContainerPath { final String path /** - * The {@link ContainerRequestData} metadata associated with the wave request + * The {@link ContainerRequest} metadata associated with the wave request */ - final ContainerRequestData request + final ContainerRequest request /** * The unique token associated with the wave container request. it may be null when mapping @@ -119,13 +119,13 @@ class RoutePath implements ContainerPath { return token && isManifest() && isTag() } - static RoutePath v2path(String type, String registry, String image, String ref, ContainerRequestData request=null, String token=null) { + static RoutePath v2path(String type, String registry, String image, String ref, ContainerRequest request=null, String token=null) { assert type in ALLOWED_TYPES, "Unknown container path type: '$type'" new RoutePath(type, registry ?: DOCKER_IO, image, ref, "/v2/$image/$type/$ref", request, token) } static RoutePath v2manifestPath(ContainerCoordinates container, PlatformId identity=null) { - ContainerRequestData data = identity!=null ? new ContainerRequestData(identity) : null + ContainerRequest data = identity!=null ? ContainerRequest.of(identity) : null return new RoutePath('manifests', container.registry, container.image, container.reference, "/v2/${container.image}/manifests/${container.reference}", data) } @@ -144,7 +144,7 @@ class RoutePath implements ContainerPath { final image = m.group(2) final type = m.group(3) final reference = m.group(4) - final data = identity!=null ? new ContainerRequestData(identity) : null + final data = identity!=null ? ContainerRequest.of(identity) : null return v2path(type, registry, image, reference, data) } else diff --git a/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy b/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy deleted file mode 100644 index 2a6020947..000000000 --- a/src/main/groovy/io/seqera/wave/service/ContainerRequestData.groovy +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service - -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.model.ContainerCoordinates -import io.seqera.wave.tower.PlatformId -import static io.seqera.wave.util.StringUtils.trunc -/** - * Model a container request - * - * @author Paolo Di Tommaso - */ -@Canonical -@CompileStatic -class ContainerRequestData { - - final PlatformId identity - final String containerImage - final String containerFile - final ContainerConfig containerConfig - final String condaFile - final ContainerPlatform platform - final String buildId - final Boolean buildNew - final Boolean freeze - final Boolean mirror - - boolean durable() { - return freeze || mirror - } - - PlatformId getIdentity() { - return identity - } - - ContainerCoordinates coordinates() { ContainerCoordinates.parse(containerImage) } - - @Override - String toString() { - return "ContainerRequestData[identity=${getIdentity()}; containerImage=$containerImage; containerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; containerConfig=${containerConfig}]" - } - -} diff --git a/src/main/groovy/io/seqera/wave/service/blob/BlobStoreImpl.groovy b/src/main/groovy/io/seqera/wave/service/blob/BlobStoreImpl.groovy index 583f6ee5b..b21ab9b14 100644 --- a/src/main/groovy/io/seqera/wave/service/blob/BlobStoreImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/blob/BlobStoreImpl.groovy @@ -48,7 +48,7 @@ class BlobStoreImpl extends AbstractStateStore implements BlobStateSt @Override protected String getPrefix() { - return 'wave-blobcache/v1:' + return 'wave-blobcache/v1' } @Override diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildCounterStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildCounterStore.groovy index e21e6bb25..0b5d5c609 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildCounterStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildCounterStore.groovy @@ -28,6 +28,7 @@ import jakarta.inject.Singleton * * @author Paolo Di Tommaso */ +@Deprecated @Singleton @CompileStatic class BuildCounterStore extends AbstractCounterStore { diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildEntry.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildEntry.groovy index 7867b9db8..030a45cfb 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildEntry.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildEntry.groovy @@ -38,6 +38,14 @@ class BuildEntry implements StateEntry, JobEntry, RequestIdAware { final BuildResult result + BuildRequest getRequest() { + return request + } + + BuildResult getResult() { + return result + } + @Override String getKey() { return request.targetImage @@ -45,8 +53,6 @@ class BuildEntry implements StateEntry, JobEntry, RequestIdAware { @Override String getRequestId() { - if( !request.buildId ) - throw new IllegalStateException("Missing build id") return request.buildId } @@ -66,4 +72,7 @@ class BuildEntry implements StateEntry, JobEntry, RequestIdAware { new BuildEntry(request, result) } + static BuildEntry create(BuildRequest request) { + new BuildEntry(request, BuildResult.create(request)) + } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 9682fb536..4798b24b3 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -43,6 +43,8 @@ class BuildRequest { static final public String SEP = '_' + static final public String ID_PREFIX = "bd-" + /** * Unique request Id. This is computed as a consistent hash generated from * the container build assets e.g. Dockerfile. Therefore the same container build @@ -129,27 +131,29 @@ class BuildRequest { * Max allow time duration for this build */ final Duration maxDuration - - volatile String buildId - - volatile Path workDir - - BuildRequest(String containerId, - String containerFile, - String condaFile, - Path workspace, - String targetImage, - PlatformId identity, - ContainerPlatform platform, - String cacheRepository, - String ip, - String configJson, - String offsetId, - ContainerConfig containerConfig, - String scanId, - BuildContext buildContext, - BuildFormat format, - Duration maxDuration + + /** + * The build unique request id + */ + final String buildId + + BuildRequest( + String containerId, + String containerFile, + String condaFile, + Path workspace, + String targetImage, + PlatformId identity, + ContainerPlatform platform, + String cacheRepository, + String ip, + String configJson, + String offsetId, + ContainerConfig containerConfig, + String scanId, + BuildContext buildContext, + BuildFormat format, + Duration maxDuration ) { this.containerId = containerId @@ -169,6 +173,8 @@ class BuildRequest { this.buildContext = buildContext this.format = format this.maxDuration = maxDuration + // NOTE: this is meant to be updated - automatically - when the request is submitted + this.buildId = ID_PREFIX + containerId + SEP + '0' } BuildRequest(Map opts) { @@ -188,9 +194,8 @@ class BuildRequest { this.scanId = opts.scanId this.buildContext = opts.buildContext as BuildContext this.format = opts.format as BuildFormat - this.workDir = opts.workDir as Path - this.buildId = opts.buildId this.maxDuration = opts.maxDuration as Duration + this.buildId = opts.buildId } @Override @@ -216,7 +221,7 @@ class BuildRequest { } Path getWorkDir() { - return workDir + return workspace.resolve(buildId).toAbsolutePath() } String getTargetImage() { @@ -263,12 +268,6 @@ class BuildRequest { format==SINGULARITY } - BuildRequest withBuildId(String id) { - this.buildId = containerId + SEP + id - this.workDir = workspace.resolve(buildId).toAbsolutePath() - return this - } - static String legacyBuildId(String id) { if( !id ) return null diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildResult.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildResult.groovy index 03e5daf02..fbbf4f5a2 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildResult.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildResult.groovy @@ -38,15 +38,18 @@ import groovy.transform.ToString @CompileStatic class BuildResult { - final String id + final String buildId final Integer exitStatus final String logs final Instant startTime final Duration duration final String digest - BuildResult(String id, Integer exitStatus, String logs, Instant startTime, Duration duration, String digest) { - this.id = id + // This field is deprecated. It's only used for de-serializing object objects + @Deprecated final private String id + + BuildResult(String buildId, Integer exitStatus, String logs, Instant startTime, Duration duration, String digest) { + this.buildId = buildId this.logs = logs?.replaceAll("\u001B\\[[;\\d]*m", "") // strip ansi escape codes this.exitStatus = exitStatus this.startTime = startTime @@ -57,7 +60,7 @@ class BuildResult { /* Do not remove - required by jackson de-ser */ protected BuildResult() {} - String getId() { id } + String getBuildId() { buildId ?: id } Duration getDuration() { duration } @@ -75,15 +78,15 @@ class BuildResult { @Override String toString() { - return "BuildResult[id=$id; exitStatus=$exitStatus; duration=$duration]" + return "BuildResult[buildId=$buildId; exitStatus=$exitStatus; duration=$duration]" } - static BuildResult completed(String buildId, Integer exitStatus, String content, Instant startTime, String digest) { - new BuildResult(buildId, exitStatus, content, startTime, Duration.between(startTime, Instant.now()), digest) + static BuildResult completed(String buildId, Integer exitStatus, String logs, Instant startTime, String digest) { + new BuildResult(buildId, exitStatus, logs, startTime, Duration.between(startTime, Instant.now()), digest) } - static BuildResult failed(String buildId, String content, Instant startTime) { - new BuildResult(buildId, -1, content, startTime, Duration.between(startTime, Instant.now()), null) + static BuildResult failed(String buildId, String logs, Instant startTime) { + new BuildResult(buildId, -1, logs, startTime, Duration.between(startTime, Instant.now()), null) } static BuildResult create(BuildRequest req) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStateStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStateStore.groovy index d36920bd6..b09811183 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStateStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStateStore.groovy @@ -22,6 +22,8 @@ import java.time.Duration import java.util.concurrent.CompletableFuture import groovy.transform.CompileStatic +import io.seqera.wave.store.state.CountResult + /** * Define build request store operations * @@ -68,7 +70,7 @@ interface BuildStateStore { void storeBuild(String imageName, BuildEntry result, Duration ttl) /** - * Store a build result only if the specified key does not exit + * Store a {@link BuildEntry} object only if the specified key does not exit * * @param imageName The container image unique key * @param build The {@link BuildEntry} desired status to be stored @@ -76,6 +78,21 @@ interface BuildStateStore { */ boolean storeIfAbsent(String imageName, BuildEntry build) + /** + * Store a {@link BuildEntry} object only if the specified key does not exit. + * When the entry is stored the {@code buildId} fields in the request and response + * and incremented by 1. + * + * @param imageName + * The container image name used as store key. + * @param build + * A {@link BuildEntry} object to be stored + * @return + * A {@link CountResult} object holding the {@link BuildEntry} object with the updated {@code buildId} + * if the store is successful, or the currently store {@link BuildEntry} if the key already exist. + */ + CountResult putIfAbsentAndCount(String imageName, BuildEntry build) + /** * Remove the build status for the given image name * diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/BuildStateStoreImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/BuildStateStoreImpl.groovy index ffc3b3e88..4b692604e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/BuildStateStoreImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/BuildStateStoreImpl.groovy @@ -31,6 +31,8 @@ import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildStateStore import io.seqera.wave.store.state.AbstractStateStore +import io.seqera.wave.store.state.CountParams +import io.seqera.wave.store.state.CountResult import io.seqera.wave.store.state.impl.StateProvider import jakarta.inject.Named import jakarta.inject.Singleton @@ -56,7 +58,7 @@ class BuildStateStoreImpl extends AbstractStateStore implements Buil @Override protected String getPrefix() { - return 'wave-build/v2:' + return 'wave-build/v2' } @Override @@ -89,6 +91,21 @@ class BuildStateStoreImpl extends AbstractStateStore implements Buil return putIfAbsent(imageName, build, buildConfig.statusDuration) } + @Override + protected CountParams counterKey(String key, BuildEntry build) { + new CountParams( "build-counters/v1", build.request.containerId) + } + + @Override + protected String counterScript() { + /string.gsub(value, '"buildId"%s*:%s*"(.-)(%d+)"', '"buildId":"%1' .. counter_value .. '"')/ + } + + @Override + CountResult putIfAbsentAndCount(String imageName, BuildEntry build) { + super.putIfAbsentAndCount(imageName, build) + } + @Override void removeBuild(String imageName) { remove(imageName) diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy index acc2b0181..a08c8a764 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy @@ -40,11 +40,10 @@ import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.HttpServerRetryableErrorException import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService -import io.seqera.wave.service.builder.BuildCounterStore +import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildStateStore import io.seqera.wave.service.builder.BuildTrack import io.seqera.wave.service.builder.ContainerBuildService @@ -55,6 +54,7 @@ import io.seqera.wave.service.job.JobState import io.seqera.wave.service.metric.MetricsService import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable @@ -110,9 +110,6 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandlermax ) { dispatcher.notifyJobTimeout(jobSpec) + jobService.cleanup(jobSpec, null) return true } else { diff --git a/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy index c37d47881..acfd0a59c 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobQueue.groovy @@ -37,7 +37,7 @@ import jakarta.inject.Singleton @CompileStatic class JobQueue extends AbstractMessageStream { - final private static String STREAM_NAME = 'jobs-queue/v1:' + final private static String STREAM_NAME = 'jobs-queue/v1' private volatile JobConfig config diff --git a/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy b/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy index 993d173f2..40035397d 100644 --- a/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/impl/DockerJobOperation.groovy @@ -42,7 +42,7 @@ class DockerJobOperation implements JobOperation { @Override JobState status(JobSpec jobSpec) { final state = getDockerContainerState(jobSpec.operationName) - log.trace "Docker container status name=$jobSpec.operationName; state=$state" + log.trace "Docker container status name=${jobSpec.operationName}; state=${state}" if (state.status == 'running') { return JobState.running() @@ -55,6 +55,7 @@ class DockerJobOperation implements JobOperation { return JobState.pending() } else { + log.warn "Unexpected state for container state=${state}" final logs = getDockerContainerLogs(jobSpec.operationName) return JobState.unknown(logs) } diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogService.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogService.groovy index deb042952..1e0f1bdd8 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogService.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogService.groovy @@ -39,4 +39,8 @@ interface BuildLogService { StreamedFile fetchLogStream(String buildId) BuildLog fetchLogString(String buildId) + + String fetchCondaLockString(String buildId) + + StreamedFile fetchCondaLockStream(String buildId) } diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index 896330267..c7abfd3ce 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -41,6 +41,7 @@ import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton import org.apache.commons.io.input.BoundedInputStream +import static org.apache.commons.lang3.StringUtils.strip /** * Implements Service to manage logs from an Object store * @@ -52,6 +53,10 @@ import org.apache.commons.io.input.BoundedInputStream @Requires(property = 'wave.build.logs.bucket') class BuildLogServiceImpl implements BuildLogService { + private static final String CONDA_LOCK_START = ">> CONDA_LOCK_START" + + private static final String CONDA_LOCK_END = "<< CONDA_LOCK_END" + @Inject @Named('build-logs') private ObjectStorageOperations objectStorageOperations @@ -69,13 +74,17 @@ class BuildLogServiceImpl implements BuildLogService { @Value('${wave.build.logs.maxLength:100000}') private long maxLength + @Nullable + @Value('${wave.build.logs.conda-lock-prefix}') + private String condaLockPrefix + @Inject @Named(TaskExecutors.IO) private volatile ExecutorService ioExecutor @PostConstruct private void init() { - log.info "Creating Build log service bucket=$bucket; prefix=$prefix; maxLength: ${maxLength}" + log.info "Creating Build log service bucket=$bucket; logs prefix=$prefix; maxLength: ${maxLength}; condaLock prefix=$condaLockPrefix" } protected String logKey(String buildId) { @@ -83,23 +92,27 @@ class BuildLogServiceImpl implements BuildLogService { return null if( !prefix ) return buildId + '.log' - final base = org.apache.commons.lang3.StringUtils.strip(prefix, '/') + final base = strip(prefix, '/') return "${base}/${buildId}.log" } @EventListener void onBuildEvent(BuildEvent event) { if(event.result.logs) { - CompletableFuture.supplyAsync(() -> storeLog(event.result.id, event.result.logs), ioExecutor) + CompletableFuture.supplyAsync(() -> storeLog(event.result.buildId, event.result.logs), ioExecutor) } } @Override void storeLog(String buildId, String content){ + try { + final String logs = removeCondaLockFile(content) log.debug "Storing logs for buildId: $buildId" - final uploadRequest = UploadRequest.fromBytes(content.getBytes(), logKey(buildId)) + final uploadRequest = UploadRequest.fromBytes(logs.bytes, logKey(buildId)) objectStorageOperations.upload(uploadRequest) + + storeCondaLock(buildId, content) } catch (Exception e) { log.warn "Unable to store logs for buildId: $buildId - reason: ${e.message}", e @@ -125,4 +138,67 @@ class BuildLogServiceImpl implements BuildLogService { final logs = new BoundedInputStream(result.getInputStream(), maxLength).getText() return new BuildLog(logs, logs.length()>=maxLength) } + + protected static removeCondaLockFile(String logs) { + if(logs.indexOf(CONDA_LOCK_START) < 0 ) { + return logs + } + return logs.replaceAll(/(?s)\n?#\d+ \d+\.\d+ $CONDA_LOCK_START.*?$CONDA_LOCK_END\n?/, '\n') + } + + protected void storeCondaLock(String buildId, String logs) { + if( !logs ) return + try { + String condaLock = extractCondaLockFile(logs) + if (condaLock){ + log.debug "Storing conda lock for buildId: $buildId" + final uploadRequest = UploadRequest.fromBytes(condaLock.bytes, condaLockKey(buildId)) + objectStorageOperations.upload(uploadRequest) + } + } + catch (Exception e) { + log.warn "Unable to store condalock for buildId: $buildId - reason: ${e.message}", e + } + } + + protected String condaLockKey(String buildId) { + if( !buildId ) + return null + if( !condaLockPrefix ) + return buildId + '.lock' + final base = strip(condaLockPrefix, '/') + return "${base}/${buildId}.lock" + } + + @Override + String fetchCondaLockString(String buildId) { + final result = fetchCondaLockStream(buildId) + if( !result ) + return null + return result.getInputStream().getText() + + } + + @Override + StreamedFile fetchCondaLockStream(String buildId) { + if( !buildId ) return null + final Optional> result = objectStorageOperations.retrieve(condaLockKey(buildId)) + return result.isPresent() ? result.get().toStreamedFile() : null + } + + protected static extractCondaLockFile(String logs) { + try { + int start = logs.lastIndexOf(CONDA_LOCK_START) + int end = logs.lastIndexOf(CONDA_LOCK_END) + if( start > end ) { // when build fails, there will be commands in the logs, so to avoid extracting wrong content + return null + } + return logs.substring(start + CONDA_LOCK_START.length(), end) + .replaceAll(/#\d+ \d+\.\d+\s*/, '') + } catch (Exception e) { + log.warn "Unable to extract conda lock file from logs - reason: ${e.message}", e + return null + } + } + } diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index 4eea0de03..537d83781 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -78,7 +78,7 @@ class MailServiceImpl implements MailService { spooler.sendMail(mail) } else { - log.debug "Missing email recipient from build id=$build.id - user=$user" + log.debug "Missing email recipient from build id=$build.buildId - user=$user" } } @@ -86,7 +86,7 @@ class MailServiceImpl implements MailService { // create template binding final binding = new HashMap(20) final status = result.succeeded() ? 'DONE': 'FAILED' - binding.build_id = result.id + binding.build_id = result.buildId binding.build_user = "${req.identity?.user ? req.identity.user.userName : '-'} (${req.ip})" binding.build_success = result.succeeded() binding.build_exit_status = result.exitStatus @@ -98,8 +98,7 @@ class MailServiceImpl implements MailService { binding.build_containerfile = req.containerFile ?: '-' binding.build_condafile = req.condaFile binding.build_digest = result.digest ?: '-' - binding.put('build_log_data', result.logs) - binding.build_url = "$serverUrl/view/builds/${result.id}" + binding.build_url = "$serverUrl/view/builds/${result.buildId}" binding.scan_url = req.scanId && result.succeeded() ? "$serverUrl/view/scans/${req.scanId}" : null binding.scan_id = req.scanId binding.put('server_url', serverUrl) diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy index 93cc71bc0..175f2252f 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorService.groovy @@ -21,7 +21,6 @@ package io.seqera.wave.service.mirror import java.util.concurrent.CompletableFuture import io.seqera.wave.service.builder.BuildTrack - /** * Define the contract for container images mirroring service * @@ -58,6 +57,6 @@ interface ContainerMirrorService { * The {@link MirrorEntry} object modelling the current state of the mirror operation, * or {@link null} otherwise */ - MirrorEntry getMirrorEntry(String id) + MirrorResult getMirrorResult(String id) } diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy index 19944a71c..aadd9ff0d 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy @@ -23,6 +23,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.micronaut.core.annotation.Nullable import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.builder.BuildTrack import io.seqera.wave.service.job.JobHandler @@ -30,10 +31,10 @@ import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.scan.ContainerScanService import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton - /** * Implement a service to mirror a container image to a repository specified by the user * @@ -58,12 +59,16 @@ class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler */ @ToString(includeNames = true, includePackage = false) -@Singleton +@EqualsAndHashCode @CompileStatic -@Canonical class MirrorEntry implements StateEntry, JobEntry, RequestIdAware { - enum Status { PENDING, COMPLETED } - final String mirrorId - final String digest - final String sourceImage - final String targetImage - final ContainerPlatform platform - final Instant creationTime - final Status status - final Duration duration - final Integer exitCode - final String logs + final MirrorRequest request + + final MirrorResult result + + protected MirrorEntry() {} + + MirrorEntry(MirrorRequest request, MirrorResult result) { + this.request = request + this.result = result + } @Override String getKey() { - return targetImage + return request.targetImage } @Override String getRequestId() { - return mirrorId + return request.mirrorId } @Override boolean done() { - status==Status.COMPLETED + result?.status==MirrorResult.Status.COMPLETED } - boolean succeeded() { - status==Status.COMPLETED && exitCode==0 + /** + * Create a {@link MirrorEntry} object with the current {@link MirrorRequest} and + * the specified {@link MirrorResult} object + * + * @param result The {@link MirrorResult} object to be use as result + * @return The new {@link MirrorEntry} instance + */ + MirrorEntry withResult(MirrorResult result) { + new MirrorEntry(this.request, result) } - MirrorEntry complete(Integer exitCode, String logs ) { - new MirrorEntry( - this.mirrorId, - this.digest, - this.sourceImage, - this.targetImage, - this.platform, - this.creationTime, - Status.COMPLETED, - Duration.between(this.creationTime, Instant.now()), - exitCode, - logs - ) + /** + * Create a {@link MirrorEntry} object with the given {@link MirrorRequest} object + */ + static MirrorEntry of(MirrorRequest request) { + new MirrorEntry(request, MirrorResult.of(request)) } - static MirrorEntry from(MirrorRequest request) { - new MirrorEntry( - request.mirrorId, - request.digest, - request.sourceImage, - request.targetImage, - request.platform, - request.creationTime, - Status.PENDING - ) - } - - BuildStatusResponse toStatusResponse() { - final status = status == Status.COMPLETED - ? BuildStatusResponse.Status.COMPLETED - : BuildStatusResponse.Status.PENDING - final succeeded = exitCode!=null - ? exitCode==0 - : null - return new BuildStatusResponse( - mirrorId, - status, - creationTime, - duration, - succeeded ) - } } diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy index 3abe05c44..bb541212c 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorRequest.groovy @@ -25,8 +25,8 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.LongRndKey - /** * Model a container mirror request * @@ -74,26 +74,89 @@ class MirrorRequest { */ final String authJson + /** + * The scanId associated this mirror request + */ + final String scanId + /** * The timestamp when the request has been submitted */ final Instant creationTime - static MirrorRequest create(String sourceImage, String targetImage, String digest, ContainerPlatform platform, Path workspace, String authJson, Instant ts=Instant.now()) { + /** + * Request timezone offset + */ + final String offsetId + + /** + * Platform identity of the user that created this request + */ + final PlatformId identity + + static MirrorRequest create(String sourceImage, String targetImage, String digest, ContainerPlatform platform, Path workspace, String authJson, String scanId, Instant ts, String offsetId, PlatformId identity) { assert sourceImage, "Argument 'sourceImage' cannot be null" assert targetImage, "Argument 'targetImage' cannot be empty" assert workspace, "Argument 'workspace' cannot be null" assert digest, "Argument 'digest' cannot be empty" - final id = LongRndKey.rndHex() + final mirrorId = ID_PREFIX + LongRndKey.rndHex() return new MirrorRequest( - ID_PREFIX + id, + mirrorId, sourceImage, targetImage, digest, platform, - workspace.resolve("mirror-${id}"), + workspace.resolve(mirrorId), authJson, - ts ) + scanId, + ts, + offsetId, + identity + ) + } + + String getMirrorId() { + return mirrorId + } + + String getSourceImage() { + return sourceImage + } + + String getTargetImage() { + return targetImage + } + + String getDigest() { + return digest + } + + ContainerPlatform getPlatform() { + return platform + } + + Path getWorkDir() { + return workDir + } + + String getAuthJson() { + return authJson + } + + String getScanId() { + return scanId + } + + Instant getCreationTime() { + return creationTime + } + + String getOffsetId() { + return offsetId + } + + PlatformId getIdentity() { + return identity } } diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorResult.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorResult.groovy new file mode 100644 index 000000000..4902ce581 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorResult.groovy @@ -0,0 +1,158 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.wave.core.ContainerPlatform +import jakarta.inject.Singleton +/** + * Model a container mirror entry object + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Singleton +@CompileStatic +@Canonical +class MirrorResult { + + enum Status { PENDING, COMPLETED } + + final String mirrorId + final String digest + final String sourceImage + final String targetImage + final ContainerPlatform platform + final Instant creationTime + final String offsetId + final String userName + final String userEmail + final Long userId + final String scanId + final Status status + final Duration duration + final Integer exitCode + final String logs + + boolean succeeded() { + status==Status.COMPLETED && exitCode==0 + } + + MirrorResult complete(Integer exitCode, String logs) { + new MirrorResult( + this.mirrorId, + this.digest, + this.sourceImage, + this.targetImage, + this.platform, + this.creationTime, + this.offsetId, + this.userName, + this.userEmail, + this.userId, + this.scanId, + Status.COMPLETED, + Duration.between(this.creationTime, Instant.now()), + exitCode, + logs + ) + } + + static MirrorResult of(MirrorRequest request) { + new MirrorResult( + request.mirrorId, + request.digest, + request.sourceImage, + request.targetImage, + request.platform, + request.creationTime, + request.offsetId, + request.identity?.user?.userName, + request.identity?.user?.email, + request.identity?.user?.id, + request.scanId, + Status.PENDING + ) + } + + String getMirrorId() { + return mirrorId + } + + String getDigest() { + return digest + } + + String getSourceImage() { + return sourceImage + } + + String getTargetImage() { + return targetImage + } + + ContainerPlatform getPlatform() { + return platform + } + + Instant getCreationTime() { + return creationTime + } + + String getOffsetId() { + return offsetId + } + + String getUserName() { + return userName + } + + String getUserEmail() { + return userEmail + } + + Long getUserId() { + return userId + } + + String getScanId() { + return scanId + } + + Status getStatus() { + return status + } + + Duration getDuration() { + return duration + } + + Integer getExitCode() { + return exitCode + } + + String getLogs() { + return logs + } +} diff --git a/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy index f518e3e27..31e61ba06 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/MirrorStateStore.groovy @@ -46,7 +46,7 @@ class MirrorStateStore extends AbstractStateStore { @Override protected String getPrefix() { - return 'wave-mirror/v1:' + return 'wave-mirror/v1' } @Override diff --git a/src/main/groovy/io/seqera/wave/service/pairing/PairingStore.groovy b/src/main/groovy/io/seqera/wave/service/pairing/PairingStore.groovy index 5132546f9..444c7b3b2 100644 --- a/src/main/groovy/io/seqera/wave/service/pairing/PairingStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/pairing/PairingStore.groovy @@ -61,7 +61,7 @@ class PairingStore extends AbstractStateStore { @Override protected String getPrefix() { - return 'pairing-keys/v1:' + return 'pairing-keys/v1' } /** diff --git a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy index 23dd2fcf9..c36a4c19f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/PersistenceService.groovy @@ -21,6 +21,8 @@ package io.seqera.wave.service.persistence import groovy.transform.CompileStatic import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult + /** * A storage for statistic data * @@ -88,19 +90,12 @@ interface PersistenceService { */ WaveContainerRecord loadContainerRequest(String token) - /** - * Create a scan record, this signal that a container scan request has been created - * - * @param scanRecord Create a record with the object specified - */ - void createScanRecord(WaveScanRecord scanRecord) - /** * Store a {@link WaveScanRecord} object in the Surreal wave_scan table. * * @param data A {@link WaveScanRecord} object representing the a container scan request */ - void updateScanRecord(WaveScanRecord scanRecord) + void saveScanRecord(WaveScanRecord scanRecord) /** * Retrieve a {@link WaveScanRecord} object for the specified build ID @@ -116,7 +111,7 @@ interface PersistenceService { * @param mirrorId The ID of the mirror record * @return The corresponding {@link MirrorEntry} object or null if it cannot be found */ - MirrorEntry loadMirrorEntry(String mirrorId) + MirrorResult loadMirrorResult(String mirrorId) /** * Load a mirror state record given the target image name and the image digest @@ -125,13 +120,13 @@ interface PersistenceService { * @param digest The image content SHA256 digest * @return The corresponding {@link MirrorEntry} object or null if it cannot be found */ - MirrorEntry loadMirrorEntry(String targetImage, String digest) + MirrorResult loadMirrorResult(String targetImage, String digest) /** * Persists a {@link MirrorEntry} state record * * @param mirror {@link MirrorEntry} object */ - void saveMirrorEntry(MirrorEntry mirror) + void saveMirrorResult(MirrorResult mirror) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index b4278af2b..299a6f2bd 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -24,12 +24,12 @@ import java.time.Instant import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import io.seqera.wave.api.BuildStatusResponse import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.api.BuildStatusResponse /** * A collection of request and response properties to be stored * @@ -71,7 +71,7 @@ class WaveBuildRecord { } static private WaveBuildRecord create0(BuildRequest request, BuildResult result) { - if( result && request.buildId != result.id ) + if( result && request.buildId != result.buildId ) throw new IllegalStateException("Build id must match the result id") return new WaveBuildRecord( buildId: request.buildId, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index 67a402ac7..a367c585c 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -27,7 +27,7 @@ import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.User import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime /** @@ -163,8 +163,18 @@ class WaveContainerRecord { */ final String fusionVersion - WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String token, String waveImage, String addr, Instant expiration) { - this.id = token + /** + * Whenever it's a "mirror" build request + */ + final Boolean mirror + + /** + * The scan id associated with this request + */ + final String scanId + + WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequest data, String waveImage, String addr, Instant expiration) { + this.id = data.requestId this.user = data.identity.user this.workspaceId = request.towerWorkspaceId this.containerImage = request.containerImage @@ -187,6 +197,8 @@ class WaveContainerRecord { this.buildNew = data.buildId ? data.buildNew : null this.freeze = data.buildId ? data.freeze : null this.fusionVersion = request?.containerConfig?.fusionVersion()?.number + this.mirror = data.mirror + this.scanId = data.scanId } WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) { @@ -211,6 +223,8 @@ class WaveContainerRecord { this.buildNew = that.buildNew this.freeze = that.freeze this.fusionVersion = that.fusionVersion + this.mirror == that.mirror + this.scanId = that.scanId // -- digest part this.sourceDigest = sourceDigest this.waveDigest = waveDigest diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index 65b5fa2c4..ea9d81f97 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -37,29 +37,41 @@ import io.seqera.wave.util.StringUtils @ToString(includeNames = true, includePackage = false) @EqualsAndHashCode @CompileStatic -class WaveScanRecord { +class WaveScanRecord implements Cloneable { String id String buildId + String mirrorId + String requestId String containerImage Instant startTime Duration duration String status List vulnerabilities + Integer exitCode + String logs /* required by jackson deserialization - do not remove */ WaveScanRecord() {} - WaveScanRecord(String id, String buildId, String containerImage, Instant startTime) { - this.id = StringUtils.surrealId(id) - this.buildId = buildId - this.containerImage = containerImage - this.startTime = startTime - } - - WaveScanRecord(String id, String buildId, String containerImage, Instant startTime, Duration duration, String status, List vulnerabilities) { + WaveScanRecord( + String id, + String buildId, + String mirrorId, + String requestId, + String containerImage, + Instant startTime, + Duration duration, + String status, + List vulnerabilities, + Integer exitCode, + String logs + ) + { this.id = StringUtils.surrealId(id) this.buildId = buildId + this.mirrorId = mirrorId + this.requestId = requestId this.containerImage = containerImage this.startTime = startTime this.duration = duration @@ -67,21 +79,52 @@ class WaveScanRecord { this.vulnerabilities = vulnerabilities ? new ArrayList(vulnerabilities) : List.of() + this.exitCode = exitCode + this.logs = sanitize0(logs) } - WaveScanRecord(String id, ScanEntry scanResult) { - this.id = StringUtils.surrealId(id) - this.buildId = scanResult.buildId - this.containerImage = scanResult.containerImage - this.startTime = scanResult.startTime - this.duration = scanResult.duration - this.status = scanResult.status - this.vulnerabilities = scanResult.vulnerabilities - ? new ArrayList(scanResult.vulnerabilities) + WaveScanRecord(ScanEntry scan) { + this.id = StringUtils.surrealId(scan.scanId) + this.buildId = scan.buildId + this.mirrorId = scan.mirrorId + this.requestId = scan.requestId + this.containerImage = scan.containerImage + this.startTime = scan.startTime + this.duration = scan.duration + this.status = scan.status + this.vulnerabilities = scan.vulnerabilities + ? new ArrayList(scan.vulnerabilities) : List.of() + this.exitCode = scan.exitCode + this.logs = sanitize0(scan.logs) + } + + private static String sanitize0(String str) { + if( !str ) + return null + // remove quotes that break sql statement + str = str.replaceAll(/'/,'') + if( str.size()>10_000 ) + str = str.substring(0,10_000) + ' [truncated]' + return str } void setId(String id) { this.id = StringUtils.surrealId(id) } + + Boolean succeeded() { + return duration != null + ? status == ScanEntry.SUCCEEDED + : null + } + + Boolean done() { + return duration != null + } + + @Override + WaveScanRecord clone() { + return (WaveScanRecord) super.clone() + } } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy index 109018f6c..d1c038d18 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/LocalPersistenceService.groovy @@ -20,7 +20,7 @@ package io.seqera.wave.service.persistence.impl import groovy.transform.CompileStatic import io.seqera.wave.core.ContainerDigestPair -import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord @@ -39,7 +39,7 @@ class LocalPersistenceService implements PersistenceService { private Map requestStore = new HashMap<>() private Map scanStore = new HashMap<>() - private Map mirrorStore = new HashMap<>() + private Map mirrorStore = new HashMap<>() @Override void saveBuild(WaveBuildRecord record) { @@ -83,13 +83,9 @@ class LocalPersistenceService implements PersistenceService { requestStore.get(token) } - @Override - void createScanRecord(WaveScanRecord scanRecord) { - scanStore.put(scanRecord.id, scanRecord) - } @Override - void updateScanRecord(WaveScanRecord scanRecord) { + void saveScanRecord(WaveScanRecord scanRecord) { scanStore.put(scanRecord.id, scanRecord) } @@ -98,15 +94,15 @@ class LocalPersistenceService implements PersistenceService { scanStore.get(scanId) } - MirrorEntry loadMirrorEntry(String mirrorId) { + MirrorResult loadMirrorResult(String mirrorId) { mirrorStore.get(mirrorId) } - MirrorEntry loadMirrorEntry(String targetImage, String digest) { - mirrorStore.values().find( (MirrorEntry mirror) -> mirror.targetImage==targetImage && mirror.digest==digest ) + MirrorResult loadMirrorResult(String targetImage, String digest) { + mirrorStore.values().find( (MirrorResult mirror) -> mirror.targetImage==targetImage && mirror.digest==digest ) } - void saveMirrorEntry(MirrorEntry mirror) { + void saveMirrorResult(MirrorResult mirror) { mirrorStore.put(mirror.mirrorId, mirror) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index 5e6ea24c4..badd7c4aa 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -26,7 +26,7 @@ import io.micronaut.http.annotation.Header import io.micronaut.http.annotation.Post import io.micronaut.http.client.annotation.Client import io.micronaut.retry.annotation.Retryable -import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ScanVulnerability @@ -79,5 +79,5 @@ interface SurrealClient { Map insertScanVulnerability(@Header String authorization, @Body ScanVulnerability scanVulnerability) @Post('/key/wave_mirror') - Flux> insertMirrorAsync(@Header String authorization, @Body MirrorEntry body) + Flux> insertMirrorAsync(@Header String authorization, @Body MirrorResult body) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 7c7167387..3ecfa5b51 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -31,6 +31,7 @@ import io.micronaut.runtime.event.annotation.EventListener import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord @@ -162,7 +163,7 @@ class SurrealPersistenceService implements PersistenceService { final query = """ select * from wave_build - where buildId ~ '${containerId}${BuildRequest.SEP}' + where buildId ~ '${containerId}${BuildRequest.SEP}' order by startTime desc limit 1 """.stripIndent() final json = surrealDb.sqlAsString(getAuthorization(), query) @@ -224,16 +225,11 @@ class SurrealPersistenceService implements PersistenceService { } static protected String patchSurrealId(String json, String table) { - json.replaceFirst(/"id":\s*"${table}:(\w*)"/) { List it-> /"id":"${it[1]}"/ } - } - - void createScanRecord(WaveScanRecord scanRecord) { - final result = surrealDb.insertScanRecord(authorization, scanRecord) - log.trace "Scan create result=$result" + return json.replaceFirst(/"id":\s*"${table}:(\w*)"/) { List it-> /"id":"${it[1]}"/ } } @Override - void updateScanRecord(WaveScanRecord scanRecord) { + void saveScanRecord(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() // save all vulnerabilities @@ -243,21 +239,25 @@ class SurrealPersistenceService implements PersistenceService { // compose the list of ids final ids = vulnerabilities - .collect(it-> "wave_scan_vuln:⟨$it.id⟩") - .join(', ') + .collect(it-> "wave_scan_vuln:⟨$it.id⟩".toString()) + + + // scan object + final copy = scanRecord.clone() + copy.vulnerabilities = List.of() + final json = JacksonHelper.toJson(copy) // create the scan record - final statement = """\ - UPDATE wave_scan:${scanRecord.id} - SET - status = '${scanRecord.status}', - duration = '${scanRecord.duration}', - vulnerabilities = ${ids ? "[$ids]" : "[]" } - """.stripIndent() + final statement = "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)}".toString() final result = surrealDb.sqlAsMap(authorization, statement) log.trace "Scan update result=$result" } + protected String patchScanVulnerabilities(String json, List ids) { + final value = "\"vulnerabilities\":${ids.collect(it-> "\"$it\"").toString()}" + json.replaceFirst(/"vulnerabilities":\s*\[]/, value) + } + @Override WaveScanRecord loadScanRecord(String scanId) { if( !scanId ) @@ -278,10 +278,10 @@ class SurrealPersistenceService implements PersistenceService { * @param mirrorId The ID of the mirror record * @return The corresponding {@link MirrorEntry} object or null if it cannot be found */ - MirrorEntry loadMirrorEntry(String mirrorId) { + MirrorResult loadMirrorResult(String mirrorId) { final query = "select * from wave_mirror where mirrorId = '$mirrorId'" final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} + final type = new TypeReference>>() {} final data= json ? JacksonHelper.fromJson(json, type) : null final result = data && data[0].result ? data[0].result[0] : null return result @@ -294,10 +294,10 @@ class SurrealPersistenceService implements PersistenceService { * @param digest The image content SHA256 digest * @return The corresponding {@link MirrorEntry} object or null if it cannot be found */ - MirrorEntry loadMirrorEntry(String targetImage, String digest) { + MirrorResult loadMirrorResult(String targetImage, String digest) { final query = "select * from wave_mirror where targetImage = '$targetImage' and digest = '$digest'" final json = surrealDb.sqlAsString(getAuthorization(), query) - final type = new TypeReference>>() {} + final type = new TypeReference>>() {} final data= json ? JacksonHelper.fromJson(json, type) : null final result = data && data[0].result ? data[0].result[0] : null return result @@ -309,7 +309,7 @@ class SurrealPersistenceService implements PersistenceService { * @param mirror {@link MirrorEntry} object */ @Override - void saveMirrorEntry(MirrorEntry mirror) { + void saveMirrorResult(MirrorResult mirror) { surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" }, {error-> diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy new file mode 100644 index 000000000..3e967469a --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy @@ -0,0 +1,198 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ScanLevel +import io.seqera.wave.api.ScanMode +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.util.LongRndKey +import static io.seqera.wave.util.StringUtils.trunc +/** + * Model a container request + * + * @author Paolo Di Tommaso + */ +@Canonical +@ToString(includePackage = false, includeNames = true) +@CompileStatic +class ContainerRequest { + + String requestId + PlatformId identity + String containerImage + String containerFile + ContainerConfig containerConfig + String condaFile + ContainerPlatform platform + String buildId + Boolean buildNew + Boolean freeze + Boolean mirror + String scanId + ScanMode scanMode + List scanLevels + Boolean scanOnRequest + Instant creationTime + + boolean durable() { + return freeze || mirror + } + + PlatformId getIdentity() { + return identity + } + + ContainerCoordinates coordinates() { ContainerCoordinates.parse(containerImage) } + + @Override + String toString() { + return "ContainerRequestData[requestId=${requestId}; identity=${getIdentity()}; containerImage=$containerImage; containerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; containerConfig=${containerConfig}; buildId=${buildId}; scanId=${scanId}; scanMode=${scanMode}]" + } + + String getRequestId() { + return requestId + } + + String getContainerImage() { + return containerImage + } + + String getContainerFile() { + return containerFile + } + + ContainerConfig getContainerConfig() { + return containerConfig + } + + String getCondaFile() { + return condaFile + } + + ContainerPlatform getPlatform() { + return platform + } + + String getBuildId() { + return buildId + } + + Boolean getBuildNew() { + return buildNew + } + + Boolean getFreeze() { + return freeze + } + + Boolean getMirror() { + return mirror + } + + String getScanId() { + return scanId + } + + ScanMode getScanMode() { + return scanMode + } + + List getScanLevels() { + return scanLevels + } + + Instant getCreationTime() { + return creationTime + } + + Boolean getScanOnRequest() { + scanOnRequest + } + + static ContainerRequest create( + PlatformId identity, + String containerImage, + String containerFile, + ContainerConfig containerConfig, + String condaFile, + ContainerPlatform platform, + String buildId, + Boolean buildNew, + Boolean freeze, + Boolean mirror, + String scanId, + ScanMode scanMode, + List scanLevels, + Boolean scanOnRequest, + Instant creationTime + ) + { + return new ContainerRequest( + LongRndKey.rndHex(), + identity, + containerImage, + containerFile, + containerConfig, + condaFile, + platform, + buildId, + buildNew, + freeze, + mirror, + scanId, + scanMode, + scanLevels, + scanOnRequest, + creationTime + ) + } + + static ContainerRequest of(PlatformId identity) { + new ContainerRequest((String)null, identity) + } + + static ContainerRequest of(Map data) { + new ContainerRequest( + data.requestId as String, + data.identity as PlatformId, + data.containerImage as String, + data.containerFile as String, + data.containerConfig as ContainerConfig, + data.condaFile as String, + data.platform as ContainerPlatform, + data.buildId as String, + (Boolean) data.buildNew, + (Boolean) data.freeze, + (Boolean) data.mirror, + data.scanId as String, + data.scanMode as ScanMode, + data.scanLevels as List, + data.scanOnRequest as Boolean, + data.creationTime as Instant + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequestService.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestService.groovy new file mode 100644 index 000000000..4bb854249 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestService.groovy @@ -0,0 +1,71 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import io.seqera.wave.service.persistence.WaveContainerRecord + +/** + * service to fulfill request for an augmented container + * + * @author Paolo Di Tommaso + */ +interface ContainerRequestService { + + /** + * Get (generate) a new container token for the specified container request data + * + * @param request + * An instance of {@link ContainerRequest} + * @return + * The {@link TokenData} representing the unique token and the expiration time + */ + TokenData computeToken(ContainerRequest request) + + /** + * Get the container image for the given container requestId + * + * @param requestId The + * container request unique id + * @return + * The {@link ContainerRequest} object for the specified id, + * or {@code null} if the requestId is unknown + */ + ContainerRequest getRequest(String requestId) + + /** + * Evict the container request entry from the cache for the given container request id + * + * @param requestId + * The id of the request to be evicted + * @return + * The corresponding token string, or null if the token is unknown + */ + ContainerRequest evictRequest(String requestId) + + /** + * Load the record persisted in the requests db + * + * @param requestId + * The unique id of the request to be loaded + * @return + * The {@link WaveContainerRecord} object corresponding the specified id or {@code null} otherwise + */ + WaveContainerRecord loadContainerRecord(String requestId) + +} diff --git a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestServiceImpl.groovy similarity index 59% rename from src/main/groovy/io/seqera/wave/service/token/ContainerTokenServiceImpl.groovy rename to src/main/groovy/io/seqera/wave/service/request/ContainerRequestServiceImpl.groovy index 109a9a948..315b0b1fc 100644 --- a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestServiceImpl.groovy @@ -16,15 +16,15 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.token +package io.seqera.wave.service.request import java.time.Instant import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.TokenConfig -import io.seqera.wave.service.ContainerRequestData -import io.seqera.wave.util.LongRndKey +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveContainerRecord import jakarta.inject.Inject import jakarta.inject.Singleton /** @@ -35,39 +35,45 @@ import jakarta.inject.Singleton @Slf4j @CompileStatic @Singleton -class ContainerTokenServiceImpl implements ContainerTokenService { +class ContainerRequestServiceImpl implements ContainerRequestService { @Inject - private ContainerTokenStore containerTokenStorage + private ContainerRequestStore containerTokenStorage @Inject private TokenConfig config @Inject - private ContainerTokenStoreImpl tokenCache + private ContainerRequestStoreImpl tokenCache + + @Inject + private PersistenceService persistenceService @Override - TokenData computeToken(ContainerRequestData request) { - final token = LongRndKey.rndHex() + TokenData computeToken(ContainerRequest request) { final expiration = Instant.now().plus(config.cache.duration); - containerTokenStorage.put(token, request) - return new TokenData(token, expiration) + containerTokenStorage.put(request.requestId, request) + return new TokenData(request.requestId, expiration) } @Override - ContainerRequestData getRequest(String token) { - return containerTokenStorage.get(token) + ContainerRequest getRequest(String requestId) { + return containerTokenStorage.get(requestId) } @Override - ContainerRequestData evictRequest(String token) { - if(!token) + ContainerRequest evictRequest(String requestId) { + if(!requestId) return null - final request = tokenCache.get(token) + final request = tokenCache.get(requestId) if( request ) { - tokenCache.remove(token) + tokenCache.remove(requestId) } return request } + + WaveContainerRecord loadContainerRecord(String requestId) { + persistenceService.loadContainerRequest(requestId) + } } diff --git a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenStore.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestStore.groovy similarity index 81% rename from src/main/groovy/io/seqera/wave/service/token/ContainerTokenStore.groovy rename to src/main/groovy/io/seqera/wave/service/request/ContainerRequestStore.groovy index b6da43faf..9fa670e31 100644 --- a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestStore.groovy @@ -16,21 +16,19 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.token - -import io.seqera.wave.service.ContainerRequestData - +package io.seqera.wave.service.request /** * Define the container request token persistence operations * * @author : jorge * */ -interface ContainerTokenStore { +interface ContainerRequestStore { - void put(String key, ContainerRequestData request) + void put(String key, ContainerRequest request) - ContainerRequestData get(String key) + ContainerRequest get(String key) void remove(String key) + } diff --git a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenStoreImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestStoreImpl.groovy similarity index 76% rename from src/main/groovy/io/seqera/wave/service/token/ContainerTokenStoreImpl.groovy rename to src/main/groovy/io/seqera/wave/service/request/ContainerRequestStoreImpl.groovy index 0ae47ddaf..a1d4cd97b 100644 --- a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenStoreImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequestStoreImpl.groovy @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.token +package io.seqera.wave.service.request import java.time.Duration @@ -24,31 +24,30 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.TokenConfig import io.seqera.wave.encoder.MoshiEncodeStrategy -import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.store.state.AbstractStateStore import io.seqera.wave.store.state.impl.StateProvider import jakarta.inject.Singleton /** - * Implements a cache store for {@link ContainerRequestData} + * Implements a cache store for {@link ContainerRequest} * * @author Paolo Di Tommaso */ @Slf4j @Singleton @CompileStatic -class ContainerTokenStoreImpl extends AbstractStateStore implements ContainerTokenStore { +class ContainerRequestStoreImpl extends AbstractStateStore implements ContainerRequestStore { private TokenConfig tokenConfig - ContainerTokenStoreImpl(StateProvider delegate, TokenConfig tokenConfig) { - super(delegate, new MoshiEncodeStrategy(){}) + ContainerRequestStoreImpl(StateProvider delegate, TokenConfig tokenConfig) { + super(delegate, new MoshiEncodeStrategy(){}) this.tokenConfig = tokenConfig log.info "Creating Tokens cache store ― duration=${tokenConfig.cache.duration}" } @Override protected String getPrefix() { - return 'wave-tokens/v1:' + return 'wave-tokens/v1' } @Override @@ -57,12 +56,12 @@ class ContainerTokenStoreImpl extends AbstractStateStore i } @Override - ContainerRequestData get(String key) { - return super.get(key) + ContainerRequest get(String key) { + return (ContainerRequest) super.get(key) } @Override - void put(String key, ContainerRequestData value) { + void put(String key, ContainerRequest value) { super.put(key, value) } @@ -70,4 +69,5 @@ class ContainerTokenStoreImpl extends AbstractStateStore i void remove(String key) { super.remove(key) } + } diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerState.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerState.groovy new file mode 100644 index 000000000..dca46c989 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerState.groovy @@ -0,0 +1,83 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.wave.service.builder.BuildEntry +import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorResult +import io.seqera.wave.service.persistence.WaveBuildRecord +/** + * Model the state of container request + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Canonical +@CompileStatic +class ContainerState { + + final Instant startTime + final Duration duration + final Boolean succeeded + + boolean isRunning() { + return duration==null + } + + static ContainerState from(BuildEntry build) { + assert build + return new ContainerState( + build.request.startTime, + build.result?.duration, + build.result?.succeeded() + ) + } + + static ContainerState from(WaveBuildRecord build) { + assert build + return new ContainerState( + build.startTime, + build.duration, + build.succeeded() + ) + } + + static ContainerState from(MirrorEntry mirror) { + assert mirror + return new ContainerState( + mirror.request.creationTime, + mirror.result?.duration, + mirror.result?.succeeded() + ) + } + + static ContainerState from(MirrorResult mirror) { + return new ContainerState( + mirror.creationTime, + mirror.duration, + mirror.succeeded() + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusService.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusService.groovy new file mode 100644 index 000000000..a45f9a71b --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusService.groovy @@ -0,0 +1,32 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + + +import io.seqera.wave.api.ContainerStatusResponse +/** + * Define the contract for container request status service + * + * @author Paolo Di Tommaso + */ +interface ContainerStatusService { + + ContainerStatusResponse getContainerStatus(String requestId) + +} diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy new file mode 100644 index 000000000..5f5afcc00 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -0,0 +1,220 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import static io.seqera.wave.api.ContainerStatus.* + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import io.seqera.wave.api.ContainerStatus +import io.seqera.wave.api.ContainerStatusResponse +import io.seqera.wave.api.ScanMode +import io.seqera.wave.exception.NotFoundException +import io.seqera.wave.service.builder.ContainerBuildService +import io.seqera.wave.service.mirror.ContainerMirrorService +import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.service.scan.ScanEntry +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * Implements the {@link ContainerStatusService} service + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class ContainerStatusServiceImpl implements ContainerStatusService { + + @Canonical + static class StageResult { + boolean succeeded + String reason + String detailsUri + } + + @Inject + private ContainerBuildService buildService + + @Inject + private ContainerMirrorService mirrorService + + @Inject + @Nullable + private ContainerScanService scanService + + @Inject + private ContainerRequestStore requestStore + + @Inject + @Value('${wave.server.url}') + private String serverUrl + + @Override + ContainerStatusResponse getContainerStatus(String requestId) { + + final request = requestStore.get(requestId) + if( !request ) + return null + + final ContainerState state = getContainerState(request) + + if( state.running ) { + return createResponse0(BUILDING, request, state) + } + else if( !request.scanId ) { + return createResponse0(DONE, request, state, buildResult(request,state)) + } + + if( request.scanId && request.scanMode == ScanMode.required && scanService ) { + final scan = scanService.getScanState(request.scanId) + if ( !scan ) + throw new NotFoundException("Missing container scan record with id: ${request.scanId}") + if ( !scan.duration ) { + return createResponse0(SCANNING, request, new ContainerState(state.startTime)) + } + else { + final newState = state + ? new ContainerState(state.startTime, state.duration+scan.duration, scan.succeeded()) + : new ContainerState(scan.startTime, scan.duration, scan.succeeded()) + return createScanResponse(request, newState, scan) + } + } + + return createResponse0(DONE, request, state) + } + + protected ContainerState getContainerState(ContainerRequest request) { + if( request.mirror && request.buildId ) { + final mirror = mirrorService.getMirrorResult(request.buildId) + if (!mirror) + throw new NotFoundException("Missing container mirror record with id: ${request.buildId}") + return ContainerState.from(mirror) + } + if( request.buildId ) { + final build = buildService.getBuildRecord(request.buildId) + if (!build) + throw new NotFoundException("Missing container build record with id: ${request.buildId}") + return ContainerState.from(build) + } + else { + final delta = Duration.between(request.creationTime, Instant.now()) + return new ContainerState( request.creationTime, delta,true ) + } + } + + protected ContainerStatusResponse createResponse0(ContainerStatus status, ContainerRequest request, ContainerState state, StageResult result=null) { + new ContainerStatusResponse( + request.requestId, + status, + !request.mirror ? request.buildId : null, + request.mirror ? request.buildId : null, + request.scanId, + null, + state.succeeded, + result?.reason, + result?.detailsUri, + state.startTime, + state.duration, + ) + } + + protected ContainerStatusResponse createScanResponse(ContainerRequest request, ContainerState state, ScanEntry scan) { + + final result = scanResult(request, scan) + return new ContainerStatusResponse( + request.requestId, + DONE, + !request.mirror ? request.buildId : null, + request.mirror ? request.buildId : null, + request.scanId, + scan.summary(), + result.succeeded, + result.reason, + result.detailsUri, + state.startTime, + state.duration, + ) + } + + protected StageResult buildResult(ContainerRequest request, ContainerState state) { + if( state.succeeded ) + return new StageResult(true) + + if( request.mirror ) { + // when 'mirror' flag is true, then it should be interpreted as a mirror operation + return new StageResult(false, + "Container mirror did not complete successfully", + "${serverUrl}/view/mirrors/${request.buildId}" ) + } + else { + // plain build + return new StageResult(false, + "Container build did not complete successfully", + "${serverUrl}/view/builds/${request.buildId}" ) + } + } + + protected StageResult scanResult(ContainerRequest request, ScanEntry scan) { + // scan was not successful + if (!scan.succeeded()) { + return new StageResult( + false, + "Container security scan did not complete successfully", + "${serverUrl}/view/scans/${request.scanId}" + ) + } + + // scan job was successful, check the required levels are matched + final allowedLevels = request + .scanLevels + ?.collect(it -> it.toString().toUpperCase()) + ?: List.of() + final foundLevels = new HashSet(scan + .summary() + ?.keySet() + ?: Set.of()) + foundLevels.removeAll(allowedLevels) + if (foundLevels) { + return new StageResult( + false, + "Container security scan operation found one or more vulnerabilities with severity: ${foundLevels.join(',')}", + "${serverUrl}/view/scans/${request.scanId}" + ) + } + + if( scan.summary() ) { + return new StageResult( + true, + "Container security scan operation found one or more vulnerabilities that are compatible with requested security levels: ${allowedLevels.join(',')}", + "${serverUrl}/view/scans/${request.scanId}" + ) + } + + // all fine + return new StageResult(true) + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/token/TokenData.groovy b/src/main/groovy/io/seqera/wave/service/request/TokenData.groovy similarity index 96% rename from src/main/groovy/io/seqera/wave/service/token/TokenData.groovy rename to src/main/groovy/io/seqera/wave/service/request/TokenData.groovy index 81d74c086..8c621b908 100644 --- a/src/main/groovy/io/seqera/wave/service/token/TokenData.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/TokenData.groovy @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.token +package io.seqera.wave.service.request import java.time.Instant diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy index 7abd784f0..b992055e2 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy @@ -18,17 +18,32 @@ package io.seqera.wave.service.scan - +import io.seqera.wave.api.ScanMode +import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.mirror.MirrorEntry import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.request.ContainerRequest + /** * Declare operations to scan containers * * @author Munish Chouhan + * @author Paolo Di Tommaso */ interface ContainerScanService { + String getScanId(String targetImage, String digest, ScanMode mode, String format) + void scan(ScanRequest request) - WaveScanRecord getScanResult(String scanId) + void scanOnBuild(BuildEvent build) + + void scanOnMirror(MirrorEntry entry) + + void scanOnRequest(ContainerRequest request) + + WaveScanRecord getScanRecord(String scanId) + + ScanEntry getScanState(String scanId) } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 331bd53a9..b71e340ec 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -19,22 +19,28 @@ package io.seqera.wave.service.scan import java.nio.file.NoSuchFileException +import java.time.Instant import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.job.JobHandler import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.request.ContainerRequest import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton @@ -43,16 +49,17 @@ import static io.seqera.wave.service.builder.BuildFormat.DOCKER * Implements ContainerScanService * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @Slf4j @Named("Scan") -@Requires(property = 'wave.scan.enabled', value = 'true') +@Requires(bean = ScanConfig) @Singleton @CompileStatic class ContainerScanServiceImpl implements ContainerScanService, JobHandler { @Inject - private ScanConfig scanConfig + private ScanConfig config @Inject @Named(TaskExecutors.IO) @@ -67,12 +74,34 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler + * @author Paolo Di Tommaso */ @Slf4j @Singleton @@ -52,7 +53,7 @@ class DockerScanStrategy extends ScanStrategy { @Override void scanContainer(String jobName, ScanRequest req) { - log.info("Launching container scan for buildId: ${req.buildId} with scanId ${req.scanId}") + log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}") // create the scan dir try { diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index b97c8d681..8ed5ef25a 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -42,6 +42,7 @@ import static java.nio.file.StandardOpenOption.WRITE * Implements ScanStrategy for Kubernetes * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @Slf4j @Primary @@ -65,7 +66,7 @@ class KubeScanStrategy extends ScanStrategy { @Override void scanContainer(String jobName, ScanRequest req) { - log.info("Launching container scan for buildId: ${req.buildId} with scanId ${req.scanId}") + log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}") try{ // create the scan dir try { diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanEntry.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanEntry.groovy index 0f1952efd..b793a11c5 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanEntry.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanEntry.groovy @@ -31,6 +31,7 @@ import io.seqera.wave.store.state.StateEntry * Model for scan result * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @ToString(includePackage = false, includeNames = true) @Canonical @@ -41,15 +42,60 @@ class ScanEntry implements StateEntry, JobEntry { static final public String SUCCEEDED = 'SUCCEEDED' static final public String FAILED = 'FAILED' - final String scanId - final String buildId - final String containerImage - final Instant startTime - final Duration duration - final String status - final List vulnerabilities - final Integer exitCode - final String logs + /** + * The scan unique Id + */ + String scanId + + /** + * The build request that original this scan entry + */ + String buildId + + /** + * The container mirror request that original this scan entry + */ + String mirrorId + + /** + * The container request that original this scan entry + */ + String requestId + + /** + * The target container image to be scanner + */ + String containerImage + + /** + * The request creation time + */ + Instant startTime + + /** + * How long the scan operation required + */ + Duration duration + + /** + * The status of the scan operation + */ + String status + + /** + * The list of security vulnerabilities reported + */ + List vulnerabilities + + /** + * The scan job exit status + */ + Integer exitCode + + /** + * The scan job logs + */ + String logs @Override String getKey() { @@ -61,15 +107,30 @@ class ScanEntry implements StateEntry, JobEntry { return duration != null } - boolean isSucceeded() { status==SUCCEEDED } + boolean succeeded() { status==SUCCEEDED } @Deprecated - boolean isCompleted() { done() } + boolean completed() { done() } + + static ScanEntry create(ScanRequest request) { + return new ScanEntry( + request.scanId, + request.buildId, + request.mirrorId, + request.requestId, + request.targetImage, + request.creationTime, + null, + PENDING, + List.of()) + } ScanEntry success(List vulnerabilities){ return new ScanEntry( this.scanId, this.buildId, + this.mirrorId, + this.requestId, this.containerImage, this.startTime, Duration.between(this.startTime, Instant.now()), @@ -79,18 +140,59 @@ class ScanEntry implements StateEntry, JobEntry { } ScanEntry failure(Integer exitCode, String logs){ - return new ScanEntry(this.scanId, this.buildId, this.containerImage, this.startTime, Duration.between(this.startTime, Instant.now()), FAILED, List.of(), exitCode, logs) + return new ScanEntry( + this.scanId, + this.buildId, + this.mirrorId, + this.requestId, + this.containerImage, + this.startTime, + Duration.between(this.startTime, Instant.now()), + FAILED, + List.of(), + exitCode, + logs) } static ScanEntry failure(ScanRequest request){ - return new ScanEntry(request.scanId, request.buildId, request.targetImage, request.creationTime, Duration.between(request.creationTime, Instant.now()), FAILED, List.of()) + return new ScanEntry( + request.scanId, + request.buildId, + request.mirrorId, + request.requestId, + request.targetImage, + request.creationTime, + Duration.between(request.creationTime, Instant.now()), + FAILED, + List.of()) } - static ScanEntry pending(String scanId, String buildId, String containerImage) { - return new ScanEntry(scanId, buildId, containerImage, Instant.now(), null, PENDING, List.of()) + + Map summary() { + final result = new HashMap() + if( !vulnerabilities ) + return result + for( ScanVulnerability it : vulnerabilities ) { + def v = result.getOrDefault(it.severity, 0) + result.put(it.severity, v+1) + } + return result } - static ScanEntry create(String scanId, String buildId, String containerImage, Instant startTime, Duration duration1, String status, List vulnerabilities){ - return new ScanEntry(scanId, buildId, containerImage, startTime, duration1, status, vulnerabilities) + static ScanEntry of(Map opts){ + return new ScanEntry( + opts.scanId as String, + opts.buildId as String, + opts.mirrorId as String, + opts.requestId as String, + opts.containerImage as String, + opts.startTime as Instant, + opts.duration as Duration, + opts.status as String, + opts.vulnerabilities as List, + opts.exitCode as Integer, + opts.logs as String + ) } + } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanId.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanId.groovy new file mode 100644 index 000000000..4af315d6b --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanId.groovy @@ -0,0 +1,54 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import io.seqera.wave.util.RegHelper + +/** + * Model a scan operation id + * + * @author Paolo Di Tommaso + */ +@Canonical +@CompileStatic +class ScanId { + + final String base + final int count + + ScanId withCount(int count) { + new ScanId(this.base, count) + } + + String toString() { + "sc-${base}_${count}" + } + + static ScanId of(String containerImage, String digest=null){ + final data = new LinkedHashMap(2) + data.image = containerImage + if( digest ) + data.digest = digest + final x = RegHelper.sipHash(data) + return new ScanId(x) + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanIdStore.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanIdStore.groovy new file mode 100644 index 000000000..4c924e824 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanIdStore.groovy @@ -0,0 +1,56 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import java.time.Duration + +import groovy.transform.CompileStatic +import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.encoder.MoshiEncodeStrategy +import io.seqera.wave.store.state.AbstractStateStore +import io.seqera.wave.store.state.impl.StateProvider +import jakarta.inject.Inject +import jakarta.inject.Singleton + +/** + * Implements a store strategy for scan ids + * + * @author Paolo Di Tommaso + */ +@Singleton +@CompileStatic +class ScanIdStore extends AbstractStateStore { + + @Inject + private ScanConfig config + + ScanIdStore(StateProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + } + + @Override + protected String getPrefix() { + return 'wave-scanid/v1' + } + + @Override + protected Duration getDuration() { + return config.scanIdDuration + } +} diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 41564a4fd..c30454f1a 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -24,7 +24,6 @@ import java.time.Instant import groovy.transform.Canonical import groovy.transform.CompileStatic import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.builder.BuildRequest /** * Model a container scan request * @@ -33,17 +32,65 @@ import io.seqera.wave.service.builder.BuildRequest @Canonical @CompileStatic class ScanRequest { + + /** + * The scan unique id + */ final String scanId + + /** + * The container build that generated this scan operation, either a container, build or mirror request + */ final String buildId + + /** + * The container mirror that generated this scan operation, either a container, build or mirror request + */ + final String mirrorId + + /** + * The container request that generated this scan operation, either a container, build or mirror request + */ + final String requestId + + /** + * The docker config json required to authenticate this request + */ final String configJson + + /** + * The container image that needs to be scanned + */ final String targetImage + + /** + * The container platform to be used + */ final ContainerPlatform platform + + /** + * The scan job work directory + */ final Path workDir + + /** + * Scan request creation time + */ final Instant creationTime - static ScanRequest fromBuild(BuildRequest request) { - final id = request.scanId - final workDir = request.workDir.resolveSibling("scan-${id}") - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir, Instant.now()) + + static ScanRequest of(Map opts) { + new ScanRequest( + opts.scanId as String, + opts.buildId as String, + opts.mirrorId as String, + opts.requestId as String, + opts.configJson as String, + opts.targetImage as String, + opts.platform as ContainerPlatform, + opts.workDir as Path, + opts.creationTime as Instant + ) } + } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanStateStore.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanStateStore.groovy index d1d267d55..bdc24ad99 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanStateStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanStateStore.groovy @@ -44,7 +44,7 @@ class ScanStateStore extends AbstractStateStore { @Override protected String getPrefix() { - return 'wave-mirror/v1:' + return 'wave-scan/v1' } @Override diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy index e333141bc..432b9c49c 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy @@ -28,6 +28,7 @@ import io.seqera.wave.configuration.ScanConfig * Implements ScanStrategy for Docker * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @Slf4j @CompileStatic diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy index 16d50f96b..3dff74a96 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy @@ -18,17 +18,18 @@ package io.seqera.wave.service.scan + import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.util.StringUtils import org.jetbrains.annotations.NotNull - /** * Model for Scan Vulnerability * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @Slf4j @ToString(includeNames = true, includePackage = false) @@ -71,4 +72,5 @@ class ScanVulnerability implements Comparable { if( that.severity==null ) return 1 return ORDER[this.severity] <=> ORDER[that.severity] } + } diff --git a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy index 71ea0d720..a1055037b 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy @@ -27,6 +27,7 @@ import io.seqera.wave.exception.ScanRuntimeException * Implements ScanStrategy for Docker * * @author Munish Chouhan + * @author Paolo Di Tommaso */ @Slf4j class TrivyResultProcessor { diff --git a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenService.groovy b/src/main/groovy/io/seqera/wave/service/token/ContainerTokenService.groovy deleted file mode 100644 index f545c3908..000000000 --- a/src/main/groovy/io/seqera/wave/service/token/ContainerTokenService.groovy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.token - -import io.seqera.wave.service.ContainerRequestData - -/** - * service to fulfill request for an augmented container - * - * @author Paolo Di Tommaso - */ -interface ContainerTokenService { - - /** - * Get (generate) a new container token for the specified container request data - * - * @param request An instance of {@link io.seqera.wave.service.ContainerRequestData} - * @return A new token string that's used to track this request - */ - TokenData computeToken(ContainerRequestData request) - - /** - * Get the container image for the given container token - * - * @param token A container token string - * @return the corresponding token string, or null if the token is unknown - */ - ContainerRequestData getRequest(String token) - - /** - * Evict the container request entry from the cache for the given container token - * - * @param token A container token string - * @return the corresponding token string, or null if the token is unknown - */ - ContainerRequestData evictRequest(String token) - -} diff --git a/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy b/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy index a5542594f..3a65eba98 100644 --- a/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy +++ b/src/main/groovy/io/seqera/wave/storage/ManifestCacheStore.groovy @@ -51,7 +51,7 @@ class ManifestCacheStore extends AbstractStateStore implements Stor @Override protected String getPrefix() { - return "wave-blobs/v1:" + return "wave-blobs/v1" } @Override diff --git a/src/main/groovy/io/seqera/wave/store/state/AbstractStateStore.groovy b/src/main/groovy/io/seqera/wave/store/state/AbstractStateStore.groovy index 06b1acd4a..22bc90509 100644 --- a/src/main/groovy/io/seqera/wave/store/state/AbstractStateStore.groovy +++ b/src/main/groovy/io/seqera/wave/store/state/AbstractStateStore.groovy @@ -24,7 +24,7 @@ import groovy.transform.CompileStatic import io.seqera.wave.encoder.EncodingStrategy import io.seqera.wave.store.state.impl.StateProvider /** - * Implements a generic cache store + * Implements a generic store for ephemeral state data * * @author Paolo Di Tommaso */ @@ -44,10 +44,44 @@ abstract class AbstractStateStore implements StateStore { protected abstract Duration getDuration() - protected String key0(String k) { return getPrefix() + k } + protected String key0(String k) { return getPrefix() + ':' + k } protected String requestId0(String requestId) { - return getPrefix() + 'request-id/' + requestId + if( !requestId ) + throw new IllegalStateException("Argument 'requestId' cannot be null") + return getPrefix() + '/request-id:' + requestId + } + + /** + * Defines the counter for auto-increment operations. By default + * uses the entry "key". Subclasses can provide a custom logic to use a + * different counter key. + * + * @param key + * The entry for which the increment should be performed + * @param value + * The entry value for which the increment should be performed + * @return + * The counter key that by default is the entry key. + */ + protected CountParams counterKey(String key, V value) { + assert key, "Argument 'key' cannot be empty" + return new CountParams(getPrefix() + '/counter', key) + } + + /** + * Defines the Lua script that's applied to increment the entry counter. + * + * It assumes the entry is serialised as JSON object and it contains a {@code count} attribute + * that will be update with the store counter value. + * + * @return The Lua script used to increment the entry count. + */ + protected String counterScript() { + // NOTE: + // "value" is expected to be a Lua variable holding the JSON object + // "counter_value" is expected to be a Lua variable holding the new count value + /string.gsub(value, '"count"%s*:%s*(%d+)', '"count":' .. counter_value)/ } protected V deserialize(String encoded) { @@ -58,10 +92,6 @@ abstract class AbstractStateStore implements StateStore { return encodingStrategy.encode(value) } - protected String getRaw(String key) { - delegate.get(key) - } - @Override V get(String key) { final result = delegate.get(key0(key)) @@ -73,6 +103,7 @@ abstract class AbstractStateStore implements StateStore { return get(key) } + @Override void put(String key, V value) { put(key, value, getDuration()) } @@ -94,10 +125,30 @@ abstract class AbstractStateStore implements StateStore { return result } + @Override boolean putIfAbsent(String key, V value) { return putIfAbsent(key, value, getDuration()) } + CountResult putIfAbsentAndCount(String key, V value) { + putIfAbsentAndCount(key, value, getDuration()) + } + + CountResult putIfAbsentAndCount(String key, V value, Duration ttl) { + final result = delegate.putJsonIfAbsentAndIncreaseCount( + key0(key), + serialize(value), + ttl, + counterKey(key,value), + counterScript()) + // update the `value` with the result one + final updated = deserialize(result.value) + if( result && updated instanceof RequestIdAware ) { + delegate.put(requestId0(updated.getRequestId()), key, ttl) + } + return new CountResult( result.succeed, updated, result.count) + } + @Override void remove(String key) { delegate.remove(key0(key)) diff --git a/src/main/groovy/io/seqera/wave/store/state/CountParams.groovy b/src/main/groovy/io/seqera/wave/store/state/CountParams.groovy new file mode 100644 index 000000000..a3a8b2fb7 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/store/state/CountParams.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.store.state + +import groovy.transform.Canonical + +/** + * Model state store auto increment params + * + * @author Paolo Di Tommaso + */ +@Canonical +class CountParams { + final String key + final String field + + static CountParams of(String key) { + final p=key.lastIndexOf('/') + return p==-1 + ? new CountParams("counters/v1", key) + : new CountParams(key.substring(0,p), key.substring(p+1)) + } + +} diff --git a/src/main/groovy/io/seqera/wave/store/state/CountResult.groovy b/src/main/groovy/io/seqera/wave/store/state/CountResult.groovy new file mode 100644 index 000000000..3855ef6d7 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/store/state/CountResult.groovy @@ -0,0 +1,33 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.store.state + +import groovy.transform.Canonical + +/** + * Model the result object of state auto-increment operation + * + * @author Paolo Di Tommaso + */ +@Canonical +class CountResult { + final Boolean succeed + final V value + final Integer count +} diff --git a/src/main/groovy/io/seqera/wave/store/state/StateStore.groovy b/src/main/groovy/io/seqera/wave/store/state/StateStore.groovy index b80dd389b..642e37323 100644 --- a/src/main/groovy/io/seqera/wave/store/state/StateStore.groovy +++ b/src/main/groovy/io/seqera/wave/store/state/StateStore.groovy @@ -63,7 +63,8 @@ interface StateStore { boolean putIfAbsent(K key, V value) /** - * Store a value in the cache only if does not exist yet + * Store a value in the cache only if does not exist + * * @param key The unique associated with this object * @param value The object to store * @param ttl The max time-to-live of the stored entry diff --git a/src/main/groovy/io/seqera/wave/store/state/impl/LocalStateProvider.groovy b/src/main/groovy/io/seqera/wave/store/state/impl/LocalStateProvider.groovy index 540b7b3fe..2aaf450c2 100644 --- a/src/main/groovy/io/seqera/wave/store/state/impl/LocalStateProvider.groovy +++ b/src/main/groovy/io/seqera/wave/store/state/impl/LocalStateProvider.groovy @@ -21,10 +21,16 @@ package io.seqera.wave.store.state.impl import java.time.Duration import java.time.Instant import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic import io.micronaut.context.annotation.Requires +import io.seqera.wave.store.state.CountParams +import io.seqera.wave.store.state.CountResult import jakarta.inject.Singleton +import org.luaj.vm2.Globals +import org.luaj.vm2.LuaValue +import org.luaj.vm2.lib.jse.JsePlatform /** * Simple cache store implementation for development purpose @@ -54,6 +60,8 @@ class LocalStateProvider implements StateProvider { private Map> store = new ConcurrentHashMap<>() + private Map counters = new ConcurrentHashMap<>() + @Override String get(String key) { final entry = store.get(key) @@ -87,6 +95,28 @@ class LocalStateProvider implements StateProvider { return putIfAbsent0(key, value, ttl) == null } + @Override + synchronized CountResult putJsonIfAbsentAndIncreaseCount(String key, String json, Duration ttl, CountParams counterKey, String luaScript) { + final counter = counterKey.key + '/' + counterKey.field + final done = putIfAbsent0(key, json, ttl) == null + final addr = counters + .computeIfAbsent(counter, (it)-> new AtomicInteger()) + if( done ) { + final count = addr.incrementAndGet() + // apply the conversion + Globals globals = JsePlatform.standardGlobals() + globals.set('value', LuaValue.valueOf(json)) + globals.set('counter_value', LuaValue.valueOf(count)) + LuaValue chunk = globals.load("return $luaScript;"); + LuaValue result = chunk.call(); + // store the result + put(key, result.toString(), ttl) + return new CountResult(true, result.toString(), count) + } + else + return new CountResult(false, get(key), addr.get()) + } + private String putIfAbsent0(String key, String value, Duration ttl) { final entry = store.get(key) if( entry?.isExpired() ) diff --git a/src/main/groovy/io/seqera/wave/store/state/impl/RedisStateProvider.groovy b/src/main/groovy/io/seqera/wave/store/state/impl/RedisStateProvider.groovy index 8777ffa40..40d444fa2 100644 --- a/src/main/groovy/io/seqera/wave/store/state/impl/RedisStateProvider.groovy +++ b/src/main/groovy/io/seqera/wave/store/state/impl/RedisStateProvider.groovy @@ -22,6 +22,8 @@ import java.time.Duration import groovy.transform.CompileStatic import io.micronaut.context.annotation.Requires +import io.seqera.wave.store.state.CountParams +import io.seqera.wave.store.state.CountResult import jakarta.inject.Inject import jakarta.inject.Singleton import redis.clients.jedis.Jedis @@ -78,6 +80,46 @@ class RedisStateProvider implements StateProvider { } } + /* + * Set a value only the specified key does not exists, if the value can be set + * the counter identified by the key provided via 'KEYS[2]' is incremented by 1, + * + * If the key already exists return the current key value. + */ + static private String putAndIncrement(String luaScript) { + """ + local value = ARGV[1] + local ttl = ARGV[2] + local pattern = ARGV[3] + if redis.call('EXISTS', KEYS[1]) == 0 then + -- increment the counter + local counter_value = redis.call('HINCRBY', KEYS[2], KEYS[3], 1) + value = ${luaScript} + redis.call('SET', KEYS[1], value, 'PX', ttl) + -- return the updated value + return {1, value, counter_value} + else + return {0, redis.call('GET', KEYS[1]), redis.call('HGET', KEYS[2], KEYS[3])} + end + """ + } + + + CountResult putJsonIfAbsentAndIncreaseCount(String key, String json, Duration ttl, String counter, String luaScript) { + return putJsonIfAbsentAndIncreaseCount(key, json, ttl, CountParams.of(counter), luaScript) + } + + @Override + CountResult putJsonIfAbsentAndIncreaseCount(String key, String json, Duration ttl, CountParams counter, String mapping) { + try( Jedis jedis=pool.getResource() ) { + final result = jedis.eval(putAndIncrement(mapping), 3, key, counter.key, counter.field, json, ttl.toMillis().toString()) + return new CountResult<>( + (result as List)[0] == 1, + (result as List)[1] as String, + (result as List)[2] as Integer) + } + } + @Override void remove(String key) { try( Jedis conn=pool.getResource() ) { diff --git a/src/main/groovy/io/seqera/wave/store/state/impl/StateProvider.groovy b/src/main/groovy/io/seqera/wave/store/state/impl/StateProvider.groovy index cdb3e24bc..42186966c 100644 --- a/src/main/groovy/io/seqera/wave/store/state/impl/StateProvider.groovy +++ b/src/main/groovy/io/seqera/wave/store/state/impl/StateProvider.groovy @@ -18,12 +18,37 @@ package io.seqera.wave.store.state.impl -import io.seqera.wave.store.state.StateStore +import java.time.Duration +import io.seqera.wave.store.state.CountParams +import io.seqera.wave.store.state.CountResult +import io.seqera.wave.store.state.StateStore /** * Define an cache interface alias to be used by cache implementation providers * * @author Paolo Di Tommaso */ interface StateProvider extends StateStore { + + /** + * Store a value in the cache only if does not exist. If the operation is successful + * the counter identified by the key specified is incremented by 1 and the counter (new) + * value is returned as result the operation. + * + * @param key + * The unique associated with this object + * @param value + * A JSON payload to be stored. It attribute "count" is updated with the counter incremented value + * @param counterKey + * The counter unique key to be incremented + * @param ttl + * The max time-to-live of the stored entry + * @return + * A tuple with 3 elements with the following semantic: , where "result" is {@code true} + * when the value was actually updated or {@code false} otherwise. "value" represent the specified value when + * "return" is true or the value currently existing if the key already exist. Finally "count" is the value + * of the count after the increment operation. + */ + CountResult putJsonIfAbsentAndIncreaseCount(K key, V value, Duration ttl, CountParams counterKey, String luaScript) + } diff --git a/src/main/groovy/io/seqera/wave/tower/auth/JwtAuthStore.groovy b/src/main/groovy/io/seqera/wave/tower/auth/JwtAuthStore.groovy index 22fba58d1..bc952b81c 100644 --- a/src/main/groovy/io/seqera/wave/tower/auth/JwtAuthStore.groovy +++ b/src/main/groovy/io/seqera/wave/tower/auth/JwtAuthStore.groovy @@ -31,6 +31,7 @@ import jakarta.inject.Singleton /** * Implements storage for {@link JwtAuth} record * + * @author Paolo Di Tommaso */ @Slf4j @Singleton @@ -46,7 +47,7 @@ class JwtAuthStore extends AbstractStateStore { @Override protected String getPrefix() { - return "tower-jwt-store/v1:" + return "tower-jwt-store/v1" } /** @@ -92,7 +93,7 @@ class JwtAuthStore extends AbstractStateStore { final now = Instant.now() final entry = auth.withUpdatedAt(now) this.put(auth.key, entry) - log.debug "JWT updating refreshed record - $entry" + log.trace "JWT updating refreshed record - $entry" } boolean storeIfAbsent(JwtAuth auth) { diff --git a/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy b/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy index 0fee156f7..5ed82a503 100644 --- a/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy +++ b/src/main/groovy/io/seqera/wave/tower/auth/JwtMonitor.groovy @@ -98,7 +98,7 @@ class JwtMonitor implements Runnable { } // ignore record without an empty refresh field if( !entry.refresh ) { - log.info "JWT record refresh ignored - entry=$entry" + log.debug "JWT record refresh ignored - entry=$entry" return } // check that's a `createdAt` field (it may be missing in legacy records) @@ -113,11 +113,11 @@ class JwtMonitor implements Runnable { // check if the JWT record is expired final deadline = entry.createdAt + tokenConfig.cache.duration if( now > deadline ) { - log.info "JWT record expired - entry=$entry; deadline=$deadline; " + log.debug "JWT record expired - entry=$entry; deadline=$deadline; " return } - log.debug "JWT refresh request - entry=$entry; deadline=$deadline" + log.trace "JWT refresh request - entry=$entry; deadline=$deadline" towerClient.userInfo(entry.endpoint, entry) jwtTimeStore.setRefreshTimer(key) } diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index 7348d27ce..896427b3e 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.core.annotation.Nullable import io.seqera.wave.api.BuildContext +import io.seqera.wave.api.ContainerStatus import io.seqera.wave.api.ImageNameStrategy import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest @@ -29,9 +30,9 @@ import io.seqera.wave.api.SubmitContainerTokenResponse import io.seqera.wave.config.CondaOpts import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildFormat -import io.seqera.wave.service.token.TokenData +import io.seqera.wave.service.request.TokenData import org.yaml.snakeyaml.Yaml import static io.seqera.wave.service.builder.BuildFormat.SINGULARITY import static io.seqera.wave.util.DockerHelper.condaEnvironmentToCondaYaml @@ -130,19 +131,22 @@ class ContainerHelper { } } - static SubmitContainerTokenResponse makeResponseV1(ContainerRequestData data, TokenData token, String waveImage) { + static SubmitContainerTokenResponse makeResponseV1(ContainerRequest data, TokenData token, String waveImage) { final target = waveImage final build = data.buildNew ? data.buildId : null - return new SubmitContainerTokenResponse(token.value, target, token.expiration, data.containerImage, build, null, null, null) + return new SubmitContainerTokenResponse(data.requestId, token.value, target, token.expiration, data.containerImage, build, null, null, null, null, null) } - static SubmitContainerTokenResponse makeResponseV2(ContainerRequestData data, TokenData token, String waveImage) { + static SubmitContainerTokenResponse makeResponseV2(ContainerRequest data, TokenData token, String waveImage) { final target = data.durable() ? data.containerImage : waveImage final build = data.buildId final Boolean cached = !data.buildNew final expiration = !data.durable() ? token.expiration : null final tokenId = !data.durable() ? token.value : null - return new SubmitContainerTokenResponse(tokenId, target, expiration, data.containerImage, build, cached, data.freeze, data.mirror) + final status = data.buildNew || data.scanId + ? ContainerStatus.PENDING + : ContainerStatus.DONE + return new SubmitContainerTokenResponse(data.requestId, tokenId, target, expiration, data.containerImage, build, cached, data.freeze, data.mirror, data.scanId, status) } static String patchPlatformEndpoint(String endpoint) { diff --git a/src/main/resources/application-buildlogs-aws-test.yml b/src/main/resources/application-buildlogs-aws-test.yml index b147c2ea1..88d3c405a 100644 --- a/src/main/resources/application-buildlogs-aws-test.yml +++ b/src/main/resources/application-buildlogs-aws-test.yml @@ -3,4 +3,6 @@ wave: build: logs: bucket: "nextflow-ci" + prefix: 'wave-build/logs' + conda-lock-prefix: 'wave-build/conda-locks' ... diff --git a/src/main/resources/application-buildlogs-local.yml b/src/main/resources/application-buildlogs-local.yml index 53ac81a8c..757bdcda0 100644 --- a/src/main/resources/application-buildlogs-local.yml +++ b/src/main/resources/application-buildlogs-local.yml @@ -4,4 +4,5 @@ wave: logs: bucket: "$PWD/build-workspace" prefix: 'wave-build/logs' + conda-lock-prefix: 'wave-build/conda-locks' ... diff --git a/src/main/resources/io/seqera/wave/build-notification.html b/src/main/resources/io/seqera/wave/build-notification.html index 0c3a96c3a..1261cdfbd 100644 --- a/src/main/resources/io/seqera/wave/build-notification.html +++ b/src/main/resources/io/seqera/wave/build-notification.html @@ -97,8 +97,8 @@

Summary

<% if (scan_url) { %> - Security - Image scan report + Security scan + ${scan_id} <% } %> @@ -111,9 +111,6 @@

Conda file

${build_condafile}
<% } %> -

Build logs

- 
${build_log_data}
-
diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 821050224..012e35f5a 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -1,16 +1,42 @@ - - {{#if build_in_progress}} - - {{/if}} - - - - - Wave container build notification - + {{#if build_in_progress}} + + {{/if}} + + + + + Wave container build notification + + + + +
@@ -113,26 +139,61 @@ {{#if scan_url}} - Security - Image scan report + Security scan + {{scan_id}} {{/if}}

Container file

- 
{{build_containerfile}}
+
+ + 
{{build_containerfile}}
+
{{#if build_condafile}}

Conda file

- 
{{build_condafile}}
+
+ + 
{{build_condafile}}
+
+ {{/if}} + + {{#if build_success}} + {{#if build_conda_lock_data }} +

Conda lockfile

+
+ + 
{{build_conda_lock_data}}
+
+ {{/if}} {{/if}} {{#if build_log_data}}

Build logs

- 
{{build_log_data}}
+
+ + 
{{build_log_data}}
{{#if build_log_truncated}} Click here to download the complete build log {{/if}} +
{{/if}}
@@ -147,6 +208,29 @@
- + diff --git a/src/main/resources/io/seqera/wave/container-view.hbs b/src/main/resources/io/seqera/wave/container-view.hbs index 78f18e99b..907620ee7 100644 --- a/src/main/resources/io/seqera/wave/container-view.hbs +++ b/src/main/resources/io/seqera/wave/container-view.hbs @@ -1,38 +1,62 @@ - - - - - - Wave container - - + + + + + Wave container + + + + +
@@ -48,7 +72,7 @@ {{request_token}} - Container image + Request container {{request_container_image}} @@ -74,13 +98,14 @@

Container info

- + - + + {{#if wave_container_show }} @@ -89,6 +114,7 @@ + {{/if}} @@ -102,16 +128,34 @@ {{#if build_container_file}} -

Docker file

- 
{{build_container_file}}
+

Container file

+
+ + 
{{build_container_file}}
+

Conda file

- 
{{build_conda_file}}
+
+ + 
{{build_conda_file}}
+
Source imageContainer image {{source_container_image}}
Source digestContainer digest {{source_container_digest}}
Wave image {{wave_container_image}}Wave digest {{wave_container_digest}}
Fusion version {{fusion_version}}
+ {{#if build_url}} + {{else}} + + {{/if}} @@ -132,7 +176,27 @@
Build ID{{build_id}}-
Cached
{{/if}} - + {{#if mirror_id }} + + + + + + + + + +
Mirror ID{{mirror_id}}
Cached{{mirror_cached}}
+ {{/if}} + + {{#if scan_id }} + + + + + +
Security scan{{scan_id}}
+ {{/if}} {{#if tower_user_id}} @@ -181,30 +245,6 @@
- - -
@@ -217,6 +257,51 @@
+ diff --git a/src/main/resources/io/seqera/wave/mirror-view.hbs b/src/main/resources/io/seqera/wave/mirror-view.hbs new file mode 100644 index 000000000..b7d9f88ad --- /dev/null +++ b/src/main/resources/io/seqera/wave/mirror-view.hbs @@ -0,0 +1,204 @@ + + + {{#if mirror_in_progress}} + + {{/if}} + + + + + Wave container mirror notification + + + + + + +
+ +
+ +

Wave container mirror completion

+
+ + {{#if mirror_success}} +
+

+ Container mirror completed successfully! +

+
+ {{else mirror_failed}} +
+

+ Container mirror failed +

+
+ {{else mirror_in_progress}} + {{! mirror is not completed, show a spinning icon }} + +
+

+
Container mirror in progress +

+
+ {{/if}} + +

Summary

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {{#if mirror_exitcode}} + + + + + {{/if}} + + {{#if mirror_logs}} + + + + {{/if}} + + {{#if scan_url}} + + + + + {{/if}} +
Mirror ID{{mirror_id}}
User{{mirror_user}}
Source container{{mirror_source_image}}
Target container{{mirror_target_image}}
Container digest{{mirror_digest}}
Platform{{mirror_platform}}
Creation time{{mirror_time}}
Duration{{mirror_duration}}
Exit status{{mirror_exitcode}}
+

Logs

+
+ + 
{{mirror_logs}}
+
+
Security scan{{scan_id}}
+ +
+
+ +

+ {{server_url}}
+ Seqera
+ Carrer de Marià Aguiló, 28
+ 08005 Barcelona
+

+
+
+ + + + diff --git a/src/main/resources/io/seqera/wave/scan-view.hbs b/src/main/resources/io/seqera/wave/scan-view.hbs index 659bb6719..7e27b23fa 100644 --- a/src/main/resources/io/seqera/wave/scan-view.hbs +++ b/src/main/resources/io/seqera/wave/scan-view.hbs @@ -40,10 +40,27 @@ Scan ID {{scan_id}} + {{! build info }} + {{#if build_id}} Build ID {{build_id}} + {{/if}} + {{! mirror info }} + {{#if mirror_id}} + + Mirror ID + {{mirror_id}} + + {{/if}} + {{! request info }} + {{#if request_id}} + + Request ID + {{request_id}} + + {{/if}} Container image {{scan_container_image}} diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy new file mode 100644 index 000000000..c6fb81ffa --- /dev/null +++ b/src/test/groovy/io/seqera/wave/auth/RegistryAuthStoreTest.groovy @@ -0,0 +1,41 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class RegistryAuthStoreTest extends Specification { + + @Inject RegistryAuthStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'registry-auth/v1:foo' + } + + +} diff --git a/src/test/groovy/io/seqera/wave/auth/RegistryTokenStoreTest.groovy b/src/test/groovy/io/seqera/wave/auth/RegistryTokenStoreTest.groovy new file mode 100644 index 000000000..e990d756e --- /dev/null +++ b/src/test/groovy/io/seqera/wave/auth/RegistryTokenStoreTest.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.auth + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class RegistryTokenStoreTest extends Specification { + + @Inject RegistryTokenStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'registry-token/v1:foo' + } + +} diff --git a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy index 5809d680e..358a1804c 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy @@ -88,8 +88,9 @@ class BuildControllerTest extends Specification { ip: "1.2.3.4", configJson: '{"config":"json"}', scanId: 'scan12345', - format: format ) - .withBuildId('1') + format: format, + buildId: "bd-${containerId}_1" + ) final result = new BuildResult(build.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(build, result) final entry = WaveBuildRecord.fromEvent(event) @@ -112,13 +113,13 @@ class BuildControllerTest extends Specification { when: def req = HttpRequest.GET("/v1alpha1/builds/${buildId}/logs") - def res = client.toBlocking().exchange(req, StreamedFile) + def res = client.toBlocking().exchange(req, String) then: 1 * buildLogService.fetchLogStream(buildId) >> response and: res.code() == 200 - new String(res.bodyBytes) == LOGS + res.body() == LOGS } def 'should get container status' () { @@ -152,4 +153,21 @@ class BuildControllerTest extends Specification { e.status == HttpStatus.NOT_FOUND } + def 'should get conda lock file' () { + given: + def buildId = 'testbuildid1234' + def condaLock = "test conda lock" + def response = new StreamedFile(new ByteArrayInputStream(condaLock.bytes), MediaType.APPLICATION_OCTET_STREAM_TYPE) + + when: + def req = HttpRequest.GET("/v1alpha1/builds/${buildId}/condalock") + def res = client.toBlocking().exchange(req, String) + + then: + 1 * buildLogService.fetchCondaLockStream(buildId) >> response + and: + res.code() == 200 + res.body() == condaLock + } + } diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy index 85bb245db..7620d81bd 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerHttpTest.groovy @@ -36,7 +36,7 @@ import io.seqera.wave.api.SubmitContainerTokenResponse import io.seqera.wave.core.RouteHandler import io.seqera.wave.exception.HttpResponseException import io.seqera.wave.exchange.DescribeWaveContainerResponse -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.pairing.PairingRecord import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.PairingServiceImpl @@ -299,7 +299,7 @@ class ContainerControllerHttpTest extends Specification { when: def req = HttpRequest.DELETE("/container-token/$waveToken").basicAuth("username", "password") - def deleteResp = httpClient.toBlocking().exchange(req, ContainerRequestData) + def deleteResp = httpClient.toBlocking().exchange(req, ContainerRequest) then: deleteResp.status.code == 200 diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 74ff3bf59..f76c6e1c7 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -34,6 +34,7 @@ import io.micronaut.http.server.util.HttpClientAddressResolver import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerStatusResponse import io.seqera.wave.api.ImageNameStrategy import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest @@ -58,8 +59,8 @@ import io.seqera.wave.service.pairing.PairingService import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveContainerRecord -import io.seqera.wave.service.token.ContainerTokenService -import io.seqera.wave.service.token.TokenData +import io.seqera.wave.service.request.ContainerRequestService +import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -88,18 +89,28 @@ class ContainerControllerTest extends Specification { @Inject ApplicationContext applicationContext + @Inject + RegistryProxyService proxyRegistry + @MockBean(JobServiceImpl) JobService mockJobService() { Mock(JobService) } + @MockBean(RegistryProxyService) + RegistryProxyService mockProxy() { + Mock(RegistryProxyService) { + getImageDigest(_,_) >> 'sha256:mock-digest' + } + } + def setup() { jwtAuthStore.clear() } def 'should create request data' () { given: - def controller = new ContainerController(inclusionService: Mock(ContainerInclusionService)) + def controller = new ContainerController(inclusionService: Mock(ContainerInclusionService), registryProxyService: proxyRegistry) when: def req = new SubmitContainerTokenRequest(containerImage: 'ubuntu:latest') @@ -135,6 +146,7 @@ class ContainerControllerTest extends Specification { def 'should create request data with freeze mode' () { given: def freeze = Mock(FreezeService) + def containerImage = 'ubuntu:latest' and: def controller = Spy(new ContainerController(freezeService: freeze, inclusionService: Mock(ContainerInclusionService))) and: @@ -143,7 +155,7 @@ class ContainerControllerTest extends Specification { getTargetImage() >> target } and: - def req = new SubmitContainerTokenRequest(containerImage: 'ubuntu:latest', freeze: true, buildRepository: 'docker.io/foo/bar') + def req = new SubmitContainerTokenRequest(containerImage: containerImage, freeze: true, buildRepository: 'docker.io/foo/bar') when: def data = controller.makeRequestData(req, PlatformId.NULL, "") @@ -151,6 +163,7 @@ class ContainerControllerTest extends Specification { 1 * freeze.freezeBuildRequest(req, _) >> req.copyWith(containerFile: 'FROM ubuntu:latest') 1 * controller.makeBuildRequest(_,_,_) >> BUILD 1 * controller.checkBuild(BUILD,false) >> new BuildTrack('1', target, false) + 1 * controller.getContainerDigest(containerImage, PlatformId.NULL) >> 'sha256:12345' and: data.containerImage == target @@ -168,7 +181,6 @@ class ContainerControllerTest extends Specification { given: def builder = Mock(ContainerBuildService) def dockerAuth = Mock(ContainerInspectServiceImpl) - def proxyRegistry = Mock(RegistryProxyService) def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' def user = new PlatformId(new User(id: 100)) @@ -195,7 +207,6 @@ class ContainerControllerTest extends Specification { given: def builder = Mock(ContainerBuildService) def dockerAuth = Mock(ContainerInspectServiceImpl) - def proxyRegistry = Mock(RegistryProxyService) def persistenceService = Mock(PersistenceService) def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, persistenceService:persistenceService, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' @@ -224,7 +235,6 @@ class ContainerControllerTest extends Specification { given: def builder = Mock(ContainerBuildService) def dockerAuth = Mock(ContainerInspectServiceImpl) - def proxyRegistry = Mock(RegistryProxyService) def controller = new ContainerController(buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig:buildConfig, inclusionService: Mock(ContainerInclusionService)) def DOCKER = 'FROM foo' def user = new PlatformId(new User(id: 100)) @@ -251,10 +261,8 @@ class ContainerControllerTest extends Specification { def 'should make a mirror request' () { given: - def jobService = applicationContext.getBean(JobService) def mirrorService = applicationContext.getBean(ContainerMirrorService) def inspectService = Mock(ContainerInspectServiceImpl) - def proxyRegistry = Mock(RegistryProxyService) def controller = new ContainerController(mirrorService: mirrorService, inspectService: inspectService, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService)) def user = new PlatformId(new User(id: 100)) def req = new SubmitContainerTokenRequest( @@ -353,7 +361,7 @@ class ContainerControllerTest extends Specification { def 'should add library prefix' () { when: def body = new SubmitContainerTokenRequest(containerImage: 'docker.io/hello-world') - def req1 = HttpRequest.POST("/container-token", body) + def req1 = HttpRequest.POST("/v1alpha2/container", body) def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) then: resp1.status() == HttpStatus.OK @@ -367,7 +375,7 @@ class ContainerControllerTest extends Specification { def 'should not add library prefix' () { when: def body = new SubmitContainerTokenRequest(containerImage: 'quay.io/hello-world') - def req1 = HttpRequest.POST("/container-token", body) + def req1 = HttpRequest.POST("/v1alpha2/container", body) def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) then: resp1.status() == HttpStatus.OK @@ -381,7 +389,7 @@ class ContainerControllerTest extends Specification { def 'should record a container request' () { when: def body = new SubmitContainerTokenRequest(containerImage: 'hello-world') - def req1 = HttpRequest.POST("/container-token", body) + def req1 = HttpRequest.POST("/v1alpha2/container", body) def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) then: resp1.status() == HttpStatus.OK @@ -523,11 +531,11 @@ class ContainerControllerTest extends Specification { def builder = Mock(ContainerBuildService) def proxyRegistry = Mock(RegistryProxyService) def addressResolver = Mock(HttpClientAddressResolver) - def tokenService = Mock(ContainerTokenService) + def tokenService = Mock(ContainerRequestService) def persistence = Mock(PersistenceService) def controller = new ContainerController(freezeService: freeze, buildService: builder, inspectService: dockerAuth, registryProxyService: proxyRegistry, buildConfig: buildConfig, inclusionService: Mock(ContainerInclusionService), - addressResolver: addressResolver, tokenService: tokenService, persistenceService: persistence, serverUrl: 'http://wave.com') + addressResolver: addressResolver, containerService: tokenService, persistenceService: persistence, serverUrl: 'http://wave.com') when:'packages with conda' def CHANNELS = ['conda-forge', 'defaults'] @@ -671,18 +679,18 @@ class ContainerControllerTest extends Specification { noExceptionThrown() } - def '/v1alpha2/container/{containerId} should return a container record' () { + def 'should return the container record' () { given: def body = new SubmitContainerTokenRequest(containerImage: 'hello-world') - def req1 = HttpRequest.POST("/container-token", body) + def req1 = HttpRequest.POST("/v1alpha2/container", body) def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) and: resp1.status() == HttpStatus.OK and: - def containerId = resp1.body().containerToken + def requestId = resp1.body().requestId when: - def req2 = HttpRequest.GET("/v1alpha2/container/${containerId}") + def req2 = HttpRequest.GET("/v1alpha2/container/${requestId}") def resp2 = client.toBlocking().exchange(req2, WaveContainerRecord) then: resp2.status() == HttpStatus.OK @@ -693,4 +701,28 @@ class ContainerControllerTest extends Specification { result.sourceImage == 'docker.io/library/hello-world:latest' result.waveImage == resp1.body().targetImage } + + def 'should return the container status' () { + given: + def body = new SubmitContainerTokenRequest(containerImage: 'hello-world') + def req1 = HttpRequest.POST("/v1alpha2/container", body) + def resp1 = client.toBlocking().exchange(req1, SubmitContainerTokenResponse) + and: + resp1.status() == HttpStatus.OK + and: + def requestId = resp1.body().requestId + + when: + def req2 = HttpRequest.GET("/v1alpha2/container/${requestId}/status") + def resp2 = client.toBlocking().exchange(req2, ContainerStatusResponse) + then: + resp2.status() == HttpStatus.OK + and: + ContainerStatusResponse result = resp2.body() + and: + result.id == requestId + result.succeeded + result.creationTime + result.duration + } } diff --git a/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy index ae6dd6754..93433c0f5 100644 --- a/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/CustomImageControllerTest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.controller - import spock.lang.Specification import spock.lang.Timeout @@ -37,11 +36,11 @@ import io.micronaut.test.annotation.MockBean import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.core.RoutePath import io.seqera.wave.model.ContentType -import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.builder.impl.ContainerBuildServiceImpl -import io.seqera.wave.service.token.ContainerTokenService +import io.seqera.wave.service.request.ContainerRequest +import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.storage.DigestStore import io.seqera.wave.storage.Storage import io.seqera.wave.test.DockerRegistryContainer @@ -78,15 +77,13 @@ class CustomImageControllerTest extends Specification implements DockerRegistryC } } - @MockBean(ContainerTokenService) - ContainerTokenService containerTokenService(){ - Mock(ContainerTokenService){ - getRequest(_) >> new ContainerRequestData( - PlatformId.NULL, - "library/hello-world", - "FROM busybox", - null, - null) + @MockBean(ContainerRequestService) + ContainerRequestService containerTokenService(){ + Mock(ContainerRequestService){ + getRequest(_) >> ContainerRequest.of( + identity: PlatformId.NULL, + containerImage: "library/hello-world", + containerFile: "FROM busybox" ) } } diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy index 6ec4b4801..004dac8ed 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerRedisTest.groovy @@ -22,6 +22,7 @@ import spock.lang.Shared import spock.lang.Specification import spock.lang.Timeout +import java.nio.file.Path import java.time.Duration import java.time.Instant @@ -35,14 +36,14 @@ import io.micronaut.http.client.exceptions.HttpClientResponseException import io.micronaut.runtime.server.EmbeddedServer import io.seqera.wave.exchange.RegistryErrorResponse import io.seqera.wave.model.ContentType -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.impl.BuildStateStoreImpl import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.job.JobFactory import io.seqera.wave.service.job.JobQueue -import io.seqera.wave.service.token.ContainerTokenStoreImpl +import io.seqera.wave.service.request.ContainerRequestStoreImpl import io.seqera.wave.storage.ManifestCacheStore import io.seqera.wave.test.DockerRegistryContainer import io.seqera.wave.test.RedisTestContainer @@ -103,18 +104,20 @@ class RegistryControllerRedisTest extends Specification implements DockerRegistr given: def client = applicationContext.createBean(HttpClient) def buildCacheStore = applicationContext.getBean(BuildStateStoreImpl) - def tokenCacheStore = applicationContext.getBean(ContainerTokenStoreImpl) + def tokenCacheStore = applicationContext.getBean(ContainerRequestStoreImpl) def jobQueue = applicationContext.getBean(JobQueue) def jobFactory = applicationContext.getBean(JobFactory) def res = BuildResult.create('1') def req = new BuildRequest( targetImage: 'library/hello-world', - buildId: '1', + containerId: '12345', + buildId: 'bd-12345_1', startTime: Instant.now(), - maxDuration: Duration.ofSeconds(5) + maxDuration: Duration.ofSeconds(5), + workspace: Path.of('/some/work/dir') ) def entry = new BuildEntry(req, res) - def containerRequestData = new ContainerRequestData(new PlatformId(new User(id:1)), "library/hello-world") + def containerRequestData = ContainerRequest.of(identity: new PlatformId(new User(id:1)), containerImage: "library/hello-world") and: tokenCacheStore.put("1234", containerRequestData) buildCacheStore.put("library/hello-world", entry) diff --git a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy index a00831bd4..67a238b12 100644 --- a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy @@ -30,7 +30,6 @@ import io.micronaut.http.client.exceptions.HttpClientResponseException import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveScanRecord -import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability import jakarta.inject.Inject @@ -49,11 +48,12 @@ class ScanControllerTest extends Specification { def "should return 200 and WaveContainerScanRecord "() { given: - def containerScanService = Mock(ContainerScanService) def startTime = Instant.now() def duration = Duration.ofMinutes(2) - def scanId = '123' - def buildId = "testbuildid" + def scanId = 'sc123' + def buildId = "bd-123" + def mirrorId = 'mr-123' + def requestId = 'rq-123' def containerImage = "testcontainerimage" def scanVulnerability = new ScanVulnerability( "id1", @@ -64,26 +64,29 @@ class ScanControllerTest extends Specification { "fix.version", "url") def results = List.of(scanVulnerability) - def scanRecord = new WaveScanRecord( - buildId, - new ScanEntry( - scanId, - buildId, - containerImage, - startTime, - duration, - 'SUCCEEDED', - results)) + def scan = new WaveScanRecord( + new ScanEntry( + scanId, + buildId, + mirrorId, + requestId, + containerImage, + startTime, + duration, + 'SUCCEEDED', + results)) and: - persistenceService.createScanRecord(scanRecord) + persistenceService.saveScanRecord(scan) when: - def req = HttpRequest.GET("/v1alpha1/scans/${scanRecord.id}") + def req = HttpRequest.GET("/v1alpha1/scans/${scan.id}") def res = client.toBlocking().exchange(req, WaveScanRecord) then: - res.body().id == scanRecord.id - res.body().buildId == scanRecord.buildId + res.body().id == scan.id + res.body().buildId == scan.buildId + res.body().mirrorId == scan.mirrorId + res.body().requestId == scan.requestId } diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index f48b1505f..fcbc74db5 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -33,14 +33,16 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.logs.BuildLogService import io.seqera.wave.service.logs.BuildLogServiceImpl +import io.seqera.wave.service.mirror.MirrorResult import io.seqera.wave.service.persistence.PersistenceService import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord +import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.tower.PlatformId @@ -73,7 +75,7 @@ class ViewControllerTest extends Specification { @Inject private ContainerInspectService inspectService - def 'should return build page mapping' () { + def 'should create build binding' () { given: def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService) and: @@ -95,6 +97,7 @@ class ViewControllerTest extends Specification { def binding = controller.renderBuildView(record) then: 1 * buildLogService.fetchLogString('12345') >> new BuildLogService.BuildLog('log content', false) + 1 * buildLogService.fetchCondaLockString('12345') >> 'conda lock content' and: binding.build_id == '12345' binding.build_containerfile == 'FROM foo' @@ -106,6 +109,7 @@ class ViewControllerTest extends Specification { binding.build_platform == 'linux/amd64' binding.build_containerfile == 'FROM foo' binding.build_condafile == 'conda::foo' + binding.build_conda_lock_data == 'conda lock content' binding.build_format == 'Docker' binding.build_log_data == 'log content' binding.build_log_truncated == false @@ -188,14 +192,14 @@ class ViewControllerTest extends Specification { def user = new User(id:1) def identity = new PlatformId(user,100) and: - def data = new ContainerRequestData(identity, 'hello-world', 'some docker', cfg, 'some conda') + def token = '12345' + def data = ContainerRequest.of(requestId: token, identity: identity, containerImage: 'hello-world', containerFile: 'some docker', containerConfig: cfg, condaFile: 'some conda') def wave = 'https://wave.io/some/container:latest' def addr = '100.200.300.400' and: def exp = Instant.now().plusSeconds(3600) - def token = '12345' - def container = new WaveContainerRecord(req, data, token, wave, addr, exp) + def container = new WaveContainerRecord(req, data, wave, addr, exp) when: persistenceService.saveContainerRequest(container) @@ -315,19 +319,23 @@ class ViewControllerTest extends Specification { binding.build_failed == true } - def 'should render in success scan page' () { + def 'should create binding' () { given: def controller = new ViewController(serverUrl: 'http://foo.com', buildLogService: buildLogService) and: - def result = new ScanEntry( - '12345', + def result = new WaveScanRecord( '12345', + 'bd-12345', + 'mr-12345', + 'cr-12345', 'docker.io/some:image', Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, [new ScanVulnerability('cve-1', 'HIGH', 'test vul', 'testpkg', '1.0.0', '1.1.0', 'http://vul/cve-1')], - 0 ) + 0, + "Some scan logs" + ) when: def binding = controller.makeScanViewBinding(result) then: @@ -337,8 +345,77 @@ class ViewControllerTest extends Specification { binding.scan_duration == formatDuration(result.duration) binding.scan_succeeded binding.scan_exitcode == 0 + binding.scan_logs == "Some scan logs" binding.vulnerabilities == [new ScanVulnerability(id:'cve-1', severity:'HIGH', title:'test vul', pkgName:'testpkg', installedVersion:'1.0.0', fixedVersion:'1.1.0', primaryUrl:'http://vul/cve-1')] - binding.build_url == 'http://foo.com/view/builds/12345' + binding.build_id == 'bd-12345' + binding.build_url == 'http://foo.com/view/builds/bd-12345' + binding.mirror_id == 'mr-12345' + binding.mirror_url == 'http://foo.com/view/mirrors/mr-12345' + binding.request_id == 'cr-12345' + binding.request_url == 'http://foo.com/view/containers/cr-12345' + } + + def 'should render scan view page' () { + given: + def scan = new WaveScanRecord( + '12345', + 'bd-12345', + 'mr-12345', + 'cr-12345', + 'docker.io/some:image', + Instant.now(), + Duration.ofMinutes(1), + ScanEntry.SUCCEEDED, + [new ScanVulnerability('cve-1', 'HIGH', 'test vul', 'testpkg', '1.0.0', '1.1.0', 'http://vul/cve-1')], + 0, + "Some scan logs" + ) + + when: + persistenceService.saveScanRecord(scan) + and: + def request = HttpRequest.GET("/view/scans/${scan.id}") + def response = client.toBlocking().exchange(request, String) + then: + response.body().contains(scan.id) + response.body().contains(scan.buildId) + response.body().contains(scan.mirrorId) + response.body().contains(scan.requestId) + response.body().contains(scan.containerImage) + } + + def 'should render mirror page' () { + given: + def record1 = new MirrorResult( + '12345', + 'sha256:abc12345', + 'docker.io/ubuntu:latest', + 'quay.io/ubuntu:latest', + ContainerPlatform.DEFAULT, + Instant.now(), + "GMT", + 'pditommaso', + 'paolo@me.com', + 1, + "sc-12456", + MirrorResult.Status.COMPLETED, + Duration.ofMinutes(1), + 0, + 'No logs' + ) + + when: + persistenceService.saveMirrorResult(record1) + and: + def request = HttpRequest.GET("/view/mirrors/${record1.mirrorId}") + def response = client.toBlocking().exchange(request, String) + then: + response.body().contains(record1.mirrorId) + response.body().contains(record1.scanId) + response.body().contains(record1.sourceImage) + response.body().contains(record1.targetImage) + response.body().contains(record1.digest) + response.body().contains(record1.userName) } @Unroll @@ -357,10 +434,12 @@ class ViewControllerTest extends Specification { '12345_1' | null '12345-1' | '/view/builds/12345_1' 'foo-887766-1' | '/view/builds/foo-887766_1' - + and: + 'bd-12345_1' | null + 'bd-12345-1' | '/view/builds/bd-12345_1' + 'bd-887766-1' | '/view/builds/bd-887766_1' } - def 'should validate redirect 2' () { given: def service = Mock(ContainerBuildService) @@ -370,16 +449,20 @@ class ViewControllerTest extends Specification { def result = controller.shouldRedirect2(BUILD) then: result == EXPECTED - TIMES * service.getLatestBuild(BUILD) >> LATEST + TIMES * service.getLatestBuild(BUILD) >> Mock(WaveBuildRecord) { buildId>>LATEST } where: - BUILD | TIMES | LATEST | EXPECTED - '12345_1' | 0 | null | null - '12345' | 1 | Mock(WaveBuildRecord) { buildId >> '12345_99' } | '/view/builds/12345_99' - '12345' | 1 | Mock(WaveBuildRecord) { buildId >> 'xyz_99' } | null - 'foo-887766' | 1 | Mock(WaveBuildRecord) { buildId >> 'foo-887766_99' } | '/view/builds/foo-887766_99' - 'foo-887766' | 1 | Mock(WaveBuildRecord) { buildId >> 'foo-887766' } | null + BUILD | TIMES | LATEST | EXPECTED + '12345_1' | 0 | null | null + '12345' | 1 | '12345_99' | '/view/builds/12345_99' + '12345' | 1 | 'xyz_99' | null + 'foo-887766' | 1 | 'foo-887766_99' | '/view/builds/foo-887766_99' + 'foo-887766' | 1 | 'foo-887766' | null + 'bd-887766' | 1 | 'bd-887766_2' | '/view/builds/bd-887766_2' + '887766' | 1 | 'bd-887766_2' | '/view/builds/bd-887766_2' + } + } diff --git a/src/test/groovy/io/seqera/wave/core/RouteHandlerTest.groovy b/src/test/groovy/io/seqera/wave/core/RouteHandlerTest.groovy index b57a97ebc..46579a7c0 100644 --- a/src/test/groovy/io/seqera/wave/core/RouteHandlerTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/RouteHandlerTest.groovy @@ -22,10 +22,8 @@ import spock.lang.Specification import spock.lang.Unroll import io.seqera.wave.exception.NotFoundException -import io.seqera.wave.service.ContainerRequestData -import io.seqera.wave.service.token.ContainerTokenService -import io.seqera.wave.tower.PlatformId - +import io.seqera.wave.service.request.ContainerRequest +import io.seqera.wave.service.request.ContainerRequestService /** * * @author Paolo Di Tommaso @@ -68,7 +66,7 @@ class RouteHandlerTest extends Specification { @Unroll def 'should get manifests route #PATH'() { given: - def tokenService = Mock(ContainerTokenService) + def tokenService = Mock(ContainerRequestService) when: def route = new RouteHandler(tokenService).parse(PATH) @@ -90,7 +88,7 @@ class RouteHandlerTest extends Specification { def 'should throw bad not found exception' () { given: - def tokenService = Mock(ContainerTokenService) + def tokenService = Mock(ContainerRequestService) when: new RouteHandler(tokenService).parse(PATH) @@ -112,12 +110,12 @@ class RouteHandlerTest extends Specification { @Unroll def 'should find container name via token service' () { given: - def tokenService = Mock(ContainerTokenService) + def tokenService = Mock(ContainerRequestService) when: def route = new RouteHandler(tokenService).parse(REQ_PATH) then: - 1 * tokenService.getRequest(ROUTE_TKN) >> new ContainerRequestData(PlatformId.NULL,REQ_IMAGE) + 1 * tokenService.getRequest(ROUTE_TKN) >> ContainerRequest.of(containerImage: REQ_IMAGE) and: route.image == ROUTE_IMAGE route.type == ROUTE_TYPE diff --git a/src/test/groovy/io/seqera/wave/core/RoutePathTest.groovy b/src/test/groovy/io/seqera/wave/core/RoutePathTest.groovy index db9bde60a..1bf1902fe 100644 --- a/src/test/groovy/io/seqera/wave/core/RoutePathTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/RoutePathTest.groovy @@ -22,7 +22,7 @@ import spock.lang.Specification import spock.lang.Unroll import io.seqera.wave.model.ContainerCoordinates -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User /** @@ -141,7 +141,7 @@ class RoutePathTest extends Specification { 'ubuntu', 'latest', '/v2/library/ubuntu/manifests/latest', - new ContainerRequestData(new PlatformId(new User(id: 100)), 'ubuntu:latest') ) + ContainerRequest.of(identity: new PlatformId(new User(id: 100)), containerImage: 'ubuntu:latest') ) then: route2.type == 'manifests' route2.registry == 'foo.com' diff --git a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy index c8a9f9337..878ba0241 100644 --- a/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/encoder/MoshiEncodingStrategyTest.groovy @@ -26,9 +26,11 @@ import java.time.Instant import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerStatus +import io.seqera.wave.api.ContainerStatusResponse import io.seqera.wave.auth.RegistryAuth import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult @@ -77,7 +79,7 @@ class MoshiEncodingStrategyTest extends Specification { when: def build = encoder.decode(json) then: - build.id == '100' + build.buildId == '100' build.exitStatus == 1 build.logs == 'logs' build.startTime == Instant.parse('2022-12-03T22:27:04.079724Z') @@ -86,15 +88,15 @@ class MoshiEncodingStrategyTest extends Specification { def 'should encode and decode ContainerRequestData' () { given: - def encoder = new MoshiEncodeStrategy() { } + def encoder = new MoshiEncodeStrategy() { } and: - def data = new ContainerRequestData( - new PlatformId(new User(id:1),2), - 'ubuntu', - 'from foo', - new ContainerConfig(entrypoint: ['some', 'entry'], cmd:['the', 'cmd']), - 'some/conda/file', - ContainerPlatform.of('amd64') ) + def data = ContainerRequest.of( + identity: new PlatformId(new User(id:1),2), + containerImage: 'ubuntu', + containerFile: 'from foo', + containerConfig: new ContainerConfig(entrypoint: ['some', 'entry'], cmd:['the', 'cmd']), + condaFile: 'some/conda/file', + platform: ContainerPlatform.of('amd64') ) when: def json = encoder.encode(data) @@ -137,7 +139,7 @@ class MoshiEncodingStrategyTest extends Specification { } ''' and: - def encoder = new MoshiEncodeStrategy() { } + def encoder = new MoshiEncodeStrategy() { } when: def result = encoder.decode(REQUEST) @@ -337,8 +339,9 @@ class MoshiEncodingStrategyTest extends Specification { ip: "1.2.3.4", configJson: '{"config":"json"}', scanId: 'scan12345', - buildContext: context ) - .withBuildId('1') + buildContext: context, + buildId: '12345_1', + ) def result = new BuildResult(build.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(build, result) @@ -427,8 +430,9 @@ class MoshiEncodingStrategyTest extends Specification { ip: "1.2.3.4", configJson: '{"config":"json"}', scanId: 'scan12345', - buildContext: context ) - .withBuildId('1') + buildContext: context, + buildId: '12345_1', + ) and: def entry = new BuildEntry(req, res) when: @@ -440,4 +444,28 @@ class MoshiEncodingStrategyTest extends Specification { and: copy == entry } + + def 'should encode container status response' () { + given: + def encoder = new MoshiEncodeStrategy() {} + def timestamp = Instant.parse('2024-10-07T20:41:00.804699Z') + def resp = new ContainerStatusResponse( + '1234', + ContainerStatus.DONE, + 'bd-12345', + 'mr-12345', + 'sc-12345', + [LOW: 1, MEDIUM: 2], + true, + 'Some err message', + 'http://foo.com/view/123', + timestamp, + Duration.ofMinutes(1), + ) + + when: + def result = encoder.encode(resp) + then: + result == '{"buildId":"bd-12345","creationTime":"2024-10-07T20:41:00.804699Z","detailsUri":"http://foo.com/view/123","duration":"60000000000","id":"1234","mirrorId":"mr-12345","reason":"Some err message","scanId":"sc-12345","status":"DONE","succeeded":true,"vulnerabilities":{"LOW":1,"MEDIUM":2}}' + } } diff --git a/src/test/groovy/io/seqera/wave/exchange/BuildStatusResponseTest.groovy b/src/test/groovy/io/seqera/wave/exchange/BuildStatusResponseTest.groovy index 492a3d655..482302db8 100644 --- a/src/test/groovy/io/seqera/wave/exchange/BuildStatusResponseTest.groovy +++ b/src/test/groovy/io/seqera/wave/exchange/BuildStatusResponseTest.groovy @@ -46,6 +46,5 @@ class BuildStatusResponseTest extends Specification { def copy = JacksonHelper.fromJson(json, BuildStatusResponse) then: copy == resp - } } diff --git a/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy index 9d1151171..61ad3465f 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/BlobStateStoreImplTest.groovy @@ -43,6 +43,12 @@ class BlobStateStoreImplTest extends Specification { @Inject StateProvider provider + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-blobcache/v1:foo' + } + + def 'should get and store an entry' () { given: def key = UUID.randomUUID().toString() diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildEntryTest.groovy new file mode 100644 index 000000000..67f74489e --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildEntryTest.groovy @@ -0,0 +1,89 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.nio.file.Path + +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId + +/** + * + * @author Paolo Di Tommaso + */ +class BuildEntryTest extends Specification { + + def 'should create entry with constructor'() { + given: + def request = Mock(BuildRequest) + def result = Mock(BuildResult) + when: + def entry = new BuildEntry(request, result) + then: + entry.request == request + entry.result == result + } + + def 'should create entry with result'() { + given: + def request = Mock(BuildRequest) + def result1 = Mock(BuildResult) + def result2 = Mock(BuildResult) + when: + def entry1 = new BuildEntry(request, result1) + then: + entry1.request == request + entry1.result == result1 + + when: + def entry2 = entry1.withResult(result2) + then: + entry1.request == request + entry1.result == result1 + and: + entry2.request == request + entry2.result == result2 + and: + result1 != result2 + } + + def 'should create entry with factory' () { + given: + def request = new BuildRequest( + containerId: '12345', + buildId: "bd-12345_1", + containerFile: 'FROM foo', + workspace: Path.of("/some/path"), + targetImage: 'some/target:12345', + identity: PlatformId.NULL, + platform: ContainerPlatform.DEFAULT, + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345', + ) + when: + def entry = BuildEntry.create(request) + then: + entry.request == request + entry.result == BuildResult.create(request) + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index 9959f7873..ea9fb485f 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.builder - import spock.lang.Specification import java.nio.file.Path @@ -72,7 +71,7 @@ class BuildRequestTest extends Specification { SCAN_ID, CONTEXT, FORMAT, - TIMEOUT + TIMEOUT, ) then: @@ -117,7 +116,7 @@ class BuildRequestTest extends Specification { SCAN_ID, CONTEXT, FORMAT, - TIMEOUT + TIMEOUT, ) then: req.containerId == '8026e3a63b5c863f' @@ -143,7 +142,7 @@ class BuildRequestTest extends Specification { SCAN_ID, CONTEXT, FORMAT, - TIMEOUT + TIMEOUT, ) then: req.containerId == '181ec22b26ae6d04' @@ -185,7 +184,7 @@ class BuildRequestTest extends Specification { null, CONTEXT, FORMAT, - TIMEOUT + TIMEOUT, ) then: req.containerId == 'd78ba9cb01188668' diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStateStoreImplTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStateStoreImplTest.groovy new file mode 100644 index 000000000..d6e7d38a0 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStateStoreImplTest.groovy @@ -0,0 +1,55 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.service.builder.impl.BuildStateStoreImpl +import io.seqera.wave.store.state.CountParams +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class BuildStateStoreImplTest extends Specification { + + @Inject + BuildStateStoreImpl store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-build/v2:foo' + } + + def 'should return record id' () { + expect: + store.requestId0('foo') == 'wave-build/v2/request-id:foo' + } + + def 'should return counter key' () { + given: + def build = Mock(BuildEntry) { + getRequest()>>Mock(BuildRequest) { getContainerId()>>'12345' } + } + expect: + store.counterKey('foo', build) == new CountParams('build-counters/v1','12345') + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStoreLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStoreLocalTest.groovy index d01248823..4567e04cb 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStoreLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStoreLocalTest.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.builder import spock.lang.Specification +import java.nio.file.Path import java.time.Duration import java.time.Instant import java.util.concurrent.ExecutorService @@ -27,8 +28,10 @@ import java.util.concurrent.ExecutorService import io.micronaut.scheduling.TaskExecutors import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.impl.BuildStateStoreImpl import io.seqera.wave.store.state.impl.LocalStateProvider +import io.seqera.wave.tower.PlatformId import jakarta.inject.Inject import jakarta.inject.Named @@ -185,4 +188,62 @@ class BuildStoreLocalTest extends Specification { cache.awaitBuild('foo').get() == three.result } + def 'should store a build entry only if absent' () { + given: + def _100ms = 100 + def config = Mock(BuildConfig) { getStatusDuration()>>Duration.ofMillis(_100ms) } + def provider = new LocalStateProvider() + def store = new BuildStateStoreImpl(provider, config, ioExecutor) + and: + def request = new BuildRequest( + containerId: '12345', + buildId: 'bd-12345_0', + containerFile: 'from foo', + condaFile: 'conda spec', + workspace: Path.of("/some/path"), + targetImage: 'docker.io/some:image:12345', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345' ) + def entry = BuildEntry.create(request) + + when: + def result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + result.succeed + result.count == 1 + result.value.request.buildId == 'bd-12345_1' + result.value.result.buildId == 'bd-12345_1' + result.value.request.workDir == Path.of("/some/path/bd-12345_1") + and: + store.findByRequestId('bd-12345_1') == result.value + + when: + result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + !result.succeed + result.count == 1 + result.value.request.buildId == 'bd-12345_1' + result.value.result.buildId == 'bd-12345_1' + result.value.request.workDir == Path.of("/some/path/bd-12345_1") + and: + store.findByRequestId('bd-12345_1') == result.value + + when: + sleep(2 *_100ms) + and: + result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + result.succeed + result.count == 2 + result.value.request.buildId == 'bd-12345_2' + result.value.result.buildId == 'bd-12345_2' + result.value.request.workDir == Path.of("/some/path/bd-12345_2") + and: + store.findByRequestId('bd-12345_1') == null + store.findByRequestId('bd-12345_2') == result.value + } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStoreRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStoreRedisTest.groovy index 853744a4c..2f7e3fe2b 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStoreRedisTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStoreRedisTest.groovy @@ -22,14 +22,19 @@ import spock.lang.Shared import spock.lang.Specification import spock.lang.Timeout +import java.nio.file.Path import java.time.Duration import java.time.Instant import io.micronaut.context.ApplicationContext +import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.impl.BuildStateStoreImpl import io.seqera.wave.service.job.JobFactory import io.seqera.wave.service.job.JobQueue +import io.seqera.wave.store.state.impl.RedisStateProvider import io.seqera.wave.test.RedisTestContainer +import io.seqera.wave.tower.PlatformId /** * * @author Paolo Di Tommaso @@ -222,7 +227,8 @@ class BuildStoreRedisTest extends Specification implements RedisTestContainer { targetImage: 'docker.io/foo:1', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofSeconds(5) + maxDuration: Duration.ofSeconds(5), + workspace: Path.of('/some/work/dir') ) def entry = new BuildEntry(req, res) and: @@ -236,4 +242,64 @@ class BuildStoreRedisTest extends Specification implements RedisTestContainer { buildCacheStore.getBuild(req.targetImage).result.exitStatus == -1 } + def 'should store a build entry only if absent' () { + given: + def _100ms = 100 + def config = Mock(BuildConfig) { getStatusDuration()>>Duration.ofMillis(_100ms) } + def provider = applicationContext.getBean(RedisStateProvider) + def store = new BuildStateStoreImpl(provider, config, null) + and: + def request = new BuildRequest( + containerId: '12345', + buildId: 'bd-12345_0', + containerFile: 'from foo', + condaFile: 'conda spec', + workspace: Path.of("/some/path"), + targetImage: 'docker.io/some:image:12345', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'cacherepo', + ip: "1.2.3.4", + configJson: '{"config":"json"}', + scanId: 'scan12345' ) + def entry = BuildEntry.create(request) + + when: + def result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + result.succeed + result.count == 1 + result.value.request.buildId == 'bd-12345_1' + result.value.result.buildId == 'bd-12345_1' + result.value.request.workDir == Path.of("/some/path/bd-12345_1") + and: + store.findByRequestId('bd-12345_1') == result.value + + when: + result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + !result.succeed + result.count == 1 + result.value.request.buildId == 'bd-12345_1' + result.value.result.buildId == 'bd-12345_1' + result.value.request.workDir == Path.of("/some/path/bd-12345_1") + and: + store.findByRequestId('bd-12345_1') == result.value + + when: + sleep(2 *_100ms) + and: + result= store.putIfAbsentAndCount('my/container:latest', entry) + then: + result.succeed + result.count == 2 + result.value.request.buildId == 'bd-12345_2' + result.value.result.buildId == 'bd-12345_2' + result.value.request.workDir == Path.of("/some/path/bd-12345_2") + and: + store.findByRequestId('bd-12345_1') == null + store.findByRequestId('bd-12345_2') == result.value + + } + } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy index fbab4a86c..0512d4e0d 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy @@ -24,6 +24,8 @@ import java.nio.file.Path import java.time.Duration import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.BuildContext +import io.seqera.wave.api.ContainerConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.ContainerHelper @@ -42,10 +44,12 @@ class BuildStrategyTest extends Specification { given: def req = new BuildRequest( containerId: 'c168dba125e28777', - workDir: Path.of('/work/foo/c168dba125e28777'), + buildId: 'bd-c168dba125e28777_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'quay.io/wave:c168dba125e28777', - cacheRepository: 'reg.io/wave/build/cache' ) + cacheRepository: 'reg.io/wave/build/cache', + ) when: def cmd = strategy.launchCmd(req) @@ -55,11 +59,11 @@ class BuildStrategyTest extends Specification { '--frontend', 'dockerfile.v0', '--local', - 'dockerfile=/work/foo/c168dba125e28777', + 'dockerfile=/work/foo/bd-c168dba125e28777_1', '--opt', 'filename=Containerfile', '--local', - 'context=/work/foo/c168dba125e28777/context', + 'context=/work/foo/bd-c168dba125e28777_1/context', '--output', 'type=image,name=quay.io/wave:c168dba125e28777,push=true,oci-mediatypes=true', '--opt', @@ -75,9 +79,10 @@ class BuildStrategyTest extends Specification { given: def req = new BuildRequest( containerId: 'c168dba125e28777', - workDir: Path.of('/work/foo/3980470531b4a52a'), + buildId: 'bd-c168dba125e28777_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/amd64'), - targetImage: 'quay.io/wave:3980470531b4a52a', + targetImage: 'quay.io/wave:c168dba125e28777', cacheRepository: 'reg.io/wave/build/cache' ) when: @@ -88,13 +93,13 @@ class BuildStrategyTest extends Specification { '--frontend', 'dockerfile.v0', '--local', - 'dockerfile=/work/foo/3980470531b4a52a', + 'dockerfile=/work/foo/bd-c168dba125e28777_1', '--opt', 'filename=Containerfile', '--local', - 'context=/work/foo/3980470531b4a52a/context', + 'context=/work/foo/bd-c168dba125e28777_1/context', '--output', - 'type=image,name=quay.io/wave:3980470531b4a52a,push=true,oci-mediatypes=true', + 'type=image,name=quay.io/wave:c168dba125e28777,push=true,oci-mediatypes=true', '--opt', 'platform=linux/amd64', '--export-cache', @@ -107,7 +112,9 @@ class BuildStrategyTest extends Specification { def 'should get singularity command' () { given: def req = new BuildRequest( - workDir: Path.of('/work/foo/c168dba125e28777'), + containerId: 'c168dba125e28777', + buildId: 'bd-c168dba125e28777_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'oras://quay.io/wave:c168dba125e28777', format: BuildFormat.SINGULARITY, @@ -118,7 +125,7 @@ class BuildStrategyTest extends Specification { cmd == [ "sh", "-c", - "singularity build image.sif /work/foo/c168dba125e28777/Containerfile && singularity push image.sif oras://quay.io/wave:c168dba125e28777" + "singularity build image.sif /work/foo/bd-c168dba125e28777_1/Containerfile && singularity push image.sif oras://quay.io/wave:c168dba125e28777" ] } @@ -133,7 +140,7 @@ class BuildStrategyTest extends Specification { def build = new BuildRequest( containerId, content, - null, + 'condaFile', workspace, targetImage, PlatformId.NULL, @@ -141,10 +148,10 @@ class BuildStrategyTest extends Specification { 'caherepo', "1.2.3.4", '{"config":"json"}', - null, - null, - 'scan12345', - null, + 'GMT+1', + Mock(ContainerConfig), + 'sc-12345', + Mock(BuildContext), BuildFormat.DOCKER, timeout ) @@ -153,17 +160,11 @@ class BuildStrategyTest extends Specification { build.containerId == 'af15cb0a413a2d48' build.workspace == Path.of("some/path") and: - !build.buildId - !build.workDir - - when: - build.withBuildId('100') - then: build.containerId == 'af15cb0a413a2d48' build.workspace == Path.of("some/path") and: - build.buildId == 'af15cb0a413a2d48_100' - build.workDir == Path.of('.').toRealPath().resolve('some/path/af15cb0a413a2d48_100') + build.buildId == 'bd-af15cb0a413a2d48_0' + build.workDir == Path.of('.').toRealPath().resolve('some/path/bd-af15cb0a413a2d48_0') build.maxDuration == timeout } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy index ff911d18c..53cbebd01 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceLiveTest.groovy @@ -90,9 +90,9 @@ class ContainerBuildServiceLiveTest extends Specification { configJson: cfg, format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: duration + maxDuration: duration, + buildId: "${containerId}_1", ) - .withBuildId('1') and: buildCacheStore.storeBuild(targetImage, new BuildEntry(req, BuildResult.create(req))) @@ -136,9 +136,9 @@ class ContainerBuildServiceLiveTest extends Specification { configJson: cfg, format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: duration + maxDuration: duration, + buildId: "${containerId}_1", ) - .withBuildId('1') and: buildCacheStore.storeBuild(targetImage, new BuildEntry(req, BuildResult.create(req))) @@ -182,9 +182,9 @@ class ContainerBuildServiceLiveTest extends Specification { configJson: cfg, format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: duration + maxDuration: duration, + buildId: "${containerId}_1", ) - .withBuildId('1') and: buildCacheStore.storeBuild(targetImage, new BuildEntry(req, BuildResult.create(req))) @@ -228,9 +228,9 @@ class ContainerBuildServiceLiveTest extends Specification { configJson: cfg, format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: duration + maxDuration: duration, + buildId: "${containerId}_1", ) - .withBuildId('1') and: buildCacheStore.storeBuild(targetImage, new BuildEntry(req, BuildResult.create(req))) @@ -281,9 +281,9 @@ class ContainerBuildServiceLiveTest extends Specification { containerConfig: containerConfig , format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: duration + maxDuration: duration, + buildId: "${containerId}_1", ) - .withBuildId('1') and: buildCacheStore.storeBuild(targetImage, new BuildEntry(req, BuildResult.create(req))) diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 1f9f72ddf..8923dec66 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -131,9 +131,9 @@ class ContainerBuildServiceTest extends Specification { cacheRepository: cacheRepo, format: BuildFormat.DOCKER, startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + buildId: "${containerId}_1", ) - .withBuildId('1') and: def store = Mock(BuildStateStore) def jobService = Mock(JobService) @@ -171,9 +171,9 @@ class ContainerBuildServiceTest extends Specification { identity: Mock(PlatformId), platform: ContainerPlatform.of('amd64'), format: BuildFormat.DOCKER, - startTime: Instant.now() + startTime: Instant.now(), + buildId: "${containerId}_1", ) - .withBuildId('1') when: def result = builder.containerFile0(req, null) @@ -206,9 +206,9 @@ class ContainerBuildServiceTest extends Specification { identity: Mock(PlatformId), platform: ContainerPlatform.of('amd64'), format: BuildFormat.SINGULARITY, - startTime: Instant.now() + startTime: Instant.now(), + buildId: "${containerId}_1", ) - .withBuildId('1') when: def result = builder.containerFile0(req, Path.of('/some/context/')) @@ -288,9 +288,9 @@ class ContainerBuildServiceTest extends Specification { configJson: '{"config":"json"}', containerConfig: config , format: BuildFormat.DOCKER, - startTime: Instant.now() + startTime: Instant.now(), + buildId: "${containerId}_1", ) - .withBuildId('1') when: service.saveLayersToContext(req, folder) @@ -305,8 +305,9 @@ class ContainerBuildServiceTest extends Specification { void "an event insert a build"() { given: + def containerId = 'container1234' final request = new BuildRequest( - containerId: 'container1234', + containerId: containerId, containerFile: 'test', condaFile: 'test', workspace: Path.of("."), @@ -316,9 +317,9 @@ class ContainerBuildServiceTest extends Specification { configJson: '{"config":"json"}', scanId: 'scan12345', format: BuildFormat.DOCKER, - startTime: Instant.now() + startTime: Instant.now(), + buildId: "${containerId}_1", ) - .withBuildId('123') and: def result = new BuildResult(request.buildId, 0, "content", Instant.now(), Duration.ofSeconds(1), 'abc123') diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 52c4c1f8c..1a6fc858a 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -97,7 +97,8 @@ class DockerBuildStrategyTest extends Specification { and: def req = new BuildRequest( containerId: '89fb83ce6ec8627b', - workDir: Path.of('/work/foo/89fb83ce6ec8627b'), + buildId: 'bd-89fb83ce6ec8627b_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'repo:89fb83ce6ec8627b', cacheRepository: 'reg.io/wave/build/cache' ) @@ -110,7 +111,7 @@ class DockerBuildStrategyTest extends Specification { '--name', 'build-job-name', '--privileged', - '-v', '/work/foo/89fb83ce6ec8627b:/work/foo/89fb83ce6ec8627b', + '-v', '/work/foo/bd-89fb83ce6ec8627b_1:/work/foo/bd-89fb83ce6ec8627b_1', '--entrypoint', 'buildctl-daemonless.sh', '-v', '/work/creds.json:/home/user/.docker/config.json:ro', @@ -120,11 +121,11 @@ class DockerBuildStrategyTest extends Specification { '--frontend', 'dockerfile.v0', '--local', - 'dockerfile=/work/foo/89fb83ce6ec8627b', + 'dockerfile=/work/foo/bd-89fb83ce6ec8627b_1', '--opt', 'filename=Containerfile', '--local', - 'context=/work/foo/89fb83ce6ec8627b/context', + 'context=/work/foo/bd-89fb83ce6ec8627b_1/context', '--output', 'type=image,name=repo:89fb83ce6ec8627b,push=true,oci-mediatypes=true', '--opt', @@ -146,7 +147,9 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( - workDir: Path.of('/work/foo/d4869cc39b8d7d55'), + containerId: 'd4869cc39b8d7d55', + buildId: 'bd-d4869cc39b8d7d55_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'oras://repo:d4869cc39b8d7d55', cacheRepository: 'reg.io/wave/build/cache', @@ -161,14 +164,14 @@ class DockerBuildStrategyTest extends Specification { 'build-job-name', '--privileged', '--entrypoint', '', - '-v', '/work/foo/d4869cc39b8d7d55:/work/foo/d4869cc39b8d7d55', + '-v', '/work/foo/bd-d4869cc39b8d7d55_1:/work/foo/bd-d4869cc39b8d7d55_1', '-v', '/work/creds.json:/root/.singularity/docker-config.json:ro', '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', '--platform', 'linux/amd64', 'quay.io/singularity/singularity:v3.11.4-slim', 'sh', '-c', - 'singularity build image.sif /work/foo/d4869cc39b8d7d55/Containerfile && singularity push image.sif oras://repo:d4869cc39b8d7d55' + 'singularity build image.sif /work/foo/bd-d4869cc39b8d7d55_1/Containerfile && singularity push image.sif oras://repo:d4869cc39b8d7d55' ] cleanup: @@ -183,7 +186,9 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( - workDir: Path.of('/work/foo/9c68af894bb2419c'), + containerId: '9c68af894bb2419c', + buildId: 'bd-9c68af894bb2419c_1', + workspace: Path.of('/work/foo'), platform: ContainerPlatform.of('linux/arm64'), targetImage: 'oras://repo:9c68af894bb2419c', cacheRepository: 'reg.io/wave/build/cache', @@ -198,14 +203,14 @@ class DockerBuildStrategyTest extends Specification { 'build-job-name', '--privileged', '--entrypoint', '', - '-v', '/work/foo/9c68af894bb2419c:/work/foo/9c68af894bb2419c', + '-v', '/work/foo/bd-9c68af894bb2419c_1:/work/foo/bd-9c68af894bb2419c_1', '-v', '/work/creds.json:/root/.singularity/docker-config.json:ro', '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', '--platform', 'linux/arm64', 'quay.io/singularity/singularity:v3.11.4-slim-arm64', 'sh', '-c', - 'singularity build image.sif /work/foo/9c68af894bb2419c/Containerfile && singularity push image.sif oras://repo:9c68af894bb2419c' + 'singularity build image.sif /work/foo/bd-9c68af894bb2419c_1/Containerfile && singularity push image.sif oras://repo:9c68af894bb2419c' ] cleanup: diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy deleted file mode 100644 index 7f2a16c8c..000000000 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.builder - -import spock.lang.Specification - -import java.nio.file.Files -import java.time.Duration -import java.time.Instant -import java.util.concurrent.CompletableFuture - -import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.builder.impl.ContainerBuildServiceImpl -import io.seqera.wave.tower.PlatformId -import io.seqera.wave.util.ContainerHelper -/** - * - * @author Jorge Aguilera - */ -class FutureContainerBuildServiceTest extends Specification { - - String buildRepo = 'build/repo' - String cacheRepo = 'cache/repo' - - def 'should wait for successful container build' () { - given: - def folder = Files.createTempDirectory('test') - and: - def dockerfile = """ - FROM busybox - RUN echo 'hello' > hello.txt - """.stripIndent() - and: - def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) - def req = new BuildRequest(containerId, dockerfile, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') - def res = new BuildResult("", 0, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') - and: - def buildStore = Mock(BuildStateStore) - def buildCounter = Mock(BuildCounterStore) - buildStore.getBuildResult(targetImage) >> res - buildStore.awaitBuild(targetImage) >> CompletableFuture.completedFuture(res) - def service = new ContainerBuildServiceImpl(buildStore: buildStore, buildCounter: buildCounter) - - when: - service.checkOrSubmit(req) - then: - noExceptionThrown() - - when: - def status = service.buildResult(req.targetImage).get() - then: - status.getExitStatus() == 0 - - cleanup: - folder?.deleteDir() - } - - - def 'should wait for unsuccessful container build' () { - given: - def folder = Files.createTempDirectory('test') - and: - def dockerfile = """ - FROM busybox - RUN echo 'hello' > hello.txt - """.stripIndent() - and: - def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), buildRepo, null) - def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, buildRepo, containerId, null, null) - def req = new BuildRequest(containerId, dockerfile, null, folder, targetImage, Mock(PlatformId), ContainerPlatform.of('amd64'), cacheRepo, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') - def res = new BuildResult("", 1, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') - and: - def buildStore = Mock(BuildStateStore) - def buildCounter = Mock(BuildCounterStore) - buildStore.getBuildResult(targetImage) >> res - buildStore.awaitBuild(targetImage) >> CompletableFuture.completedFuture(res) - def service = new ContainerBuildServiceImpl(buildStore: buildStore, buildCounter: buildCounter) - - when: - service.checkOrSubmit(req) - then: - noExceptionThrown() - - when: - def status = service.buildResult(req.targetImage).get() - then: - status.getExitStatus() == 1 - - cleanup: - folder?.deleteDir() - } - -} diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 5e79c4824..74ffaae38 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -31,8 +31,8 @@ import io.seqera.wave.service.k8s.K8sService import io.seqera.wave.service.k8s.K8sServiceImpl import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User -import jakarta.inject.Inject import io.seqera.wave.util.ContainerHelper +import jakarta.inject.Inject /** * * @author Paolo Di Tommaso @@ -70,7 +70,7 @@ class KubeBuildStrategyTest extends Specification { when: def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), repo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null) - def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)) Files.createDirectories(req.workDir) strategy.build('build-job-name', req) @@ -78,7 +78,7 @@ class KubeBuildStrategyTest extends Specification { 1 * k8sService.launchBuildJob( _, _, _, _, _, _, [service:'wave-build']) >> null when: - def req2 = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def req2 = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)) Files.createDirectories(req2.workDir) strategy.build('job-name', req2) @@ -98,19 +98,19 @@ class KubeBuildStrategyTest extends Specification { when:'getting docker with amd64 arch in build request' def containerId = ContainerHelper.makeContainerId(dockerfile, null, ContainerPlatform.of('amd64'), repo, null) def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null) - def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)).withBuildId('1') + def req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER, Duration.ofMinutes(1)) then: 'should return buildkit image' strategy.getBuildImage(req) == 'moby/buildkit:v0.14.1-rootless' when:'getting singularity with amd64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY,Duration.ofMinutes(1)).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY,Duration.ofMinutes(1)) then:'should return singularity amd64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim' when:'getting singularity with arm64 arch in build request' - req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY, Duration.ofMinutes(1)).withBuildId('1') + req = new BuildRequest(containerId, dockerfile, null, PATH, targetImage, USER, ContainerPlatform.of('arm64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY, Duration.ofMinutes(1)) then:'should return singularity arm64 image' strategy.getBuildImage(req) == 'quay.io/singularity/singularity:v3.11.4-slim-arm64' diff --git a/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy index acffd5b8d..84ad22825 100644 --- a/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/cleanup/CleanupStrategyTest.groovy @@ -38,9 +38,11 @@ class CleanupStrategyTest extends Specification { STATUS | CONFIG | DEBUG | EXPECTED 0 | null | false | true 1 | null | false | true + null | null | false | true and: 0 | null | true | false 1 | null | true | false + null | null | true | false and: 0 | 'always' | false | true 0 | 'never' | false | false @@ -48,6 +50,9 @@ class CleanupStrategyTest extends Specification { 1 | 'always' | false | true 1 | 'never' | false | false 1 | 'onsuccess' | false | false + null | 'always' | false | true + null | 'never' | false | false + null | 'onsuccess' | false | false and: 0 | 'always' | true | true 0 | 'never' | true | false @@ -55,5 +60,8 @@ class CleanupStrategyTest extends Specification { 1 | 'always' | true | true 1 | 'never' | true | false 1 | 'onsuccess' | true | false + null | 'always' | true | true + null | 'never' | true | false + null | 'onsuccess' | true | false } } diff --git a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy index 9d2b695e1..458fb1443 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobFactoryTest.groovy @@ -31,6 +31,7 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest +import io.seqera.wave.tower.PlatformId /** * @@ -44,22 +45,23 @@ class JobFactoryTest extends Specification { def factory = new JobFactory() and: def request = new BuildRequest( + containerId: '12345', targetImage: 'docker.io/foo:bar', - buildId: '12345_9', + buildId: 'bd-12345_9', startTime: ts, maxDuration: Duration.ofMinutes(1), - workDir: Path.of('/some/work/dir') + workspace: Path.of('/some/work/dir') ) when: def job = factory.build(request) then: job.entryKey == 'docker.io/foo:bar' - job.operationName == 'build-12345-9' + job.operationName == 'bd-12345-9' job.creationTime == ts job.type == JobSpec.Type.Build job.maxDuration == Duration.ofMinutes(1) - job.workDir == Path.of('/some/work/dir') + job.workDir == Path.of('/some/work/dir/bd-12345_9') } def 'should create transfer job' () { @@ -83,20 +85,20 @@ class JobFactoryTest extends Specification { def duration = Duration.ofMinutes(1) def config = new ScanConfig(timeout: duration) def factory = new JobFactory(scanConfig: config) - def request = new ScanRequest( - '12345', - 'build-123', - '{ jsonConfig }', - 'docker.io/foo:bar', - ContainerPlatform.of('linux/amd64'), - workdir + def request = ScanRequest.of( + scanId: 'sc-12345_1', + buildId: 'bd-67890_2', + configJson: '{ jsonConfig }', + targetImage: 'docker.io/foo:bar', + platform: ContainerPlatform.of('linux/amd64'), + workDir: workdir ) when: def job = factory.scan(request) then: - job.entryKey == '12345' - job.operationName == 'scan-12345' + job.entryKey == 'sc-12345_1' + job.operationName == 'sc-12345-1' job.type == JobSpec.Type.Scan job.maxDuration == duration job.creationTime == request.creationTime @@ -116,16 +118,22 @@ class JobFactoryTest extends Specification { 'sha256:12345', Mock(ContainerPlatform), workspace, - '{config}' ) + '{config}', + 'sc-123', + Instant.now(), + "GMT", + Mock(PlatformId) + ) when: def job = factory.mirror(request) then: job.entryKey == "target/foo" - job.operationName == /mirror-${request.mirrorId.substring(3)}/ + job.operationName == request.mirrorId + job.operationName =~ /mr-.+/ job.type == JobSpec.Type.Mirror job.maxDuration == duration - job.workDir == workspace.resolve(/mirror-${request.mirrorId.substring(3)}/) + job.workDir == workspace.resolve(request.mirrorId) job.creationTime == request.creationTime } } diff --git a/src/test/groovy/io/seqera/wave/service/logs/BuildLogsServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/logs/BuildLogsServiceTest.groovy index 6cbf7fb2c..c0d1e3423 100644 --- a/src/test/groovy/io/seqera/wave/service/logs/BuildLogsServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/logs/BuildLogsServiceTest.groovy @@ -21,11 +21,20 @@ package io.seqera.wave.service.logs import spock.lang.Specification import spock.lang.Unroll +import io.micronaut.objectstorage.InputStreamMapper +import io.micronaut.objectstorage.aws.AwsS3Configuration +import io.micronaut.objectstorage.aws.AwsS3Operations +import io.seqera.wave.test.AwsS3TestContainer +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client + /** * * @author Paolo Di Tommaso */ -class BuildLogsServiceTest extends Specification { +class BuildLogsServiceTest extends Specification implements AwsS3TestContainer { @Unroll def 'should make log key name' () { @@ -40,4 +49,107 @@ class BuildLogsServiceTest extends Specification { '/foo/bar/' | '123' | 'foo/bar/123.log' } + def 'should remove conda lockfile from logs' () { + def logs = """ + #9 12.23 logs.... + #10 12.24 >> CONDA_LOCK_START + #10 12.24 # This file may be used to create an environment using: + #10 12.24 # \$ conda create --name --file + #10 12.24 # platform: linux-aarch64 + #10 12.24 @EXPLICIT + #10 12.25 << CONDA_LOCK_END + #11 12.26 logs....""".stripIndent() + def service = new BuildLogServiceImpl() + + when: + def result = service.removeCondaLockFile(logs) + then: + result == """ + #9 12.23 logs.... + #11 12.26 logs....""".stripIndent() + } + + @Unroll + def 'should make conda lock key name' () { + expect: + new BuildLogServiceImpl(condaLockPrefix: PREFIX).condaLockKey(BUILD) == EXPECTED + + where: + PREFIX | BUILD | EXPECTED + null | null | null + null | '123' | '123.lock' + 'foo' | '123' | 'foo/123.lock' + '/foo/bar/' | '123' | 'foo/bar/123.lock' + } + + def 'should extract conda lockfile' () { + def logs = """ + #9 12.23 logs.... + #10 12.24 >> CONDA_LOCK_START + #10 12.24 # This file may be used to create an environment using: + #10 12.24 # \$ conda create --name --file + #10 12.24 # platform: linux-aarch64 + #10 12.24 @EXPLICIT + #10 12.25 << CONDA_LOCK_END + #11 12.26 logs....""".stripIndent() + def service = new BuildLogServiceImpl() + + when: + def result = service.extractCondaLockFile(logs) + + then: + result == """ + # This file may be used to create an environment using: + # \$ conda create --name --file + # platform: linux-aarch64 + @EXPLICIT + """.stripIndent() + } + + def 'should extract conda lockfile from s3' (){ + given: + def s3Client = S3Client.builder() + .endpointOverride(URI.create("http://${awsS3HostName}:${awsS3Port}")) + .region(Region.EU_WEST_1) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("accesskey", "secretkey"))) + .forcePathStyle(true) + .build() + + + def inputStreamMapper = Mock(InputStreamMapper) + + and: "create s3 bucket" + def storageBucket = "test-bucket" + s3Client.createBucket { it.bucket(storageBucket) } + and: + def configuration = new AwsS3Configuration('build-logs') + configuration.setBucket(storageBucket) + and: + AwsS3Operations awsS3Operations = new AwsS3Operations(configuration, s3Client, inputStreamMapper) + and: + def service = new BuildLogServiceImpl(objectStorageOperations: awsS3Operations, condaLockPrefix: "build-logs/conda-lock") + and: + def buildID = "123" + def logs = """ + #9 12.23 logs.... + #10 12.24 >> CONDA_LOCK_START + #10 12.24 # This file may be used to create an environment using: + #10 12.24 # \$ conda create --name --file + #10 12.24 # platform: linux-aarch64 + #10 12.24 @EXPLICIT + #10 12.25 << CONDA_LOCK_END + #11 12.26 logs....""".stripIndent() + + when: + service.storeCondaLock(buildID, logs) + + then: + service.fetchCondaLockString(buildID) == """ + # This file may be used to create an environment using: + # \$ conda create --name --file + # platform: linux-aarch64 + @EXPLICIT + """.stripIndent() + } + } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy index 7bf0a287e..d892af4f6 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy @@ -62,7 +62,7 @@ class ContainerMirrorServiceTest extends Specification { def source = 'docker.io/hello-world:latest' def target = 'docker.io/pditommaso/wave-tests' def folder = Files.createTempDirectory('test') - println "Temp path: $folder" + when: def creds = dockerAuthService.credentialsConfigJson(null, source, target, Mock(PlatformId)) def request = MirrorRequest.create( @@ -71,13 +71,18 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, folder, - creds ) + creds, + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) and: mirrorService.mirrorImage(request) then: mirrorService.awaitCompletion(target) .get(90, TimeUnit.SECONDS) - .succeeded() + .done() cleanup: folder?.deleteDir() @@ -91,13 +96,18 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/some/dir'), - '{config}' ) + '{config}', + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) and: - def state = MirrorEntry.from(request) + def state = MirrorResult.of(request) and: - persistenceService.saveMirrorEntry(state) + persistenceService.saveMirrorResult(state) when: - def copy = mirrorService.getMirrorEntry(request.mirrorId) + def copy = mirrorService.getMirrorResult(request.mirrorId) then: copy == state } @@ -110,15 +120,20 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/some/dir'), - '{config}' ) + '{config}', + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) and: - def state = MirrorEntry.from(request) + def state = MirrorEntry.of(request) and: mirrorStateStore.put('target/foo', state) when: - def copy = mirrorService.getMirrorEntry(request.mirrorId) + def result = mirrorService.getMirrorResult(request.mirrorId) then: - copy == state + result == state.result } def 'should update mirror state on job completion' () { @@ -129,9 +144,14 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/some/dir'), - '{config}' ) - and: - def state = MirrorEntry.from(request) + '{config}', + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) + and: + def state = MirrorEntry.of(request) def job = JobSpec.mirror(request.mirrorId, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) when: mirrorService.onJobCompletion(job, state, new JobState(JobState.Status.SUCCEEDED, 0, 'OK')) @@ -139,13 +159,13 @@ class ContainerMirrorServiceTest extends Specification { def s1 = mirrorStateStore.get(request.targetImage) and: s1.done() - s1.succeeded() - s1.exitCode == 0 - s1.logs == 'OK' + s1.result.succeeded() + s1.result.exitCode == 0 + s1.result.logs == 'OK' and: - def s2 = persistenceService.loadMirrorEntry(request.mirrorId) + def s2 = persistenceService.loadMirrorResult(request.mirrorId) and: - s2 == s1 + s2 == s1.result } def 'should update mirror state on job exception' () { @@ -156,9 +176,14 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/some/dir'), - '{config}' ) - and: - def state = MirrorEntry.from(request) + '{config}', + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) + and: + def state = MirrorEntry.of(request) def job = JobSpec.mirror(request.mirrorId, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) when: mirrorService.onJobException(job, state, new Exception('Oops something went wrong')) @@ -166,13 +191,14 @@ class ContainerMirrorServiceTest extends Specification { def s1 = mirrorStateStore.get(request.targetImage) and: s1.done() - !s1.succeeded() - s1.exitCode == null - s1.logs == 'Oops something went wrong' and: - def s2 = persistenceService.loadMirrorEntry(request.mirrorId) + !s1.result.succeeded() + s1.result.exitCode == null + s1.result.logs == 'Oops something went wrong' and: - s2 == s1 + def s2 = persistenceService.loadMirrorResult(request.mirrorId) + and: + s2 == s1.result } def 'should update mirror state on job timeout' () { @@ -183,9 +209,14 @@ class ContainerMirrorServiceTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/some/dir'), - '{config}' ) - and: - def state = MirrorEntry.from(request) + '{config}', + null, + Instant.now(), + 'GMT', + Mock(PlatformId) + ) + and: + def state = MirrorEntry.of(request) def job = JobSpec.mirror(request.mirrorId, 'mirror-123', Instant.now(), Duration.ofMillis(1), Mock(Path)) when: mirrorService.onJobTimeout(job, state) @@ -193,13 +224,14 @@ class ContainerMirrorServiceTest extends Specification { def s1 = mirrorStateStore.get(request.targetImage) and: s1.done() - !s1.succeeded() - s1.exitCode == null - s1.logs == 'Container mirror timed out' and: - def s2 = persistenceService.loadMirrorEntry(request.mirrorId) + !s1.result.succeeded() + s1.result.exitCode == null + s1.result.logs == 'Container mirror timed out' + and: + def s2 = persistenceService.loadMirrorResult(request.mirrorId) and: - s2 == s1 + s2 == s1.result } } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorEntryTest.groovy index 6dd19e6f6..d6dff6190 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/MirrorEntryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorEntryTest.groovy @@ -23,99 +23,70 @@ import spock.lang.Specification import java.nio.file.Path import java.time.Instant -import io.seqera.wave.api.BuildStatusResponse import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId + /** * * @author Paolo Di Tommaso */ class MirrorEntryTest extends Specification { - def 'should create a result from a request' () { + def 'should create mirror entry object' () { given: + def ts = Instant.now() def request = MirrorRequest.create( 'source.io/foo', 'target.io/foo', 'sha256:12345', Mock(ContainerPlatform), Path.of('/workspace'), - '{auth json}' ) + '{auth json}', + 'scan-123', + ts, + 'utc', + Mock(PlatformId) + ) when: - def result = MirrorEntry.from(request) + def entry = MirrorEntry.of(request) then: - result.mirrorId == request.mirrorId - result.digest == request.digest - result.platform == request.platform - result.sourceImage == request.sourceImage - result.targetImage == request.targetImage - result.creationTime == request.creationTime - result.status == MirrorEntry.Status.PENDING + entry.requestId == request.mirrorId + entry.key == request.targetImage + and: + entry.request == request and: - result.duration == null - result.exitCode == null - result.logs == null + entry.result == MirrorResult.of(request) } - def 'should complete a result result' () { + def 'should validate with result' () { given: + def ts = Instant.now() def request = MirrorRequest.create( 'source.io/foo', 'target.io/foo', 'sha256:12345', Mock(ContainerPlatform), Path.of('/workspace'), - '{auth json}' ) + '{auth json}', + 'scan-123', + ts, + 'utc', + Mock(PlatformId) + ) when: - def m1 = MirrorEntry.from(request) + def entry = MirrorEntry.of(request) then: - m1.status == MirrorEntry.Status.PENDING - m1.duration == null - m1.exitCode == null - m1.logs == null + !entry.done() when: - def m2 = m1.complete(0, 'Some logs') + entry = entry.withResult( entry.result.complete(0, 'All ok') ) then: - m2.mirrorId == request.mirrorId - m2.digest == request.digest - m2.sourceImage == request.sourceImage - m2.targetImage == request.targetImage - m2.creationTime == request.creationTime - m2.platform == request.platform - and: - m2.status == MirrorEntry.Status.COMPLETED - m2.duration != null - m2.exitCode == 0 - m2.logs == 'Some logs' + entry.done() + entry.result.succeeded() + entry.result.exitCode == 0 + entry.result.logs == 'All ok' } - def 'should convert to status response' () { - when: - def result1 = new MirrorEntry('mr-123', 'sha256:12345', 'source/foo', 'target/foo', Mock(ContainerPlatform), Instant.now()) - def resp = result1.toStatusResponse() - then: - resp.id == result1.mirrorId - resp.status == BuildStatusResponse.Status.PENDING - resp.startTime == result1.creationTime - and: - resp.duration == null - resp.succeeded == null - - when: - final result2 = result1.complete(1, 'Some error') - final resp2 = result2.toStatusResponse() - then: - resp2.duration == result2.duration - resp2.succeeded == false - - when: - final result3 = result1.complete(0, 'OK') - final resp3 = result3.toStatusResponse() - then: - resp3.duration == result3.duration - resp3.succeeded == true - - } } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy index bc386d875..2709e7a10 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorRequestTest.groovy @@ -24,6 +24,7 @@ import java.nio.file.Path import java.time.Instant import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId /** * @@ -40,17 +41,25 @@ class MirrorRequestTest extends Specification { 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/workspace'), - '{json config}') + '{json config}', + 'sc-123', + ts, + 'utc', + Mock(PlatformId) + ) then: req.mirrorId req.sourceImage == 'docker.io/foo:latest' req.targetImage == 'quay.io/foo:latest' req.digest == 'sha256:12345' req.platform == ContainerPlatform.DEFAULT - req.workDir == Path.of("/workspace/mirror-${req.mirrorId.substring(3)}") + req.workDir == Path.of("/workspace/${req.mirrorId}") req.authJson == '{json config}' req.creationTime >= ts req.creationTime <= Instant.now() + req.scanId == 'sc-123' + req.creationTime == ts + req.offsetId == 'utc' } } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy new file mode 100644 index 000000000..78f945dde --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorResultTest.groovy @@ -0,0 +1,115 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Instant + +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +/** + * + * @author Paolo Di Tommaso + */ +class MirrorResultTest extends Specification { + + def 'should create a result from a request' () { + given: + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + Mock(ContainerPlatform), + Path.of('/workspace'), + '{auth json}', + 'scan-123', + Instant.now(), + 'GMT', + Mock(PlatformId) + ) + + when: + def result = MirrorResult.of(request) + then: + result.mirrorId == request.mirrorId + result.digest == request.digest + result.platform == request.platform + result.sourceImage == request.sourceImage + result.targetImage == request.targetImage + result.creationTime == request.creationTime + result.status == MirrorResult.Status.PENDING + and: + result.duration == null + result.exitCode == null + result.logs == null + } + + def 'should complete a result result' () { + given: + def ts = Instant.now() + def request = MirrorRequest.create( + 'source.io/foo', + 'target.io/foo', + 'sha256:12345', + Mock(ContainerPlatform), + Path.of('/workspace'), + '{auth json}', + 'scan-123', + ts, + 'utc+1', + new PlatformId(new User(id: 1, userName: 'me', email: 'me@host.com')) + ) + + when: + def m1 = MirrorResult.of(request) + then: + m1.status == MirrorResult.Status.PENDING + m1.userId == 1 + m1.userName == 'me' + m1.userEmail == 'me@host.com' + m1.duration == null + m1.exitCode == null + m1.logs == null + + when: + def m2 = m1.complete(0, 'Some logs') + then: + m2.mirrorId == request.mirrorId + m2.digest == request.digest + m2.sourceImage == request.sourceImage + m2.targetImage == request.targetImage + m2.creationTime == request.creationTime + m2.platform == request.platform + m2.creationTime == request.creationTime + m2.offsetId == request.offsetId + m2.userId == 1 + m2.userName == 'me' + m2.userEmail == 'me@host.com' + and: + m2.status == MirrorResult.Status.COMPLETED + m2.duration != null + m2.exitCode == 0 + m2.logs == 'Some logs' + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/MirrorStateStoreTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/MirrorStateStoreTest.groovy new file mode 100644 index 000000000..9c3f6e187 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/mirror/MirrorStateStoreTest.groovy @@ -0,0 +1,45 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.mirror + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class MirrorStateStoreTest extends Specification { + + @Inject + MirrorStateStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-mirror/v1:foo' + } + + def 'should return record id' () { + expect: + store.requestId0('foo') == 'wave-mirror/v1/request-id:foo' + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy index 99d8205c5..1d1c48e59 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/strategy/MirrorStrategyTest.groovy @@ -18,14 +18,15 @@ package io.seqera.wave.service.mirror.strategy -import io.seqera.wave.service.mirror.MirrorRequest - import spock.lang.Specification import spock.lang.Unroll import java.nio.file.Path +import java.time.Instant import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.mirror.MirrorRequest +import io.seqera.wave.tower.PlatformId /** * @@ -44,7 +45,12 @@ class MirrorStrategyTest extends Specification { 'sha256:12345', PLATFORM ? ContainerPlatform.of(PLATFORM) : null, Path.of('/workspace'), - '{auth json}') + '{auth json}', + 'scan-123', + Instant.now(), + 'GMT', + Mock(PlatformId) + ) when: def cmd = strategy.copyCommand(request) then: @@ -55,7 +61,7 @@ class MirrorStrategyTest extends Specification { null | "copy --preserve-digests --multi-arch all docker://source.io/foo docker://target.io/foo" 'linux/amd64' | "--override-os linux --override-arch amd64 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" 'linux/arm64' | "--override-os linux --override-arch arm64 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" - 'linux/arm64/7'| "--override-os linux --override-arch arm64 --override-variant 7 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" + 'linux/arm64/7' | "--override-os linux --override-arch arm64 --override-variant 7 copy --preserve-digests --multi-arch system docker://source.io/foo docker://target.io/foo" } diff --git a/src/test/groovy/io/seqera/wave/service/pairing/PairingStoreTest.groovy b/src/test/groovy/io/seqera/wave/service/pairing/PairingStoreTest.groovy index 260174d0f..cef19ee0b 100644 --- a/src/test/groovy/io/seqera/wave/service/pairing/PairingStoreTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/pairing/PairingStoreTest.groovy @@ -22,10 +22,21 @@ import spock.lang.Specification import java.time.Instant +import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.store.state.impl.LocalStateProvider +import jakarta.inject.Inject +@MicronautTest class PairingStoreTest extends Specification{ + @Inject + PairingStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'pairing-keys/v1:foo' + } + def 'pairing cache store properly serializes/deserializes pairing record'() { given: 'a store' final store = new PairingStore(new LocalStateProvider()) diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index 6b4fef59a..bf820fd2e 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -25,6 +25,7 @@ import java.nio.file.Path import java.time.Duration import java.time.Instant +import io.seqera.wave.api.BuildStatusResponse import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -32,8 +33,6 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.JacksonHelper -import io.seqera.wave.api.BuildStatusResponse - /** * * @author Paolo Di Tommaso @@ -58,7 +57,7 @@ class WaveBuildRecordTest extends Specification { 'scan12345', null, BuildFormat.DOCKER, - Duration.ofMinutes(1) + Duration.ofMinutes(1), ) final result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(request, result) @@ -89,8 +88,8 @@ class WaveBuildRecordTest extends Specification { 'scan12345', null, BuildFormat.DOCKER, - Duration.ofMinutes(1) - ).withBuildId('123') + Duration.ofMinutes(1), + ) and: final event = new BuildEvent(request) @@ -144,8 +143,8 @@ class WaveBuildRecordTest extends Specification { 'scan12345', null, BuildFormat.DOCKER, - Duration.ofMinutes(1) - ).withBuildId('123') + Duration.ofMinutes(1), + ) final result = new BuildResult(request.buildId, EXIT, "ok", Instant.now(), DURATION, null) final event = new BuildEvent(request, result) diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy index 035586280..ee7895573 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveContainerRecordTest.groovy @@ -27,7 +27,7 @@ import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ContainerLayer import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -52,15 +52,15 @@ class WaveContainerRecordTest extends Specification { fingerprint: 'xyz', timestamp: Instant.now().toString() ) and: + def token = '1234' def user = new User(id:1) - def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world', 'some docker', cfg, 'some conda') + def data = ContainerRequest.of(requestId: token, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: 'some docker', containerConfig: cfg, condaFile: 'some conda') def wave = 'https://wave.io/some/container:latest' def addr = '100.200.300.400' when: def exp = Instant.now().plusSeconds(3600) - def token = '1234' - def container = new WaveContainerRecord(req, data, token, wave, addr, exp) + def container = new WaveContainerRecord(req, data, wave, addr, exp) then: container.id == token container.user == user @@ -96,15 +96,15 @@ class WaveContainerRecordTest extends Specification { cacheRepository: 'cache.docker.io', fingerprint: 'xyz' ) and: + def token = '1234' def user = new User(id:1) - def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world', 'some docker', cfg, 'some conda') + def data = ContainerRequest.of(requestId: token, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: 'some docker', containerConfig: cfg, condaFile: 'some conda') def wave = 'https://wave.io/some/container:latest' def addr = '100.200.300.400' when: def exp = Instant.now().plusSeconds(3600) - def token = '1234' - def container = new WaveContainerRecord(req, data, token, wave, addr, exp) + def container = new WaveContainerRecord(req, data, wave, addr, exp) then: container.id == token container.user == user diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy index 67fd16154..8d06192a0 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveScanRecordTest.groovy @@ -35,8 +35,10 @@ class WaveScanRecordTest extends Specification { given: def startTime = Instant.now() def duration = Duration.ofMinutes(2) - def scanId = '12345' - def buildId = "testbuildid" + def scanId = 'sc-12345' + def buildId = 'bd-1234' + def mirrorId = 'mr-1234' + def requestId = 'cr-1234' def containerImage = "testcontainerimage" def scanVulnerability = new ScanVulnerability( "id1", @@ -49,29 +51,40 @@ class WaveScanRecordTest extends Specification { def vulns = List.of(scanVulnerability) when: - def scanResult= new ScanEntry( + def entry= new ScanEntry( scanId, buildId, + mirrorId, + requestId, containerImage, startTime, duration, 'SUCCEEDED', - vulns) + vulns, + 0, + "Some logs" + ) then: - scanResult.scanId == scanId - scanResult.buildId == buildId - scanResult.containerImage == containerImage - scanResult.isCompleted() - scanResult.isSucceeded() - scanResult.done() + entry.scanId == scanId + entry.buildId == buildId + entry.mirrorId == mirrorId + entry.requestId == requestId + entry.containerImage == containerImage + entry.completed() + entry.succeeded() + entry.done() + entry.exitCode == 0 + entry.logs == "Some logs" when: - def scanRecord = new WaveScanRecord(scanId, scanResult) + def record = new WaveScanRecord(entry) then: - scanRecord.id == scanId - scanRecord.buildId == buildId - scanRecord.vulnerabilities[0] == scanVulnerability + record.id == scanId + record.buildId == buildId + record.vulnerabilities[0] == scanVulnerability + record.exitCode == 0 + record.logs == "Some logs" } } diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index cdbd876aa..61a82e1d7 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -32,13 +32,13 @@ import io.seqera.wave.api.ContainerLayer import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.core.ContainerDigestPair import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult -import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveContainerRecord import io.seqera.wave.service.persistence.WaveScanRecord @@ -105,23 +105,21 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe """ def storage = applicationContext.getBean(SurrealPersistenceService) final request = new BuildRequest( - 'container1234', - dockerFile, - condaFile, - Path.of("."), - 'docker.io/my/repo:container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER, - Duration.ofMinutes(1) - ).withBuildId('1') + containerId: 'container1234', + containerFile: dockerFile, + condaFile: condaFile, + workspace: Path.of("."), + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + cacheRepository: 'docker.io/my/cache', + ip: '127.0.0.1', + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(1), + buildId: '12345_1', + ) def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(request, result) def build = WaveBuildRecord.fromEvent(event) @@ -141,23 +139,21 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe given: def persistence = applicationContext.getBean(SurrealPersistenceService) final request = new BuildRequest( - 'container1234', - 'FROM foo:latest', - 'conda::recipe', - Path.of("."), - 'docker.io/my/repo:container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER, - Duration.ofMinutes(1) - ).withBuildId('123') + containerId: 'container1234', + containerFile: 'FROM foo:latest', + condaFile: 'conda::recipe', + workspace: Path.of("."), + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + cacheRepository: 'docker.io/my/cache', + ip: '127.0.0.1', + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(1), + buildId: '12345_1', + ) def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) def event = new BuildEvent(request, result) def record = WaveBuildRecord.fromEvent(event) @@ -178,9 +174,9 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def surreal = applicationContext.getBean(SurrealClient) def persistence = applicationContext.getBean(SurrealPersistenceService) def auth = persistence.getAuthorization() - def request1 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL ).withBuildId('1') - def request2 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL ).withBuildId('2') - def request3 = new BuildRequest( containerId: 'abc', workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL ).withBuildId('3') + def request1 = new BuildRequest( containerId: 'abc', buildId: 'bd-abc_1' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(30), identity: PlatformId.NULL) + def request2 = new BuildRequest( containerId: 'abc', buildId: 'bd-abc_2' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(20), identity: PlatformId.NULL) + def request3 = new BuildRequest( containerId: 'abc', buildId: 'bd-abc_3' , workspace: Path.of('.'), startTime: Instant.now().minusSeconds(10), identity: PlatformId.NULL) def result1 = new BuildResult(request1.buildId, -1, "ok", request1.startTime, Duration.ofSeconds(2), null) surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request1, result1))) @@ -191,34 +187,31 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def result3 = new BuildResult(request3.buildId, -1, "ok", request3.startTime, Duration.ofSeconds(2), null) surreal.insertBuild(auth, WaveBuildRecord.fromEvent(new BuildEvent(request3, result3))) - when: - def loaded = persistence.latestBuild('abc') - - then: - loaded.buildId == 'abc_3' + expect: + persistence.latestBuild('abc').buildId == 'bd-abc_3' + persistence.latestBuild('bd-abc').buildId == 'bd-abc_3' + persistence.latestBuild('xyz') == null } def 'should save and update a build' () { given: def persistence = applicationContext.getBean(SurrealPersistenceService) final request = new BuildRequest( - 'container1234', - 'FROM foo:latest', - 'conda::recipe', - Path.of("/some/path"), - 'buildrepo:recipe-container1234', - PlatformId.NULL, - ContainerPlatform.of('amd64'), - 'docker.io/my/cache', - '127.0.0.1', - '{"config":"json"}', - null, - null, - 'scan12345', - null, - BuildFormat.DOCKER, - Duration.ofMinutes(1) - ).withBuildId('123') + containerId: 'container1234', + containerFile: 'FROM foo:latest', + condaFile: 'conda::recipe', + workspace: Path.of("/some/path"), + targetImage: 'buildrepo:recipe-container1234', + identity: PlatformId.NULL, + platform: ContainerPlatform.of('amd64'), + cacheRepository: 'docker.io/my/cache', + ip: '127.0.0.1', + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(1), + buildId: '12345_1' + ) and: def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') @@ -252,13 +245,12 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe timestamp: Instant.now().toString() ) def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world', largeContainerFile ) + def data = ContainerRequest.of(requestId: TOKEN, identity: new PlatformId(user,100), containerImage: 'hello-world', containerFile: largeContainerFile ) def wave = "wave.io/wt/$TOKEN/hello-world" def addr = "100.200.300.400" def exp = Instant.now().plusSeconds(3600) and: - def request = new WaveContainerRecord(req, data, TOKEN, wave, addr, exp) - + def request = new WaveContainerRecord(req, data, wave, addr, exp) and: persistence.saveContainerRequest(request) and: @@ -289,6 +281,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def 'should save a scan and load a result' () { given: def persistence = applicationContext.getBean(SurrealPersistenceService) + def auth = persistence.getAuthorization() + def surrealDb = applicationContext.getBean(SurrealClient) def NOW = Instant.now() def SCAN_ID = 'a1' def BUILD_ID = '100' @@ -296,27 +290,36 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') - def scanRecord = new WaveScanRecord(SCAN_ID, BUILD_ID, CONTAINER_IMAGE, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3]) + def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') + def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: - persistence.createScanRecord(new WaveScanRecord(SCAN_ID, BUILD_ID, CONTAINER_IMAGE, NOW)) - persistence.updateScanRecord(scanRecord) + persistence.saveScanRecord(scan) then: def result = persistence.loadScanRecord(SCAN_ID) and: - result == scanRecord + result == scan + and: + surrealDb + .sqlAsMap(auth, "select * from wave_scan_vuln") + .result + .size() == 3 when: def SCAN_ID2 = 'b2' def BUILD_ID2 = '102' - def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, CONTAINER_IMAGE, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1]) + def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, null, null, CONTAINER_IMAGE, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1, CVE4], 1, "Error 'quote'") and: - persistence.createScanRecord(new WaveScanRecord(SCAN_ID2, BUILD_ID2, CONTAINER_IMAGE, NOW)) // should save the same CVE into another build - persistence.updateScanRecord(scanRecord2) + persistence.saveScanRecord(scanRecord2) then: def result2 = persistence.loadScanRecord(SCAN_ID2) and: result2 == scanRecord2 + and: + surrealDb + .sqlAsMap(auth, "select * from wave_scan_vuln") + .result + .size() == 4 } //== mirror records tests @@ -331,16 +334,21 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/workspace'), - '{auth json}' ) + '{auth json}', + 'scan-123', + Instant.now(), + "GMT", + Mock(PlatformId) + ) and: storage.initializeDb() and: - def result = MirrorEntry.from(request) - storage.saveMirrorEntry(result) + def result = MirrorEntry.of(request).getResult() + storage.saveMirrorResult(result) sleep 100 when: - def stored = storage.loadMirrorEntry(request.mirrorId) + def stored = storage.loadMirrorResult(request.mirrorId) then: stored == result } @@ -355,16 +363,21 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe 'sha256:12345', ContainerPlatform.DEFAULT, Path.of('/workspace'), - '{auth json}' ) + '{auth json}', + 'scan-123', + Instant.now(), + "GMT", + Mock(PlatformId) + ) and: storage.initializeDb() and: - def result = MirrorEntry.from(request) - storage.saveMirrorEntry(result) + def result = MirrorEntry.of(request).getResult() + storage.saveMirrorResult(result) sleep 100 when: - def stored = storage.loadMirrorEntry(request.targetImage, request.digest) + def stored = storage.loadMirrorResult(request.targetImage, request.digest) then: stored == result } @@ -398,7 +411,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe null, BuildFormat.DOCKER, Duration.ofMinutes(1) - ).withBuildId('123') + ) and: def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') diff --git a/src/test/groovy/io/seqera/wave/service/token/ContainerTokenServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestServiceImplTest.groovy similarity index 76% rename from src/test/groovy/io/seqera/wave/service/token/ContainerTokenServiceImplTest.groovy rename to src/test/groovy/io/seqera/wave/service/request/ContainerRequestServiceImplTest.groovy index 780994a63..014deb630 100644 --- a/src/test/groovy/io/seqera/wave/service/token/ContainerTokenServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestServiceImplTest.groovy @@ -16,13 +16,12 @@ * along with this program. If not, see . */ -package io.seqera.wave.service.token +package io.seqera.wave.service.request import spock.lang.Specification import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.TokenConfig -import io.seqera.wave.service.ContainerRequestData import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User import jakarta.inject.Inject @@ -31,27 +30,28 @@ import jakarta.inject.Inject * @author Munish Chouhan */ @MicronautTest -class ContainerTokenServiceImplTest extends Specification { +class ContainerRequestServiceImplTest extends Specification { @Inject private TokenConfig config @Inject - private ContainerTokenStoreImpl tokenCache + private ContainerRequestStoreImpl requestStore def 'should evict container request from cache'(){ given: - def containerTokenService = new ContainerTokenServiceImpl( tokenCache: tokenCache, config: config ) + def containerTokenService = new ContainerRequestServiceImpl( tokenCache: requestStore, config: config ) def TOKEN = '123abc' def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') - def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world') + def data = ContainerRequest.of(identity: new PlatformId(user,100), containerImage: 'hello-world') and: - tokenCache.put(TOKEN, data) + requestStore.put(TOKEN, data) when: def request = containerTokenService.evictRequest(TOKEN) then: request == data - tokenCache.get(TOKEN) == null + requestStore.get(TOKEN) == null } + } diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerRequestStoreImplTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestStoreImplTest.groovy new file mode 100644 index 000000000..6fd381257 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestStoreImplTest.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class ContainerRequestStoreImplTest extends Specification { + + @Inject ContainerRequestStoreImpl store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-tokens/v1:foo' + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy similarity index 68% rename from src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy rename to src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy index fd1c647ac..bc95e9cda 100644 --- a/src/test/groovy/io/seqera/wave/service/ContainerRequestDataTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy @@ -16,11 +16,15 @@ * along with this program. If not, see . */ -package io.seqera.wave.service +package io.seqera.wave.service.request import spock.lang.Specification +import java.time.Instant + import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ScanLevel +import io.seqera.wave.api.ScanMode import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -29,14 +33,14 @@ import io.seqera.wave.tower.User * * @author Paolo Di Tommaso */ -class ContainerRequestDataTest extends Specification { +class ContainerRequestTest extends Specification { def 'should return request identity' () { given: - ContainerRequestData req + ContainerRequest req when: - req = new ContainerRequestData(new PlatformId(new User(id:1))) + req = ContainerRequest.of(new PlatformId(new User(id:1))) then: req.identity req.identity == new PlatformId(new User(id:1)) @@ -44,45 +48,52 @@ class ContainerRequestDataTest extends Specification { def 'should validate constructor' () { when: + def ts = Instant.now() def cfg = Mock(ContainerConfig) - def req = new ContainerRequestData( + def req = new ContainerRequest( + 'r-1234', new PlatformId(new User(id:1)), 'foo', 'from docker', cfg, 'conda file', ContainerPlatform.DEFAULT, - '12345', + 'build-12345', + true, true, true, - true ) + 'scan-1234', + ScanMode.required, + List.of(ScanLevel.HIGH), + true, + ts + ) then: + req.requestId == 'r-1234' req.identity == new PlatformId(new User(id:1)) req.containerImage == 'foo' req.containerFile == 'from docker' req.containerConfig == cfg req.condaFile == 'conda file' req.platform == ContainerPlatform.DEFAULT - req.buildId == '12345' + req.buildId == 'build-12345' req.buildNew req.freeze req.mirror + req.scanId == 'scan-1234' + req.scanMode == ScanMode.required + req.scanLevels == List.of(ScanLevel.HIGH) + req.scanOnRequest == true + req.creationTime == ts } def 'should validate durable flag' () { given: - def req = new ContainerRequestData( - new PlatformId(new User(id:1)), - null, - null, - null, - null, - null, - null, - null, - FREEZE, - MIRROR ) + def req = ContainerRequest.of( + identity: new PlatformId(new User(id:1)), + freeze: FREEZE, + mirror: MIRROR ) expect: req.durable() == EXPECTED diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy new file mode 100644 index 000000000..a4eb51604 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy @@ -0,0 +1,622 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import java.time.Duration +import java.time.Instant + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.ContainerStatus +import io.seqera.wave.api.ScanLevel +import io.seqera.wave.api.ScanMode +import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.service.scan.ScanEntry +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class ContainerStatusServiceTest extends Specification { + + def 'should create scan result' () { + given: + def service = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com') + def request = Mock(ContainerRequest) + def scan = Mock(ScanEntry) + + when: + def res1 = service.scanResult(request, scan) + then: + scan.succeeded() >> false + request.getScanId() >> 'scan-123' + and: + res1 == new ContainerStatusServiceImpl.StageResult( + false, + "Container security scan did not complete successfully", + "http://foo.com/view/scans/scan-123" + ) + + when: + def res2 = service.scanResult(request, scan) + then: + scan.succeeded() >> true + scan.summary() >> [MEDIUM: 5] + request.getScanId() >> 'scan-123' + request.getScanLevels() >> null + and: + res2 == new ContainerStatusServiceImpl.StageResult( + false, + "Container security scan operation found one or more vulnerabilities with severity: MEDIUM", + "http://foo.com/view/scans/scan-123" + ) + + when: + def res3 = service.scanResult(request, scan) + then: + scan.succeeded() >> true + scan.summary() >> [HIGH:2, CRITICAL:1, MEDIUM: 5] + request.getScanId() >> 'scan-123' + request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] + and: + res3 == new ContainerStatusServiceImpl.StageResult( + false, + "Container security scan operation found one or more vulnerabilities with severity: HIGH,CRITICAL", + "http://foo.com/view/scans/scan-123" + ) + + when: + def res4 = service.scanResult(request, scan) + then: + scan.succeeded() >> true + scan.summary() >> [MEDIUM: 5] + request.getScanId() >> 'scan-123' + request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] + and: + res4 == new ContainerStatusServiceImpl.StageResult( + true, + "Container security scan operation found one or more vulnerabilities that are compatible with requested security levels: LOW,MEDIUM", + "http://foo.com/view/scans/scan-123" + ) + + when: + def res5 = service.scanResult(request, scan) + then: + scan.succeeded() >> true + scan.summary() >> [:] + request.getScanId() >> 'scan-123' + request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] + and: + res5 == new ContainerStatusServiceImpl.StageResult(true) + + } + + def 'should validate status for build running' () { + given: + def startTime = Instant.now().minusSeconds(10) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building is running + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime) + requestData.buildId >> 'build-123' + requestData.requestId >> requestId + and: + resp.id == requestId + resp.status == ContainerStatus.BUILDING + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == null + resp.creationTime == startTime + resp.duration == null + resp.succeeded == null + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def "should validate status for build successful and no scan" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building is successful, no scan + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + and: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == null + resp.creationTime == startTime + resp.duration == _1min + resp.succeeded == true + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def "should validate status build failed" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building failed + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, false) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + and: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == null + resp.creationTime == startTime + resp.duration == _1min + resp.succeeded == false + resp.vulnerabilities == null + resp.reason == "Container build did not complete successfully" + resp.detailsUri == "http://foo.com/view/builds/build-123" + } + + def "should validate status for build successful and scan running" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan is running + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) + then: + resp.id == requestId + resp.status == ContainerStatus.SCANNING + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == null + resp.succeeded == null + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + def "should validate status for build successful and security scan successful" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan is successful + when: + def resp5 = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true } + then: + resp5.id == requestId + resp5.status == ContainerStatus.DONE + resp5.buildId == "build-123" + resp5.mirrorId == null + resp5.scanId == 'scan-abc' + resp5.creationTime == startTime + resp5.duration == _1min + _2min // build + scan time + resp5.succeeded == true + resp5.vulnerabilities == null + resp5.reason == null + resp5.detailsUri == null + } + + def "should validate status for build successful and scan failed" ( ) { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan failed + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>false } + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1min + _2min // build + scan time + resp.succeeded == false + resp.vulnerabilities == null + resp.reason == "Container security scan did not complete successfully" + resp.detailsUri == "http://foo.com/view/scans/scan-abc" + } + + def "should validate status for build successful and scan with vulnerabilities" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan found vulnerabilities + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1min + _2min // build + scan time + resp.succeeded == false + resp.vulnerabilities == [HIGH:1] + resp.reason == "Container security scan operation found one or more vulnerabilities with severity: HIGH" + resp.detailsUri == "http://foo.com/view/scans/scan-abc" + } + + def "should validate status for build successful and scan with not allowed vulnerabilities" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan has vulnerabilities that are allowed + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of(ScanLevel.MEDIUM,ScanLevel.HIGH) + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1min + _2min // build + scan time + resp.succeeded == true + resp.vulnerabilities == [HIGH:1] + resp.reason == "Container security scan operation found one or more vulnerabilities that are compatible with requested security levels: MEDIUM,HIGH" + resp.detailsUri == "http://foo.com/view/scans/scan-abc" + } + + def 'should validate status for build is successful and security scan is not enabled' () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.none + requestData.scanLevels >> List.of(ScanLevel.MEDIUM,ScanLevel.HIGH) + and: + 0 * scanService.getScanState('scan-abc') >> null + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == "build-123" + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1min + resp.succeeded == true + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def "should validate status for container request and scan with vulnerabilities ie. failed" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan found vulnerabilities + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> null + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getStartTime()>>startTime } + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == null + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1min + _2min // request time + scan time + resp.succeeded == false + resp.vulnerabilities == [HIGH:1] + resp.reason == "Container security scan operation found one or more vulnerabilities with severity: HIGH" + resp.detailsUri == "http://foo.com/view/scans/scan-abc" + } + + def "should validate status for container request and scan with no vulnerabilities" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1sec = Duration.ofSeconds(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // container build is successful, security scan found vulnerabilities + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1sec, true) + requestData.requestId >> requestId + requestData.buildId >> null + requestData.scanId >> 'scan-abc' + requestData.scanMode >> ScanMode.required + requestData.scanLevels >> List.of() + and: + scanService.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>null; getStartTime()>>startTime } + then: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == null + resp.mirrorId == null + resp.scanId == 'scan-abc' + resp.creationTime == startTime + resp.duration == _1sec + _2min // request time + scan time + resp.succeeded == true + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def 'should validate status for mirror running' () { + given: + def startTime = Instant.now().minusSeconds(10) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building is running + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.mirror >> true + and: + resp.id == requestId + resp.status == ContainerStatus.BUILDING + resp.buildId == null + resp.mirrorId == "build-123" + resp.scanId == null + resp.creationTime == startTime + resp.duration == null + resp.succeeded == null + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def "should validate status for mirror successful and no scan" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building is successful, no scan + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, true) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.mirror >> true + and: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == null + resp.mirrorId == "build-123" + resp.scanId == null + resp.creationTime == startTime + resp.duration == _1min + resp.succeeded == true + resp.vulnerabilities == null + resp.reason == null + resp.detailsUri == null + } + + def "should validate status for mirror failed" () { + given: + def startTime = Instant.now().minusSeconds(10) + def _1min = Duration.ofMinutes(1) + def _2min = Duration.ofMinutes(2) + def scanService = Mock(ContainerScanService) + def store = Mock(ContainerRequestStore) + def impl = new ContainerStatusServiceImpl(serverUrl: 'http://foo.com', requestStore: store, scanService:scanService) + def service = Spy(impl) + def requestId = '123456' + def requestData = Mock(ContainerRequest) + + // building is successful, no scan + when: + def resp = service.getContainerStatus(requestId) + then: + store.get(requestId) >> requestData + service.getContainerState(requestData) >> new ContainerState(startTime, _1min, false) + requestData.requestId >> requestId + requestData.buildId >> 'build-123' + requestData.mirror >> true + and: + resp.id == requestId + resp.status == ContainerStatus.DONE + resp.buildId == null + resp.mirrorId == "build-123" + resp.scanId == null + resp.creationTime == startTime + resp.duration == _1min + resp.succeeded == false + resp.vulnerabilities == null + resp.reason == "Container mirror did not complete successfully" + resp.detailsUri == "http://foo.com/view/mirrors/build-123" + } +} diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusTest.groovy new file mode 100644 index 000000000..d8a1c6842 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusTest.groovy @@ -0,0 +1,138 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.request + +import spock.lang.Specification +import spock.lang.Unroll + +import java.time.Duration +import java.time.Instant + +import io.seqera.wave.service.builder.BuildEntry +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildResult +import io.seqera.wave.service.mirror.MirrorEntry +import io.seqera.wave.service.mirror.MirrorRequest +import io.seqera.wave.service.mirror.MirrorResult +import io.seqera.wave.service.persistence.WaveBuildRecord +/** + * + * @author Paolo Di Tommaso + */ +class ContainerStatusTest extends Specification { + + def 'should create status from build entry' () { + given: + def ts = Instant.now().minusSeconds(60) + def request = Mock(BuildRequest) + def result = Mock(BuildResult) + + when: + def state1 = ContainerState.from(new BuildEntry(request, result)) + then: + request.getStartTime() >> ts + and: + state1.startTime == ts + state1.duration == null + !state1.succeeded + + when: + def state2 = ContainerState.from(new BuildEntry(request, result)) + then: + request.getStartTime() >> ts + result.getDuration() >> Duration.ofMinutes(1) + result.succeeded() >> true + and: + state2.startTime == ts + state2.duration >= Duration.ofMinutes(1) + state2.succeeded + } + + @Unroll + def 'should create status from build record' () { + given: + def ts = Instant.now().minusSeconds(60) + def build = Mock(WaveBuildRecord) + + when: + def state = ContainerState.from(build) + then: + build.getStartTime() >> TIME + build.duration >> DURATION + build.succeeded() >> SUCCEEDED + and: + state.startTime == TIME + state.duration == DURATION + state.succeeded == SUCCEEDED + + where: + TIME | DURATION | SUCCEEDED + Instant.now() | null | false + Instant.now() | Duration.ofSeconds(10) | true + } + + def 'should create status from mirror entry' () { + given: + def ts = Instant.now().minusSeconds(60) + def request = Mock(MirrorRequest) + def result = Mock(MirrorResult) + + when: + def state1 = ContainerState.from(new MirrorEntry(request, result)) + then: + request.getCreationTime() >> ts + and: + state1.startTime == ts + state1.duration == null + !state1.succeeded + + when: + def state2 = ContainerState.from(new MirrorEntry(request, result)) + then: + request.getCreationTime() >> ts + result.getDuration() >> Duration.ofMinutes(1) + result.succeeded() >> true + and: + state2.startTime == ts + state2.duration == Duration.ofMinutes(1) + state2.succeeded + } + + @Unroll + def 'should create status from mirror result' () { + given: + def mirror = Mock(MirrorResult) + + when: + def state = ContainerState.from(mirror) + then: + mirror.getCreationTime() >> TIME + mirror.duration >> DURATION + mirror.succeeded() >> SUCCEEDED + and: + state.startTime == TIME + state.duration == DURATION + state.succeeded == SUCCEEDED + + where: + TIME | DURATION | SUCCEEDED + Instant.now() | null | false + Instant.now() | Duration.ofSeconds(10) | true + } +} diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 57e0e4814..7dea14a09 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -26,11 +26,18 @@ import java.time.Duration import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.builder.BuildFormat +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.request.ContainerRequest +import io.seqera.wave.tower.PlatformId import jakarta.inject.Inject /** * Tests for ContainerScanServiceImpl @@ -40,20 +47,23 @@ import jakarta.inject.Inject @MicronautTest class ContainerScanServiceImplTest extends Specification { - @Inject ContainerScanServiceImpl containerScanService + @Inject + ContainerScanServiceImpl scanService - @Inject PersistenceService persistenceService + @Inject + PersistenceService persistenceService - @Inject ScanStateStore stateStore + @Inject + ScanStateStore stateStore def 'should start scan successfully'() { given: def KEY = 'scan-10' def workDir = Files.createTempDirectory('test') - def scanRequest = new ScanRequest(KEY, 'build-1', null, 'ubuntu:latest', ContainerPlatform.of('linux/amd64'), workDir, Instant.now()) + def scanRequest = ScanRequest.of(scanId: KEY, buildId:'build-1', targetImage: 'ubuntu:latest', platform: ContainerPlatform.of('linux/amd64'), workDir: workDir, creationTime: Instant.now()) when: - containerScanService.scan(scanRequest) + scanService.scan(scanRequest) sleep 500 then: def scanRecord = stateStore.getScan(scanRequest.scanId) @@ -132,7 +142,7 @@ class ContainerScanServiceImplTest extends Specification { def jobService = Mock(JobService) def service = new ContainerScanServiceImpl(scanStore: stateStore, persistenceService: persistenceService, jobService: jobService) def job = JobSpec.scan(KEY, 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), workDir) - def scan = new ScanEntry(KEY, 'build-20', 'ubuntu:latest', Instant.now()) + def scan = ScanEntry.of(scanId: KEY, buildId: 'build-20', containerImage: 'ubuntu:latest', startTime: Instant.now()) when: service.onJobCompletion(job, scan, new JobState(JobState.Status.SUCCEEDED,0)) @@ -183,7 +193,7 @@ class ContainerScanServiceImplTest extends Specification { def service = new ContainerScanServiceImpl(scanStore: stateStore, persistenceService: persistenceService, jobService: jobService) def job = JobSpec.scan(KEY, 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def error = new Exception('Some error msg') - def scan = new ScanEntry(KEY, 'build-30', 'ubuntu:latest', Instant.now()) + def scan = ScanEntry.of(scanId: KEY, buildId: 'build-30', containerImage: 'ubuntu:latest', startTime: Instant.now()) when: service.onJobException(job, scan, error) @@ -214,7 +224,7 @@ class ContainerScanServiceImplTest extends Specification { def jobService = Mock(JobService) def service = new ContainerScanServiceImpl(scanStore: stateStore, persistenceService: persistenceService, jobService: jobService) def job = JobSpec.scan(KEY, 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) - def scan = new ScanEntry(KEY, 'build-40', 'ubuntu:latest', Instant.now()) + def scan = ScanEntry.of(scanId: KEY, buildId: 'build-40', containerImage: 'ubuntu:latest', startTime: Instant.now()) when: service.onJobTimeout(job, scan) @@ -240,4 +250,103 @@ class ContainerScanServiceImplTest extends Specification { stateStore.clear() } + + def 'should create a scan request' () { + given: + def scanService = new ContainerScanServiceImpl() + def containerId = 'container1234' + and: + def workspace = Path.of('/some/workspace') + def platform = ContainerPlatform.of('amd64') + final build = + new BuildRequest( + containerId: containerId, + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanId: 'scan12345', + format: BuildFormat.DOCKER, + buildId: "${containerId}_1", + ) + + when: + def scan = scanService.fromBuild(build) + then: + scan.scanId == build.scanId + scan.buildId == build.buildId + scan.workDir != build.workDir + scan.configJson == build.configJson + scan.targetImage == build.targetImage + scan.platform == build.platform + scan.workDir.startsWith(workspace) + } + + def 'should create scan request from mirror' () { + given: + def scanService = new ContainerScanServiceImpl() + and: + def timestamp = Instant.now() + def request = MirrorRequest.create( + 'source/foo', + 'target/foo', + 'sha256:12345', + ContainerPlatform.DEFAULT, + Path.of('/some/workspace'), + '{config}', + 'sc-123', + timestamp, + "GMT", + Mock(PlatformId) + ) + + when: + ScanRequest scan = scanService.fromMirror(request) + then: + scan.scanId == 'sc-123' + scan.mirrorId == request.mirrorId // build id is used to carry the mirror id + scan.configJson == '{config}' + scan.targetImage == 'target/foo' + scan.platform == ContainerPlatform.DEFAULT + scan.workDir == Path.of('/some/workspace/sc-123') + scan.creationTime >= timestamp + } + + def 'should create scan request from container request' () { + given: + def config = Mock(ScanConfig) + def inspectService = Mock(ContainerInspectService) + and: + def scanService = new ContainerScanServiceImpl(inspectService: inspectService, config: config) + def userId = Mock(PlatformId) + and: + def timestamp = Instant.now() + def request = Mock(ContainerRequest) + request.buildId >> 'bd-1234' + request.mirrorId >> 'mr-1234' + request.requestId >> 'cr-abc123' + request.scanId >> 'sc-345' + request.containerImage >> 'docker.io/foo:bar' + request.identity >> userId + request.platform >> ContainerPlatform.DEFAULT + + when: + ScanRequest scan = scanService.fromContainer(request) + then: + config.workspace >> Path.of('/some/workspace') + inspectService.credentialsConfigJson(null,'docker.io/foo:bar',null,userId) >> '{docker json config}' + and: + scan.scanId == 'sc-345' + scan.buildId == 'bd-1234' + scan.mirrorId == null + scan.requestId == 'cr-abc123' + scan.configJson == '{docker json config}' + scan.targetImage == 'docker.io/foo:bar' + scan.platform == ContainerPlatform.DEFAULT + scan.workDir == Path.of('/some/workspace/sc-345') + scan.creationTime >= timestamp + } + } diff --git a/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy index ef64a6505..64deb7b8b 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/KubeScanStrategyTest.groovy @@ -63,7 +63,7 @@ class KubeScanStrategyTest extends Specification { def folder = Files.createTempDirectory('test') when: - def request = new ScanRequest('100', 'abc', null, 'ubuntu', ContainerPlatform.of('amd64'), folder.resolve('foo')) + def request = ScanRequest.of(scanId: '100', buildId: 'abc', targetImage: 'ubuntu', platform: ContainerPlatform.of('amd64'), workDir: folder.resolve('foo')) Files.createDirectories(request.workDir) strategy.scanContainer('job-name', request) @@ -71,7 +71,7 @@ class KubeScanStrategyTest extends Specification { 1 * k8sService.launchScanJob(_, _, _, _, _, _, [service:'wave-scan']) >> null when: - def request2 = new ScanRequest('100', 'abc', null, 'ubuntu', ContainerPlatform.of('arm64'), folder.resolve('bar')) + def request2 = ScanRequest.of(scanId: '100', buildId: 'abc', targetImage: 'ubuntu', platform: ContainerPlatform.of('arm64'), workDir: folder.resolve('bar')) Files.createDirectories(request.workDir) strategy.scanContainer('job-name', request2) diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy similarity index 61% rename from src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy rename to src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy index 66a1cc1b6..1ff25188f 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanResultTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy @@ -30,7 +30,7 @@ import io.seqera.wave.core.ContainerPlatform * * @author Paolo Di Tommaso */ -class ScanResultTest extends Specification { +class ScanEntryTest extends Specification { boolean nearly(Duration given, Duration expected) { given >= expected @@ -43,14 +43,14 @@ class ScanResultTest extends Specification { def ts = Instant.now().minus(elapsed) def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') when: - def result = new ScanEntry( - '123', - 'build-123', - 'docker.io/foo/bar:latest', - ts, - elapsed, - 'DONE', - [ CVE1 ] + def result = ScanEntry.of( + scanId: '123', + buildId: 'build-123', + containerImage: 'docker.io/foo/bar:latest', + startTime: ts, + duration: elapsed, + status: 'DONE', + vulnerabilities: [ CVE1 ] ) then: result.scanId == '123' @@ -65,17 +65,17 @@ class ScanResultTest extends Specification { @Unroll def 'should validate completed' () { when: - def result = new ScanEntry( - '123', - 'build-123', - 'docker.io/foo/bar:latest', - Instant.now(), - DURATION, - 'DONE', - [] + def result = ScanEntry.of( + scanId: '123', + buildId: 'build-123', + containerImage: 'docker.io/foo/bar:latest', + startTime: Instant.now(), + duration: DURATION, + status: 'DONE', + vulnerabilities: [] ) then: - result.isCompleted() == EXPECTED + result.completed() == EXPECTED result.done() == EXPECTED where: @@ -87,17 +87,16 @@ class ScanResultTest extends Specification { @Unroll def 'should validate completed' () { when: - def result = new ScanEntry( - '123', - 'build-123', - 'docker.io/foo/bar:latest', - Instant.now(), - null, - STATUS, - [] + def result = ScanEntry.of( + scanId: '123', + buildId: 'build-123', + containerImage: 'docker.io/foo/bar:latest', + startTime: Instant.now(), + status: STATUS, + vulnerabilities: [] ) then: - result.isSucceeded() == EXPECTED + result.succeeded() == EXPECTED where: STATUS | EXPECTED @@ -111,7 +110,7 @@ class ScanResultTest extends Specification { def elapsed = Duration.ofMinutes(1) def ts = Instant.now().minus(elapsed) when: - def result = ScanEntry.create('scan-123', 'build-123', 'ubuntu:latest', ts, Duration.ofMinutes(1), 'XYZ', [cve1]) + def result = ScanEntry.of(scanId: 'scan-123', buildId: 'build-123', containerImage: 'ubuntu:latest', startTime: ts, duration: Duration.ofMinutes(1), status: 'XYZ', vulnerabilities: [cve1]) then: result.scanId == 'scan-123' result.buildId == 'build-123' @@ -128,14 +127,14 @@ class ScanResultTest extends Specification { def elapsed = Duration.ofMinutes(1) def ts = Instant.now().minus(elapsed) and: - def scan = new ScanEntry( - '12345', - 'build-12345', - 'docker.io/some:image', - ts, - elapsed, - ScanEntry.SUCCEEDED, - [cve1] ) + def scan = ScanEntry.of( + scanId: '12345', + buildId: 'build-12345', + containerImage: 'docker.io/some:image', + startTime: ts, + duration: elapsed, + status: ScanEntry.SUCCEEDED, + vulnerabilities: [cve1] ) when: def result = scan.success(scan.vulnerabilities) then: @@ -153,14 +152,14 @@ class ScanResultTest extends Specification { def elapsed = Duration.ofMinutes(1) def ts = Instant.now().minus(elapsed) and: - def scan = new ScanEntry( - '12345', - 'build-12345', - 'docker.io/some:image', - ts, - elapsed, - ScanEntry.FAILED, - [] ) + def scan = ScanEntry.of( + scanId: '12345', + buildId: 'build-12345', + containerImage: 'docker.io/some:image', + startTime: ts, + duration: elapsed, + status: ScanEntry.FAILED, + vulnerabilities: [] ) when: def result = scan.failure(1, "Oops something has failed") then: @@ -180,14 +179,14 @@ class ScanResultTest extends Specification { def elapsed = Duration.ofMinutes(1) def ts = Instant.now().minus(elapsed) and: - def request = new ScanRequest( - 'scan-123', - 'build-345', - 'config', - 'docker.io/foo/bar', - ContainerPlatform.DEFAULT, - Path.of('/some/path'), - ts ) + def request = ScanRequest.of( + scanId:'scan-123', + buildId: 'build-345', + configJson: 'config', + targetImage: 'docker.io/foo/bar', + platform: ContainerPlatform.DEFAULT, + workDir: Path.of('/some/path'), + creationTime: ts ) when: def result = ScanEntry.failure(request) then: @@ -203,17 +202,35 @@ class ScanResultTest extends Specification { def 'should create scan pending' () { given: def ts = Instant.now() - + def request = ScanRequest.of(scanId: 'sc-123', buildId: 'bd-345', mirrorId: 'mr-1234', requestId: 'rq-123', targetImage: 'docker.io/foo/bar', creationTime: ts) when: - def scan = ScanEntry.pending('result-123', 'build-345', 'docker.io/foo/bar') + def scan = ScanEntry.create(request) then: - scan.scanId == 'result-123' - scan.buildId == 'build-345' + scan.scanId == 'sc-123' + scan.buildId == 'bd-345' + scan.mirrorId == 'mr-1234' + scan.requestId == 'rq-123' scan.containerImage == 'docker.io/foo/bar' - scan.startTime >= ts + scan.startTime == ts scan.status == ScanEntry.PENDING scan.vulnerabilities == [] scan.exitCode == null scan.logs == null } + + def 'should create vul summary' () { + given: + def s1 = Mock(ScanVulnerability) { severity>>'low' } + def s2 = Mock(ScanVulnerability) { severity>>'low' } + def s3 = Mock(ScanVulnerability) { severity>>'low' } + def s4 = Mock(ScanVulnerability) { severity>>'high' } + def s5 = Mock(ScanVulnerability) { severity>>'critical' } + and: + def entry = ScanEntry.of(vulnerabilities: [s1,s2,s3,s4,s5]) + + when: + def result = entry.summary() + then: + result == [low:3, high:1, critical:1] + } } diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreLocalTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreLocalTest.groovy new file mode 100644 index 000000000..3508b825b --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreLocalTest.groovy @@ -0,0 +1,75 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.store.state.CountParams +import jakarta.inject.Inject +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class ScanIdStoreLocalTest extends Specification { + + @Inject ScanIdStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-scanid/v1:foo' + } + + def 'should return counter key' () { + expect: + store.counterKey('foo', Mock(ScanId)) == new CountParams('wave-scanid/v1/counter','foo') + } + + def 'should get scan id' () { + given: + def _100ms = 100 + def image = 'ubuntu' + + when: + def ret1 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + ret1.succeed + ret1.value.toString() == 'sc-f4fb059823db8f4d_1' + ret1.count == 1 + + when: + def ret2 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + !ret2.succeed + ret2.value.toString() == 'sc-f4fb059823db8f4d_1' + ret1.count == 1 + + when: + sleep 2 *_100ms + def ret3 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + ret3.succeed + ret3.value.toString() == 'sc-f4fb059823db8f4d_2' + ret3.count == 2 + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreRedisTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreRedisTest.groovy new file mode 100644 index 000000000..a5d2c4711 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanIdStoreRedisTest.groovy @@ -0,0 +1,79 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.test.RedisTestContainer +/** + * + * @author Paolo Di Tommaso + */ +class ScanIdStoreRedisTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext applicationContext + + def setup() { + applicationContext = ApplicationContext.run([ + REDIS_HOST : redisHostName, + REDIS_PORT : redisPort + ], 'test', 'redis') + sleep(500) // workaround to wait for Redis connection + } + + def cleanup() { + applicationContext.close() + } + + def 'should get scan id' () { + given: + def store = applicationContext.getBean(ScanIdStore) + def _100ms = 100 + def image = 'ubuntu' + + when: + def ret1 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + ret1.succeed + ret1.value.toString() == 'sc-f4fb059823db8f4d_1' + ret1.count == 1 + + when: + def ret2 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + !ret2.succeed + ret2.value.toString() == 'sc-f4fb059823db8f4d_1' + ret1.count == 1 + + when: + sleep 2 *_100ms + def ret3 = store.putIfAbsentAndCount(image, ScanId.of(image), Duration.ofMillis(_100ms)) + then: + ret3.succeed + ret3.value.toString() == 'sc-f4fb059823db8f4d_2' + ret3.count == 2 + + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanIdTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanIdTest.groovy new file mode 100644 index 000000000..69ef3cc22 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanIdTest.groovy @@ -0,0 +1,44 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class ScanIdTest extends Specification { + + def 'should create scanid' () { + ScanId.of("docker.io/foo/bar:latest").toString() == 'sc-057d44d43bb7f81c_0' + } + + def 'create a count with the specified count' () { + expect: + ScanId.of('x').toString() == 'sc-94be9fbddcc3af8e_0' + ScanId.of('x').withCount(1).toString() == 'sc-94be9fbddcc3af8e_1' + ScanId.of('x').withCount(9).toString() == 'sc-94be9fbddcc3af8e_9' + and: + ScanId.of('x', 'y').toString() == 'sc-00769af315dcb1fe_0' + ScanId.of('x', 'y').withCount(1).toString() == 'sc-00769af315dcb1fe_1' + ScanId.of('x', 'y').withCount(9).toString() == 'sc-00769af315dcb1fe_9' + } + +} diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy index 7502c81c6..ac45bc9c8 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanRequestTest.groovy @@ -21,11 +21,10 @@ package io.seqera.wave.service.scan import spock.lang.Specification import java.nio.file.Path +import java.time.Instant import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.builder.BuildFormat -import io.seqera.wave.service.builder.BuildRequest -import io.seqera.wave.tower.PlatformId + /** * * @author Paolo Di Tommaso @@ -34,32 +33,30 @@ class ScanRequestTest extends Specification { def 'should create a scan request' () { given: - def workspace = Path.of('/some/workspace') - def platform = ContainerPlatform.of('amd64') - final build = - new BuildRequest( - containerId: 'container1234', - containerFile: 'FROM ubuntu', - workspace: workspace, - targetImage: 'docker.io/my/repo:container1234', - identity: PlatformId.NULL, - platform: platform, - configJson: '{"config":"json"}', - scanId: 'scan12345', - format: BuildFormat.DOCKER - ) - .withBuildId('123') - + def timestamp = Instant.now() when: - def scan = ScanRequest.fromBuild(build) + def scan = ScanRequest.of( + scanId: 'sc-123', + buildId: 'bd-123', + mirrorId: 'mr-123', + requestId: 'rq-123', + configJson: '{config}', + targetImage: 'tg-image', + platform: ContainerPlatform.DEFAULT, + workDir: Path.of('/some/dir'), + creationTime: timestamp + ) + then: - scan.scanId == build.scanId - scan.buildId == build.buildId - scan.workDir != build.workDir - scan.configJson == build.configJson - scan.targetImage == build.targetImage - scan.platform == build.platform - scan.workDir.startsWith(workspace) + scan.scanId == 'sc-123' + scan.buildId == 'bd-123' + scan.mirrorId == 'mr-123' + scan.requestId == 'rq-123' + scan.configJson == '{config}' + scan.targetImage == 'tg-image' + scan.platform == ContainerPlatform.DEFAULT + scan.workDir == Path.of('/some/dir') + scan.creationTime == timestamp } } diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanStateStoreTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanStateStoreTest.groovy new file mode 100644 index 000000000..ba9faca65 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanStateStoreTest.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject + +/** + * + * @author Paolo Di Tommaso + */ +@MicronautTest +class ScanStateStoreTest extends Specification { + + @Inject ScanStateStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-scan/v1:foo' + } + +} diff --git a/src/test/groovy/io/seqera/wave/storage/ManifestCacheStoreTest.groovy b/src/test/groovy/io/seqera/wave/storage/ManifestCacheStoreTest.groovy index 7956d085a..ba5148fb9 100644 --- a/src/test/groovy/io/seqera/wave/storage/ManifestCacheStoreTest.groovy +++ b/src/test/groovy/io/seqera/wave/storage/ManifestCacheStoreTest.groovy @@ -20,11 +20,22 @@ package io.seqera.wave.storage import spock.lang.Specification +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject + /** * * @author Paolo Di Tommaso */ +@MicronautTest class ManifestCacheStoreTest extends Specification { + @Inject + ManifestCacheStore store + + def 'should return entry key' () { + expect: + store.key0('foo') == 'wave-blobs/v1:foo' + } } diff --git a/src/test/groovy/io/seqera/wave/store/state/AbstractStateStoreTest.groovy b/src/test/groovy/io/seqera/wave/store/state/AbstractStateStoreTest.groovy index e2db2370f..bec901877 100644 --- a/src/test/groovy/io/seqera/wave/store/state/AbstractStateStoreTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/state/AbstractStateStoreTest.groovy @@ -65,7 +65,7 @@ class AbstractStateStoreTest extends Specification { @Override protected String getPrefix() { - return 'test/v1:' + return 'test/v1' } @Override @@ -87,7 +87,7 @@ class AbstractStateStoreTest extends Specification { def store = new MyCacheStore(provider) expect: - store.requestId0('one') == 'test/v1:request-id/one' + store.requestId0('one') == 'test/v1/request-id:one' } def 'should get and put a value' () { diff --git a/src/test/groovy/io/seqera/wave/store/state/CountParamsTest.groovy b/src/test/groovy/io/seqera/wave/store/state/CountParamsTest.groovy new file mode 100644 index 000000000..8d0fc0d9c --- /dev/null +++ b/src/test/groovy/io/seqera/wave/store/state/CountParamsTest.groovy @@ -0,0 +1,40 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.store.state + +import spock.lang.Specification +/** + * + * @author Paolo Di Tommaso + */ +class CountParamsTest extends Specification { + + def 'should split key'() { + when: + def result = CountParams.of(KEY) + then: + result == EXPECTED + where: + KEY | EXPECTED + 'one' | new CountParams("counters/v1", "one") + 'one/two' | new CountParams("one", "two") + 'one/two/three' | new CountParams("one/two", "three") + } + +} diff --git a/src/test/groovy/io/seqera/wave/store/state/impl/LocalStateProviderTest.groovy b/src/test/groovy/io/seqera/wave/store/state/impl/LocalStateProviderTest.groovy index c1c865352..f222ef9b8 100644 --- a/src/test/groovy/io/seqera/wave/store/state/impl/LocalStateProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/state/impl/LocalStateProviderTest.groovy @@ -23,13 +23,14 @@ import spock.lang.Specification import java.time.Duration import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.store.state.CountParams import jakarta.inject.Inject @MicronautTest(environments = ['test']) class LocalStateProviderTest extends Specification { @Inject - LocalStateProvider localCacheProvider + LocalStateProvider provider def 'should get and put a key-value pair' () { @@ -37,12 +38,12 @@ class LocalStateProviderTest extends Specification { def k = UUID.randomUUID().toString() expect: - localCacheProvider.get(k) == null + provider.get(k) == null when: - localCacheProvider.put(k, "hello") + provider.put(k, "hello") then: - localCacheProvider.get(k) == 'hello' + provider.get(k) == 'hello' } def 'should get and put a key-value pair with ttl' () { @@ -51,16 +52,16 @@ class LocalStateProviderTest extends Specification { def k = UUID.randomUUID().toString() expect: - localCacheProvider.get(k) == null + provider.get(k) == null when: - localCacheProvider.put(k, "hello", Duration.ofMillis(TTL)) + provider.put(k, "hello", Duration.ofMillis(TTL)) then: - localCacheProvider.get(k) == 'hello' + provider.get(k) == 'hello' then: sleep(TTL *2) and: - localCacheProvider.get(k) == null + provider.get(k) == null } def 'should get and put only if absent' () { @@ -68,21 +69,21 @@ class LocalStateProviderTest extends Specification { def k = UUID.randomUUID().toString() expect: - localCacheProvider.get(k) == null + provider.get(k) == null when: - def done = localCacheProvider.putIfAbsent(k, 'foo') + def done = provider.putIfAbsent(k, 'foo') then: done and: - localCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - done = localCacheProvider.putIfAbsent(k, 'bar') + done = provider.putIfAbsent(k, 'bar') then: !done and: - localCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' } def 'should get and put if absent with ttl' () { @@ -91,27 +92,76 @@ class LocalStateProviderTest extends Specification { def k = UUID.randomUUID().toString() when: - def done = localCacheProvider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) + def done = provider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) then: done and: - localCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - done = localCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) + done = provider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: !done and: - localCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: sleep(TTL *2) and: - done = localCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) + done = provider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: done and: - localCacheProvider.get(k) == 'bar' + provider.get(k) == 'bar' + } + + def 'should get and put if absent and increment' () { + given: + def ttlMillis = 100 + def k = UUID.randomUUID().toString() + def c = CountParams .of(UUID.randomUUID().toString()) + def luaScript1 = /string.gsub(value, '"count"%s*:%s*(%d+)', '"count":' .. counter_value)/ + def luaScript2 = /string.gsub(value, '"count"%s*:%s*"(.-)(%d+)"', '"count":"%1' .. counter_value .. '"')/ + + expect: + provider.get(k) == null + + when: + def result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"foo":"x","count":0}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + result.succeed + result.value == '{"foo":"x","count":1}' + result.count == 1 + and: + provider.get(k) == '{"foo":"x","count":1}' + + when: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y","count":0}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + !result.succeed + result.value == '{"foo":"x","count":1}' + result.count == 1 + and: + provider.get(k) == '{"foo":"x","count":1}' + + when: + sleep(ttlMillis *2) + and: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y","count":0}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + result.succeed + result.value == '{"bar":"y","count":2}' + result.count == 2 + + when: + sleep(ttlMillis *2) + and: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y", "count":"xx-a1b2c3_100"}', Duration.ofMillis(ttlMillis), c, luaScript2) + then: + result.succeed + result.value == '{"bar":"y", "count":"xx-a1b2c3_3"}' + result.count == 3 + } def 'should put and remove a value' () { @@ -120,14 +170,14 @@ class LocalStateProviderTest extends Specification { def k = UUID.randomUUID().toString() when: - localCacheProvider.put(k, 'foo') + provider.put(k, 'foo') then: - localCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - localCacheProvider.remove(k) + provider.remove(k) then: - localCacheProvider.get(k) == null + provider.get(k) == null } } diff --git a/src/test/groovy/io/seqera/wave/store/state/impl/RedisStateProviderTest.groovy b/src/test/groovy/io/seqera/wave/store/state/impl/RedisStateProviderTest.groovy index 0d0290ab0..73c9274a5 100644 --- a/src/test/groovy/io/seqera/wave/store/state/impl/RedisStateProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/store/state/impl/RedisStateProviderTest.groovy @@ -32,14 +32,14 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer ApplicationContext applicationContext @Shared - RedisStateProvider redisCacheProvider + RedisStateProvider provider def setup() { applicationContext = ApplicationContext.run([ REDIS_HOST : redisHostName, REDIS_PORT : redisPort ], 'test', 'redis') - redisCacheProvider = applicationContext.getBean(RedisStateProvider) + provider = applicationContext.getBean(RedisStateProvider) sleep(500) // workaround to wait for Redis connection } @@ -52,12 +52,12 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer def k = UUID.randomUUID().toString() expect: - redisCacheProvider.get(k) == null + provider.get(k) == null when: - redisCacheProvider.put(k, "hello") + provider.put(k, "hello") then: - redisCacheProvider.get(k) == 'hello' + provider.get(k) == 'hello' } def 'should get and put a key-value pair with ttl' () { @@ -66,16 +66,16 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer def k = UUID.randomUUID().toString() expect: - redisCacheProvider.get(k) == null + provider.get(k) == null when: - redisCacheProvider.put(k, "hello", Duration.ofMillis(TTL)) + provider.put(k, "hello", Duration.ofMillis(TTL)) then: - redisCacheProvider.get(k) == 'hello' + provider.get(k) == 'hello' then: sleep(TTL *2) and: - redisCacheProvider.get(k) == null + provider.get(k) == null } def 'should get and put only if absent' () { @@ -83,21 +83,21 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer def k = UUID.randomUUID().toString() expect: - redisCacheProvider.get(k) == null + provider.get(k) == null when: - def done = redisCacheProvider.putIfAbsent(k, 'foo') + def done = provider.putIfAbsent(k, 'foo') then: done and: - redisCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - done = redisCacheProvider.putIfAbsent(k, 'bar') + done = provider.putIfAbsent(k, 'bar') then: !done and: - redisCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' } def 'should get and put if absent with ttl' () { @@ -106,27 +106,27 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer def k = UUID.randomUUID().toString() when: - def done = redisCacheProvider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) + def done = provider.putIfAbsent(k, 'foo', Duration.ofMillis(TTL)) then: done and: - redisCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - done = redisCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) + done = provider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: !done and: - redisCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: sleep(TTL *2) and: - done = redisCacheProvider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) + done = provider.putIfAbsent(k, 'bar', Duration.ofMillis(TTL)) then: done and: - redisCacheProvider.get(k) == 'bar' + provider.get(k) == 'bar' } def 'should put and remove a value' () { @@ -135,13 +135,65 @@ class RedisStateProviderTest extends Specification implements RedisTestContainer def k = UUID.randomUUID().toString() when: - redisCacheProvider.put(k, 'foo') + provider.put(k, 'foo') then: - redisCacheProvider.get(k) == 'foo' + provider.get(k) == 'foo' when: - redisCacheProvider.remove(k) + provider.remove(k) then: - redisCacheProvider.get(k) == null + provider.get(k) == null } + + def 'should get and put if absent and increment' () { + given: + def ttlMillis = 100 + def k = UUID.randomUUID().toString() + def c = UUID.randomUUID().toString() + def luaScript1 = /string.gsub(value, '"count"%s*:%s*(%d+)', '"count":' .. counter_value)/ + def luaScript2 = /string.gsub(value, '"count"%s*:%s*"(.-)(%d+)"', '"count":"%1' .. counter_value .. '"')/ + + expect: + provider.get(k) == null + + when: + def result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"foo":"x","count":0}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + result.succeed + result.value == '{"foo":"x","count":1}' + result.count == 1 + and: + provider.get(k) == '{"foo":"x","count":1}' + + when: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y"}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + !result.succeed + result.value == '{"foo":"x","count":1}' + result.count == 1 + and: + provider.get(k) == '{"foo":"x","count":1}' + + when: + sleep(ttlMillis *2) + and: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y","count":0}', Duration.ofMillis(ttlMillis), c, luaScript1) + then: + result.succeed + result.value == '{"bar":"y","count":2}' + result.count == 2 + and: + provider.get(k) == '{"bar":"y","count":2}' + + when: + sleep(ttlMillis *2) + and: + result = provider.putJsonIfAbsentAndIncreaseCount(k, '{"bar":"y", "count":"xx-a1b2c3_100"}', Duration.ofMillis(ttlMillis), c, luaScript2) + then: + result.succeed + result.value == '{"bar":"y", "count":"xx-a1b2c3_3"}' + result.count == 3 + } + + } diff --git a/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy b/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy index 3ba68147f..698c11042 100644 --- a/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/AwsS3TestContainer.groovy @@ -17,6 +17,8 @@ */ package io.seqera.wave.test +import java.time.Duration + import org.slf4j.Logger import org.slf4j.LoggerFactory import org.testcontainers.containers.GenericContainer @@ -33,10 +35,14 @@ trait AwsS3TestContainer { static { log.debug "Starting AWS S3 test container" - awsS3Container = new GenericContainer("localstack/localstack:3.0.2") - .withExposedPorts(5000) + awsS3Container = new GenericContainer<>("localstack/localstack:3.8.0") + .withExposedPorts(4566) .withEnv("SERVICES", "s3") - .waitingFor(Wait.forLogMessage(".*Ready\\.\n", 1)) + .withEnv("DEFAULT_REGION", "eu-west-1") + .withEnv("EDGE_PORT", "4566") + .waitingFor(Wait.forHttp("/_localstack/health").forStatusCode(200)) + .withStartupTimeout(Duration.ofMinutes(1)) + awsS3Container.start() log.debug "Started AWS S3 test container" } @@ -46,10 +52,6 @@ trait AwsS3TestContainer { } String getAwsS3Port(){ - awsS3Container.getMappedPort(5000) - } - - def cleanupSpec(){ - awsS3Container.stop() + awsS3Container.getMappedPort(4566) } } diff --git a/src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreLocalTest.groovy b/src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreTest.groovy similarity index 91% rename from src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreLocalTest.groovy rename to src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreTest.groovy index df8c00624..736c39355 100644 --- a/src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/tower/auth/JwtAuthStoreTest.groovy @@ -24,13 +24,12 @@ import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest import jakarta.inject.Inject - /** * * @author Paolo Di Tommaso */ @MicronautTest -class JwtAuthStoreLocalTest extends Specification { +class JwtAuthStoreTest extends Specification { @Inject JwtAuthStore store @@ -60,4 +59,10 @@ class JwtAuthStoreLocalTest extends Specification { updatedAt >= auth.updatedAt } } + + def 'should return entry key' () { + expect: + store.key0('foo') == 'tower-jwt-store/v1:foo' + } + } diff --git a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy index bb68799fe..056cdcf42 100644 --- a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy @@ -29,9 +29,9 @@ import io.seqera.wave.api.PackagesSpec import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.config.CondaOpts import io.seqera.wave.exception.BadRequestException -import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildFormat -import io.seqera.wave.service.token.TokenData +import io.seqera.wave.service.request.TokenData /** * Container helper methods * @@ -52,10 +52,14 @@ class ContainerHelperTest extends Specification { then: result =='''\ BootStrap: docker - From: mambaorg/micromamba:1.5.8-lunar + From: mambaorg/micromamba:1.5.10-noble %post micromamba install -y -n base -c conda-forge -c defaults -f https://foo.com/lock.yml micromamba install -y -n base foo::one bar::two + micromamba env export --name base --explicit > environment.lock + echo ">> CONDA_LOCK_START" + cat environment.lock + echo "<< CONDA_LOCK_END" micromamba clean -a -y %environment export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" @@ -75,10 +79,14 @@ class ContainerHelperTest extends Specification { then: result =='''\ - FROM mambaorg/micromamba:1.5.8-lunar + FROM mambaorg/micromamba:1.5.10-noble RUN \\ micromamba install -y -n base -c conda-forge -c defaults -f https://foo.com/lock.yml \\ && micromamba install -y -n base foo::one bar::two \\ + && micromamba env export --name base --explicit > environment.lock \\ + && echo ">> CONDA_LOCK_START" \\ + && cat environment.lock \\ + && echo "<< CONDA_LOCK_END" \\ && micromamba clean -a -y USER root ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" @@ -99,12 +107,16 @@ class ContainerHelperTest extends Specification { then: result =='''\ BootStrap: docker - From: mambaorg/micromamba:1.5.8-lunar + From: mambaorg/micromamba:1.5.10-noble %files {{wave_context_dir}}/conda.yml /scratch/conda.yml %post micromamba install -y -n base -f /scratch/conda.yml micromamba install -y -n base foo::one bar::two + micromamba env export --name base --explicit > environment.lock + echo ">> CONDA_LOCK_START" + cat environment.lock + echo "<< CONDA_LOCK_END" micromamba clean -a -y %environment export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" @@ -123,10 +135,14 @@ class ContainerHelperTest extends Specification { then: result =='''\ - FROM mambaorg/micromamba:1.5.8-lunar + FROM mambaorg/micromamba:1.5.10-noble COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml RUN micromamba install -y -n base -f /tmp/conda.yml \\ && micromamba install -y -n base foo::one bar::two \\ + && micromamba env export --name base --explicit > environment.lock \\ + && echo ">> CONDA_LOCK_START" \\ + && cat environment.lock \\ + && echo "<< CONDA_LOCK_END" \\ && micromamba clean -a -y USER root ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" @@ -208,15 +224,10 @@ class ContainerHelperTest extends Specification { def 'should create response v1' () { given: - def data = new ContainerRequestData(null, - 'docker.io/some/container', - null, - null, - null, - null, - '123', - NEW_BUILD - ) + def data = ContainerRequest.of( + containerImage: 'docker.io/some/container', + buildId: '123', + buildNew: NEW_BUILD ) def token = new TokenData('123abc', Instant.now().plusSeconds(100)) def target = 'wave.com/this/that' @@ -240,21 +251,20 @@ class ContainerHelperTest extends Specification { @Unroll def 'should create response v2' () { given: - def data = new ContainerRequestData(null, - 'docker.io/some/container', - null, - null, - null, - null, - '123', - NEW_BUILD, - IS_FREEZE + def data = ContainerRequest.of( + containerImage: 'docker.io/some/container', + buildId: "build-123", + buildNew: NEW_BUILD, + freeze: IS_FREEZE, + mirror: IS_MIRROR, + scanId: "scan-123", ) def token = new TokenData('123abc', Instant.now().plusSeconds(100)) def target = 'wave.com/this/that' and: def EXPECTED_IMAGE = 'docker.io/some/container' - def EXPECTED_BUILD = '123' + def EXPECTED_BUILD = 'build-123' + def EXPECTED_SCAN = 'scan-123' when: def result = ContainerHelper.makeResponseV2(data, token, target) @@ -266,15 +276,20 @@ class ContainerHelperTest extends Specification { buildId == EXPECTED_BUILD cached == EXPECTED_CACHE freeze == IS_FREEZE + mirror == IS_MIRROR + scanId == EXPECTED_SCAN } where: - NEW_BUILD | IS_FREEZE | EXPECTED_TOKEN | EXPECTED_TARGET | EXPECTED_CACHE - false | false | '123abc' | 'wave.com/this/that' | true - true | false | '123abc' | 'wave.com/this/that' | false + NEW_BUILD | IS_FREEZE | IS_MIRROR | EXPECTED_TOKEN | EXPECTED_TARGET | EXPECTED_CACHE + false | false | false | '123abc' | 'wave.com/this/that' | true + true | false | false | '123abc' | 'wave.com/this/that' | false + and: + false | true | false | null | 'docker.io/some/container'| true + true | true | false | null | 'docker.io/some/container'| false and: - false | true | null | 'docker.io/some/container'| true - true | true | null | 'docker.io/some/container'| false + false | false | true | null | 'docker.io/some/container'| true + true | false | true | null | 'docker.io/some/container'| false } def 'should check if is a conda lock file' () { diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index b56b9bd5d..71dfec879 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -57,6 +57,7 @@ wave: enabled : true bucket : 'nextflow-ci' prefix : 'wave-build/logs' + conda-lock-prefix: 'wave-build/conda-locks' scan: enabled: true --- diff --git a/typespec/models/BuildStatusResponse.tsp b/typespec/models/BuildStatusResponse.tsp index 18da9ea1c..d54133e1b 100644 --- a/typespec/models/BuildStatusResponse.tsp +++ b/typespec/models/BuildStatusResponse.tsp @@ -1,8 +1,10 @@ +import "./Status.tsp"; + @doc("Response payload for build status.") model BuildStatusResponse { duration: string; id: string; startTime: string; - status: "PENDING" | "COMPLETED"; + status: Status; succeeded: boolean; } diff --git a/typespec/models/ContainerMirrorResponse.tsp b/typespec/models/ContainerMirrorResponse.tsp new file mode 100644 index 000000000..419575b9d --- /dev/null +++ b/typespec/models/ContainerMirrorResponse.tsp @@ -0,0 +1,16 @@ +import "./ContainerPlatform.tsp"; +import "./Status.tsp"; + +@doc("Response payload for container mirroring.") +model ContainerMirrorResponse { + mirrorId: string; + digest: string; + sourceImage: string; + targetImage: string; + platform: ContainerPlatform; + creationTime: string; + status: Status; + duration: string; + exitCode: int32; + logs: string; +} diff --git a/typespec/models/ContainerPlatform.tsp b/typespec/models/ContainerPlatform.tsp new file mode 100644 index 000000000..67da1a2f6 --- /dev/null +++ b/typespec/models/ContainerPlatform.tsp @@ -0,0 +1,21 @@ +@doc("Represents os platform of a container.") +model ContainerPlatform { + os: string; + arch: string; + variant?: string; +} + +enum Architecture { + ARM64, + AARCH64, + AMD64, + X86_64, + X86_64_ALT, + ARM +} + +enum OS { + LINUX, + WINDOWS, + MACOS +} diff --git a/typespec/models/Status.tsp b/typespec/models/Status.tsp new file mode 100644 index 000000000..c95ec9f0b --- /dev/null +++ b/typespec/models/Status.tsp @@ -0,0 +1,4 @@ +enum Status { + PENDING, + COMPLETED +} diff --git a/typespec/models/models.tsp b/typespec/models/models.tsp index e010857b1..a7f6a4f71 100644 --- a/typespec/models/models.tsp +++ b/typespec/models/models.tsp @@ -8,3 +8,4 @@ import "./WaveScanRecord.tsp"; import "./WaveBuildRecord.tsp"; import "./ValidateRegistryCredsRequest.tsp"; import "./WaveContainerRecord.tsp"; +import "./ContainerMirrorResponse.tsp"; diff --git a/typespec/routes.tsp b/typespec/routes.tsp index 061e3e287..862bea78f 100644 --- a/typespec/routes.tsp +++ b/typespec/routes.tsp @@ -102,4 +102,16 @@ namespace wave { @route("validate-creds") @post op validateCreds(@body request: ValidateRegistryCredsRequest): boolean; + @route("/v1alpha1/mirrors") + interface getMirrorRecord { + + @route("/{mirrorId}") + @get op containerMirror(@path mirrorId: string): { + @body response: ContainerMirrorResponse; + @statusCode statusCode: 200; + }|{ + @statusCode statusCode: 404; + }; + } + }