Merge branch 'master' into java21

.codespellignore

@@ -1 +1,2 @@
 carrer
+ser

.github/workflows/typespec.yml

@@ -0,0 +1,40 @@
+---
+name: Typespec_Validation
+
+on:
+  push:
+    branches:
+      - '**'
+    paths :
+      - 'typespec/**'
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'typespec/**'
+
+permissions:
+  contents: read
+
+jobs:
+  typespec_validation:
+    name: validate typespec files
+    runs-on: ubuntu-latest
+
+    steps:
+      - name : Checkout
+        uses : actions/checkout@v4
+
+      - name : Setup Node.js environment
+        uses : actions/setup-node@v4
+        with :
+          node-version : '20.9.0'
+
+      - name : Install tsp
+        run : npm install -g @typespec/compiler
+
+      - name : Validate tsp files
+        run : |
+          cd typespec
+          tsp install
+          tsp compile .
+

.gitignore

@@ -32,3 +32,8 @@ scan-workspace/
 .cache
 site/
 deployment-url.txt
+
+#typespec
+tsp-output/
+node_modules/
+package-lock.json

README.md

@@ -108,6 +108,17 @@ container registry where the image is stored, while the instrumented layers are
     '-Djdk.httpclient.HttpClient.log=requests,headers'
     ```
 
+## TypeSpec API Specifications
+
+- You can find the API specifications using (typespec)[https://github.com/microsoft/typespec] in typespec directory. Use following command to generate the API specifications.
+
+    ```bash
+    'cd typespec'
+    'tsp install'
+    'tsp compile .'
+    ```
+
+- Check `typespec/tsp-output` directory for the generated API specifications.
 
 ## Related links
 * [Wave command line tool](https://github.com/seqeralabs/wave-cli)

docs/api.mdx

@@ -285,13 +285,11 @@ Provides status of build against buildId passed as path variable
 
 ```json
 {
-    serviceInfo: {
-        id: string,
-        status: string,
-        startTime: string,
-        duration: string,
-        succeeded: boolean
-    }
+    id: string,
+    status: string,
+    startTime: string,
+    duration: string,
+    succeeded: boolean
 }
 ```
 
@@ -527,7 +525,7 @@ This endpoint is used to retrieve the builds performed by Wave.
 
 ```json
 {
-    metric: "builds|pulls|fusion",
+    metric: "builds",
     count: integer,
     orgs: {
         String: integer,
@@ -590,7 +588,13 @@ This endpoint is used to get the pulls performed through Wave.
 
 ```json
 {
-    count: integer
+    metric: "pulls",
+    count: integer,
+    orgs: {
+        String: integer,
+        String: integer,
+        ...
+    }
 }
 ```
 
@@ -648,7 +652,13 @@ This endpoint is used to get the pulls of Fusion-based containers performed thro
 
 ```json
 {
-    count: integer
+    metric: "fusion",
+    count: integer,
+    orgs: {
+        String: integer,
+        String: integer,
+        ...
+    }
 }
 ```
 

src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy

@@ -39,7 +39,7 @@ import io.seqera.wave.util.StringUtils
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
-import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS
+import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
 /**
  * Implement Docker authentication & login service
  *
@@ -116,7 +116,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> response.statusCode() in HTTP_SERVER_ERRORS)
+                .retryIf( (response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS)
                 .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event}"))
         // make the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))
@@ -203,7 +203,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_SERVER_ERRORS )
+                .retryIf( (response) -> ((HttpResponse)response).statusCode() in HTTP_RETRYABLE_ERRORS )
                 .onRetry((event) -> log.warn("Unable to connect '$login' - event: $event"))
         // submit http request
         final response = retryable.apply(()-> httpClient.send(req, HttpResponse.BodyHandlers.ofString()))

src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy

@@ -34,7 +34,7 @@ import jakarta.inject.Inject
 import jakarta.inject.Singleton
 import static io.seqera.wave.WaveDefault.DOCKER_IO
 import static io.seqera.wave.WaveDefault.DOCKER_REGISTRY_1
-import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS
+import static io.seqera.wave.WaveDefault.HTTP_RETRYABLE_ERRORS
 /**
  * Lookup service for container registry. The role of this component
  * is to registry the retrieve the registry authentication realm
@@ -73,7 +73,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService {
         // retry strategy
         final retryable = Retryable
                 .<HttpResponse<String>>of(httpConfig)
-                .retryIf((response) -> response.statusCode() in HTTP_SERVER_ERRORS)
+                .retryIf((response) -> response.statusCode() in HTTP_RETRYABLE_ERRORS )
                 .onRetry((event) -> log.warn("Unable to connect '$endpoint' - event: $event"))
         // submit the request
         final response = retryable.apply(()-> httpClient.send(request, HttpResponse.BodyHandlers.ofString()))

src/main/groovy/io/seqera/wave/configuration/BlobCacheConfig.groovy

@@ -41,6 +41,9 @@ class BlobCacheConfig {
     @Value('${wave.blobCache.status.delay:5s}')
     Duration statusDelay
 
+    @Value('${wave.blobCache.failure.duration:4s}')
+    Duration failureDuration
+
     @Value('${wave.blobCache.timeout:5m}')
     Duration transferTimeout
 

src/main/groovy/io/seqera/wave/controller/MetricsController.groovy

@@ -36,11 +36,9 @@ import io.micronaut.security.annotation.Secured
 import io.micronaut.security.authentication.AuthorizationException
 import io.micronaut.security.rules.SecurityRule
 import io.seqera.wave.exception.BadRequestException
-import io.seqera.wave.service.metric.MetricConstants
+import io.seqera.wave.service.metric.MetricsConstants
 import io.seqera.wave.service.metric.MetricsService
-import io.seqera.wave.service.metric.model.GetBuildsCountResponse
-import io.seqera.wave.service.metric.model.GetFusionPullsCountResponse
-import io.seqera.wave.service.metric.model.GetPullsCountResponse
+
 import jakarta.inject.Inject
 
 import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE
@@ -65,25 +63,25 @@ class MetricsController {
     @Get(uri = "/v1alpha2/metrics/builds", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getBuildsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
-            return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_BUILDS))
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_BUILDS))
         validateQueryParams(date)
-        return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_BUILDS, date, org))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_BUILDS, date, org))
     }
 
     @Get(uri = "/v1alpha2/metrics/pulls", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
-            return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_PULLS))
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_PULLS))
         validateQueryParams(date)
-        return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_PULLS, date, org))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_PULLS, date, org))
     }
 
     @Get(uri = "/v1alpha2/metrics/fusion/pulls", produces = MediaType.APPLICATION_JSON)
     HttpResponse<?> getFusionPullsMetrics(@Nullable @QueryValue String date, @Nullable @QueryValue String org) {
         if(!date && !org)
-            return HttpResponse.ok(metricsService.getAllOrgCount(MetricConstants.PREFIX_FUSION))
+            return HttpResponse.ok(metricsService.getAllOrgCount(MetricsConstants.PREFIX_FUSION))
         validateQueryParams(date)
-        return HttpResponse.ok(metricsService.getOrgCount(MetricConstants.PREFIX_FUSION, date, org))
+        return HttpResponse.ok(metricsService.getOrgCount(MetricsConstants.PREFIX_FUSION, date, org))
 
     }
 

src/main/groovy/io/seqera/wave/service/blob/impl/BlobCacheServiceImpl.groovy

@@ -20,6 +20,7 @@ package io.seqera.wave.service.blob.impl
 import java.net.http.HttpClient
 import java.net.http.HttpRequest
 import java.net.http.HttpResponse
+import java.util.concurrent.CompletableFuture
 import java.util.concurrent.ExecutorService
 
 import groovy.transform.CompileStatic
@@ -38,13 +39,17 @@ import io.seqera.wave.service.blob.BlobSigningService
 import io.seqera.wave.service.blob.BlobStore
 import io.seqera.wave.service.blob.TransferStrategy
 import io.seqera.wave.service.blob.TransferTimeoutException
+import io.seqera.wave.util.BucketTokenizer
 import io.seqera.wave.util.Escape
 import io.seqera.wave.util.Retryable
 import io.seqera.wave.util.StringUtils
 import jakarta.annotation.PostConstruct
 import jakarta.inject.Inject
 import jakarta.inject.Named
 import jakarta.inject.Singleton
+import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest
 import static io.seqera.wave.WaveDefault.HTTP_SERVER_ERRORS
 /**
  * Implements cache for container image layer blobs
@@ -82,6 +87,10 @@ class BlobCacheServiceImpl implements BlobCacheService {
     @Inject
     private HttpClientConfig httpConfig
 
+    @Inject
+    @Named('BlobS3Client')
+    private S3Client s3Client
+
     private HttpClient httpClient
 
     @PostConstruct
@@ -184,21 +193,38 @@ class BlobCacheServiceImpl implements BlobCacheService {
             else {
                 log.debug "== Blob cache begin for object '${info.locationUri}'"
                 result = store(route, info)
+                //check if the cached blob size is correct
+                result = checkUploadedBlobSize(result, route)
             }
         }
         finally {
             // use a short time-to-live for failed downloads
-            // this is needed to allow re-try downloads failed for
-            // temporary error conditions e.g. expired credentials
+            // this is needed to allow re-try caching of failure transfers
             final ttl = result.succeeded()
                     ? blobConfig.statusDuration
-                    : blobConfig.statusDelay.multipliedBy(10)
+                    : blobConfig.failureDuration
 
             blobStore.storeBlob(route.targetPath, result, ttl)
             return result
         }
     }
 
+    /**
+     * Check the size of the blob stored in the cache
+     *
+     * @return {@link BlobCacheInfo} the blob cache info
+     */
+    protected BlobCacheInfo checkUploadedBlobSize(BlobCacheInfo info, RoutePath route) {
+        if( !info.succeeded() )
+            return info
+        final blobSize = getBlobSize(route)
+        if( blobSize == info.contentLength )
+            return info
+        log.warn("== Blob cache mismatch size for uploaded object '${info.locationUri}'; upload blob size: ${blobSize}; expect size: ${info.contentLength}")
+        CompletableFuture.supplyAsync(() -> deleteBlob(route), executor)
+        return info.failed("Mismatch cache size for object ${info.locationUri}")
+    }
+
     protected BlobCacheInfo store(RoutePath route, BlobCacheInfo info) {
         final target = route.targetPath
         try {
@@ -226,15 +252,14 @@ class BlobCacheServiceImpl implements BlobCacheService {
         StringUtils.pathConcat(blobConfig.storageBucket, route.targetPath)
     }
 
-
     /**
      * The HTTP URI from there the cached layer blob is going to be downloaded
      *
      * @param route The source HTTP request of container layer to be cached
      * @return The HTTP URI from the cached layer blob is going to be downloaded
      */
     protected String blobDownloadUri(RoutePath route) {
-        final bucketPath = StringUtils.pathConcat(blobConfig.storageBucket, route.targetPath)
+        final bucketPath = blobStorePath(route)
         final presignedUrl = signingService.createSignedUri(bucketPath)
 
         if( blobConfig.baseUrl ) {
@@ -247,7 +272,6 @@ class BlobCacheServiceImpl implements BlobCacheService {
             return presignedUrl
     }
 
-
     /**
      * Await for the container layer blob download
      *
@@ -293,4 +317,50 @@ class BlobCacheServiceImpl implements BlobCacheService {
             }
         }
     }
+
+   /**
+    * get the size of the blob stored in the cache
+    *
+    * @return {@link Long} the size of the blob stored in the cache
+    */
+   protected Long getBlobSize(RoutePath route) {
+       final objectUri = blobStorePath(route)
+       final object = BucketTokenizer.from(objectUri)
+       try {
+            final request =
+                    HeadObjectRequest.builder()
+                            .bucket(object.bucket)
+                            .key(object.key)
+                            .build()
+            final headObjectResponse = s3Client.headObject(request as HeadObjectRequest)
+            final contentLength = headObjectResponse.contentLength()
+            return contentLength!=null ? contentLength : -1L
+       }
+       catch (Exception e){
+           log.error("== Blob cache Error getting content length of object $objectUri from bucket ${blobConfig.storageBucket}", e)
+            return -1L
+       }
+    }
+
+   /**
+    * delete the blob stored in the cache
+    *
+    */
+   protected void deleteBlob(RoutePath route) {
+       final objectUri = blobStorePath(route)
+       log.debug "== Blob cache Deleting object $objectUri"
+       final object = BucketTokenizer.from(objectUri)
+       try {
+            final request =
+                    DeleteObjectRequest.builder()
+                            .bucket(object.bucket)
+                            .key(object.key)
+                            .build()
+            s3Client.deleteObject(request as DeleteObjectRequest)
+            log.debug("== Blob cache Deleted object $objectUri from bucket ${blobConfig.storageBucket}")
+        }
+        catch (Exception e){
+            log.error("== Blob cache Error deleting object $objectUri from bucket ${blobConfig.storageBucket}", e)
+        }
+   }
 }