diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..c579fb3 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,56 @@ +name: CI +on: + push: + branches: + - main + tags: + - '*' + pull_request: + branches: + - '*' + workflow_dispatch: # allow this workflow to be called by other workflows + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install commitlint + run: | + npm install conventional-changelog-conventionalcommits + npm install @commitlint/config-conventional + npm install commitlint@latest + - name: Validate current commit (last commit) with commitlint + if: github.event_name == 'push' + run: npx commitlint --config .github/workflows/commitlint.config.js --from HEAD~1 --to HEAD --verbose + - name: Validate PR commits with commitlint + if: github.event_name == 'pull_request' + run: npx commitlint --config .github/workflows/commitlint.config.js --from ${{ github.event.pull_request.head.sha }}~${{ github.event.pull_request.commits }} --to ${{ github.event.pull_request.head.sha }} --verbose + integration-tests: + runs-on: ubuntu-latest + name: Integration Tests + needs: commitlint + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '21' + cache: 'maven' + - name: Run integration tests + run: | + ./mvnw -B verify -Pnative + env: + GITHUB_ACTOR: ${{ secrets.GITHUB_ACTOR }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/commitlint.config.js b/.github/workflows/commitlint.config.js new file mode 100644 index 0000000..33a7b5c --- /dev/null +++ b/.github/workflows/commitlint.config.js @@ -0,0 +1 @@ +module.exports = { extends: ['@commitlint/config-conventional'] } \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8c7863e --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +#Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +release.properties +.flattened-pom.xml + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# IntelliJ +.idea +*.ipr +*.iml +*.iws + +# NetBeans +nb-configuration.xml + +# Visual Studio Code +.vscode +.factorypath + +# OSX +.DS_Store + +# Vim +*.swp +*.swo + +# patch +*.orig +*.rej + +# Local environment +.env + +# Plugin directory +/.quarkus/cli/plugins/ diff --git a/.mvn/wrapper/.gitignore b/.mvn/wrapper/.gitignore new file mode 100644 index 0000000..e72f5e8 --- /dev/null +++ b/.mvn/wrapper/.gitignore @@ -0,0 +1 @@ +maven-wrapper.jar diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 0000000..84d1e60 --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +public final class MavenWrapperDownloader +{ + private static final String WRAPPER_VERSION = "3.2.0"; + + private static final boolean VERBOSE = Boolean.parseBoolean( System.getenv( "MVNW_VERBOSE" ) ); + + public static void main( String[] args ) + { + log( "Apache Maven Wrapper Downloader " + WRAPPER_VERSION ); + + if ( args.length != 2 ) + { + System.err.println( " - ERROR wrapperUrl or wrapperJarPath parameter missing" ); + System.exit( 1 ); + } + + try + { + log( " - Downloader started" ); + final URL wrapperUrl = new URL( args[0] ); + final String jarPath = args[1].replace( "..", "" ); // Sanitize path + final Path wrapperJarPath = Paths.get( jarPath ).toAbsolutePath().normalize(); + downloadFileFromURL( wrapperUrl, wrapperJarPath ); + log( "Done" ); + } + catch ( IOException e ) + { + System.err.println( "- Error downloading: " + e.getMessage() ); + if ( VERBOSE ) + { + e.printStackTrace(); + } + System.exit( 1 ); + } + } + + private static void downloadFileFromURL( URL wrapperUrl, Path wrapperJarPath ) + throws IOException + { + log( " - Downloading to: " + wrapperJarPath ); + if ( System.getenv( "MVNW_USERNAME" ) != null && System.getenv( "MVNW_PASSWORD" ) != null ) + { + final String username = System.getenv( "MVNW_USERNAME" ); + final char[] password = System.getenv( "MVNW_PASSWORD" ).toCharArray(); + Authenticator.setDefault( new Authenticator() + { + @Override + protected PasswordAuthentication getPasswordAuthentication() + { + return new PasswordAuthentication( username, password ); + } + } ); + } + try ( InputStream inStream = wrapperUrl.openStream() ) + { + Files.copy( inStream, wrapperJarPath, StandardCopyOption.REPLACE_EXISTING ); + } + log( " - Downloader complete" ); + } + + private static void log( String msg ) + { + if ( VERBOSE ) + { + System.out.println( msg ); + } + } + +} diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..70f4f50 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.8/apache-maven-3.8.8-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/.tekton/osv-ingester-pull-request.yaml b/.tekton/osv-ingester-pull-request.yaml new file mode 100644 index 0000000..9cabd79 --- /dev/null +++ b/.tekton/osv-ingester-pull-request.yaml @@ -0,0 +1,397 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RHEcosystemAppEng/osv-ingester?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: exhort + appstudio.openshift.io/component: osv-ingester + pipelines.appstudio.openshift.io/type: build + name: osv-ingester-on-pull-request + namespace: trusted-content-tenant +spec: + params: + - name: dockerfile + value: src/main/docker/Dockerfile.multi-stage + - name: git-url + value: '{{source_url}}' + - name: image-expires-after + value: 5d + - name: output-image + value: quay.io/redhat-user-workloads/trusted-content-tenant/exhort/osv-ingester:on-pr-{{revision}} + - name: path-context + value: . + - name: revision + value: '{{revision}}' + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:1f90faefa39c2e4965793c1d8321e7d5d99a6c941276a9094a4e0d483a598fca + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-container.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.2@sha256:bdf58a8a6bf10482fff841ce6c78c54e87d306bc6aae9515821c436d26daff35 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "false" + description: Java build + name: java + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + - description: "" + name: JAVA_COMMUNITY_DEPENDENCIES + value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.2@sha256:686109bd8088258f73211618824aee5d3cf9e370f65fa3e85d361790a54260ef + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone:0.1@sha256:30709df067659a407968154fd39e99763823d8ecfc6b5cd00a55b68818ec94ba + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:c6fdbf404dc61bf8cf8bec5fc4d7fb15f37ba62f1684de0c68bfbad5723c0052 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.hermetic) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-8gb + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-8gb:0.1@sha256:e3c5aecec9379b1aeb1f404f60669bb7c02ca1a809a9a6d602b8257e864fc98c + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: BASE_IMAGES + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:1f62eaf64a188fcf61f808ad78a15ebf9a8f7f51c644266ad195718b6a2dd372 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: BASE_IMAGES_DIGESTS + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.4@sha256:6b1b325de0af29b6e9a0696f4d2b669a1e6a046941726cc97c5e42785aad870c + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:a6107f78e5fa9e087992f11d788701e4241d9875b153def796fb3bf257c3b7fd + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:b3d2d07394ff983d5f2578c294cd8c4e9428fecc801495feeb929d932c10f740 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:6ba32717bd837ca0d5714b518cc4530e1f1d5bef137df54c02b0c2151b9d217e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sbom-json-check + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: sbom-json-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:dbd467a0507cff1981d3c98f683339feaab1b387c5b5fbf1ff957e9be2e27027 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + - name: git-auth + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/osv-ingester-push.yaml b/.tekton/osv-ingester-push.yaml new file mode 100644 index 0000000..47134b0 --- /dev/null +++ b/.tekton/osv-ingester-push.yaml @@ -0,0 +1,394 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RHEcosystemAppEng/osv-ingester?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: exhort + appstudio.openshift.io/component: osv-ingester + pipelines.appstudio.openshift.io/type: build + name: osv-ingester-on-push + namespace: trusted-content-tenant +spec: + params: + - name: dockerfile + value: src/main/docker/Dockerfile.multi-stage + - name: git-url + value: '{{source_url}}' + - name: output-image + value: quay.io/redhat-user-workloads/trusted-content-tenant/exhort/osv-ingester:{{revision}} + - name: path-context + value: . + - name: revision + value: '{{revision}}' + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:1f90faefa39c2e4965793c1d8321e7d5d99a6c941276a9094a4e0d483a598fca + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-container.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.2@sha256:bdf58a8a6bf10482fff841ce6c78c54e87d306bc6aae9515821c436d26daff35 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "false" + description: Java build + name: java + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + - description: "" + name: JAVA_COMMUNITY_DEPENDENCIES + value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.2@sha256:686109bd8088258f73211618824aee5d3cf9e370f65fa3e85d361790a54260ef + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone:0.1@sha256:30709df067659a407968154fd39e99763823d8ecfc6b5cd00a55b68818ec94ba + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:c6fdbf404dc61bf8cf8bec5fc4d7fb15f37ba62f1684de0c68bfbad5723c0052 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.hermetic) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-8gb + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-8gb:0.1@sha256:e3c5aecec9379b1aeb1f404f60669bb7c02ca1a809a9a6d602b8257e864fc98c + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: BASE_IMAGES + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:1f62eaf64a188fcf61f808ad78a15ebf9a8f7f51c644266ad195718b6a2dd372 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: BASE_IMAGES_DIGESTS + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.4@sha256:6b1b325de0af29b6e9a0696f4d2b669a1e6a046941726cc97c5e42785aad870c + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:a6107f78e5fa9e087992f11d788701e4241d9875b153def796fb3bf257c3b7fd + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:b3d2d07394ff983d5f2578c294cd8c4e9428fecc801495feeb929d932c10f740 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:6ba32717bd837ca0d5714b518cc4530e1f1d5bef137df54c02b0c2151b9d217e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sbom-json-check + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: sbom-json-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:dbd467a0507cff1981d3c98f683339feaab1b387c5b5fbf1ff957e9be2e27027 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + - name: git-auth + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/deploy/openshift/template.yaml b/deploy/openshift/template.yaml new file mode 100644 index 0000000..0a6f33b --- /dev/null +++ b/deploy/openshift/template.yaml @@ -0,0 +1,104 @@ +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: onguard +labels: + template: onguard +objects: + - kind: ServiceAccount + apiVersion: v1 + metadata: + name: '${SERVICE_ACCOUNT_NAME}' + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: '${APP_NAME}' + spec: + concurrencyPolicy: Forbid + schedule: "${CRONTAB_EXPRESSION}" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: '${SERVICE_ACCOUNT_NAME}' + containers: + - name: '${APP_NAME}' + image: '${IMAGE}:${IMAGE_TAG}' + imagePullPolicy: IfNotPresent + command: + - ./application + - gcs + env: + - name: DB_REDIS_HOST + valueFrom: + secretKeyRef: + name: '${ELASTICACHE_SECRET}' + key: db.host + - name: DB_REDIS_PORT + valueFrom: + secretKeyRef: + name: '${ELASTICACHE_SECRET}' + key: db.port + resources: + limits: + cpu: ${CPU_LIMIT} + memory: ${MEMORY_LIMIT} + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} +parameters: + - name: APP_NAME + displayName: Application name + description: Application name + value: onguard + required: true + - name: IMAGE + displayName: Container image name + description: Container image name + value: quay.io/ecosystem-appeng/osv-ingester + required: true + - name: IMAGE_TAG + displayName: Container image tag + description: Container image tag + value: latest + required: true + - name: SERVICE_ACCOUNT_NAME + displayName: ServiceAccount name + description: The name of the ServiceAccount to use to run this pod. + value: onguard-sa + - name: ELASTICACHE_SECRET + displayName: Elasticache Secret + description: Name of the secret containing the Elasticache settings + value: onguard-elasticache + required: true + - name: SERVICE_NAME + displayName: Service name + description: Service name + value: onguard + required: true + - name: CRONTAB_EXPRESSION + displayName: Crontab expression + description: Crontab expression + value: "30 00,12 * * *" + required: true + - name: CPU_REQUEST + description: The minimum amount of CPU required by a container + displayName: Memory Limit + required: true + value: 5m + - name: CPU_LIMIT + description: The maximum amount of CPU the container can use. + displayName: Memory Limit + required: true + value: 10m + - name: MEMORY_REQUEST + description: The minimum amount of memory required by a container + displayName: Memory Limit + required: true + value: 10Mi + - name: MEMORY_LIMIT + description: The maximum amount of memory the container can use. + displayName: Memory Limit + required: true + value: 100Mi diff --git a/deploy/osv-ingester.yaml b/deploy/osv-ingester.yaml new file mode 100644 index 0000000..dbf3571 --- /dev/null +++ b/deploy/osv-ingester.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: osv-ingester +spec: + concurrencyPolicy: Forbid + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: osv-ingester + image: quay.io/ruben/osv-ingester:latest + imagePullPolicy: IfNotPresent + command: + - ./application + - gcs + env: + - name: DB_REDIS_HOST + valueFrom: + secretKeyRef: + name: exhort-onguard-secret + key: db.host + - name: DB_REDIS_PORT + valueFrom: + secretKeyRef: + name: exhort-onguard-secret + key: db.port + resources: + limits: + cpu: 10m + memory: 100Mi + requests: + cpu: 1m + memory: 50Mi \ No newline at end of file diff --git a/deploy/redis.yaml b/deploy/redis.yaml new file mode 100644 index 0000000..75e5486 --- /dev/null +++ b/deploy/redis.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: docker.io/redis/redis-stack:latest + imagePullPolicy: IfNotPresent + ports: + - name: redis + containerPort: 6379 + protocol: TCP + - name: http + containerPort: 8001 + protocol: TCP + volumeMounts: + - name: logs + mountPath: /redisinsight/logs + - name: data + mountPath: /data + volumes: + - name: logs + emptyDir: {} + - name: data + persistentVolumeClaim: + claimName: redis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + labels: + app: redis +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: 6379 + - name: http + port: 8001 + protocol: TCP + targetPort: 8001 + selector: + app: redis +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data +spec: + storageClassName: gp2 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi \ No newline at end of file diff --git a/devfile.yaml b/devfile.yaml new file mode 100644 index 0000000..75040c0 --- /dev/null +++ b/devfile.yaml @@ -0,0 +1,32 @@ +schemaVersion: 2.2.0 +metadata: + name: osv-ingester + version: 1.0.0 + provider: Red Hat + supportUrl: https://github.com/RHEcosystemAppEng/osv-ingester/issues + website: https://github.com/RHEcosystemAppEng/osv-ingester + displayName: OSV Ingester + description: OSV Ingester Service that populates the databes data from OSV + tags: + - Exhort + - RHTPA + - Java + - Quarkus + - OSV + projectType: Quarkus + language: Java +parent: + id: java-quarkus + registryUrl: 'https://registry.devfile.io' +components: + - name: image-build + image: + imageName: osv-ingester:latest + dockerfile: + uri: src/main/docker/Dockerfile.multi-stage + buildContext: . + rootRequired: false +commands: + - id: build-image + apply: + component: image-build diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..8d937f4 --- /dev/null +++ b/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..c4586b5 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,205 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.2.0 +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %WRAPPER_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file +SET WRAPPER_SHA_256_SUM="" +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +) +IF NOT %WRAPPER_SHA_256_SUM%=="" ( + powershell -Command "&{"^ + "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ + "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ + " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ + " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ + " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ + " exit 1;"^ + "}"^ + "}" + if ERRORLEVEL 1 goto error +) + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..c946d65 --- /dev/null +++ b/pom.xml @@ -0,0 +1,413 @@ + + + 4.0.0 + com.redhat.ecosystemappeng.exhort + osv-ingester + 1.0.0-SNAPSHOT + OSV Ingester + Red Hat Trusted Profile Analyser :: Exhort :: OSV Ingester + + + 17 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + true + + 3.2.0 + 3.11.0 + 3.5.0 + 3.1.1 + 3.3.0 + 3.1.0 + 3.0.1 + 1.6.2 + 3.1.2 + 4.1 + 1.9.0 + + + 3.9.1 + 3.4.2 + + + + + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + GitHub Issues + https://github.com/RHEcosystemAppEng/osv-ingester/issues + + + + https://github.com/RHEcosystemAppEng/osv-ingester + scm:git:git@github.com:RHEcosystemAppEng/osv-ingester.git + scm:git:git@github.com:RHEcosystemAppEng/osv-ingester.git + + + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + ${quarkus.platform.group-id} + quarkus-google-cloud-services-bom + ${quarkus.platform.version} + pom + import + + + + + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-smallrye-health + + + io.quarkus + quarkus-redis-client + + + io.quarkus + quarkus-redis-cache + + + io.quarkus + quarkus-rest-jackson + + + io.quarkus + quarkus-smallrye-openapi + + + io.quarkus + quarkus-smallrye-fault-tolerance + + + io.quarkus + quarkus-scheduler + + + io.quarkiverse.googlecloudservices + quarkus-google-cloud-storage + + + io.quarkus + quarkus-elytron-security-properties-file + + + io.quarkus + quarkus-picocli + + + io.quarkus + quarkus-junit5 + test + + + io.quarkus + quarkus-junit5-mockito + test + + + io.rest-assured + rest-assured + test + + + org.wiremock + wiremock-standalone + ${wiremock.version} + test + + + + + + + + maven-clean-plugin + ${maven-clean-plugin.version} + + + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + maven-deploy-plugin + ${maven-deploy-plugin.version} + + + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + + com.mycila + license-maven-plugin + ${license-maven-plugin.version} + + + Red Hat, Inc. and/or its affiliates + + + + + **/module-info.java + **/package-info.java + **/module-info.test + src/test/resources/** + src/main/resources/** + src/it/**/test/resources/** + src/it/**/main/resources/** + + + src/main/** + src/test/** + src/it/**/src/main/** + src/it/**/src/test/** + + + + + + + false + + + + net.revelc.code + impsort-maven-plugin + ${impsort-maven-plugin.version} + + java.,javax.,org.,com. + java,* + + + + sort-imports + + check + + + + + + + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + + + + + + maven-compiler-plugin + + + -parameters + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + ${project.build.directory}/${project.build.finalName}-runner + org.jboss.logmanager.LogManager + ${maven.home} + + + + + + + maven-release-plugin + ${maven-release-plugin.version} + + -Pprepare-deployment,sign-deployment + v@{project.version} + build(release): + + + + maven-deploy-plugin + ${maven-deploy-plugin.version} + + true + + + + maven-enforcer-plugin + + + + enforce + + + + + + org.codehaus.mojo + extra-enforcer-rules + ${extra-enforcer-rules.version} + + + + + + + [11,) + + + [3.8,) + + + + compile + provided + + + + + + + + com.mycila + license-maven-plugin + + + + check + + + + + + net.revelc.code + impsort-maven-plugin + + + + + + native + + + native + + + + false + native + + + + prepare-deployment + + + + maven-source-plugin + + + + jar-no-fork + + + + + + + + + sign-deployment + + + + maven-gpg-plugin + ${maven-gpg-plugin.version} + + + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + + + \ No newline at end of file diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm new file mode 100644 index 0000000..896adf1 --- /dev/null +++ b/src/main/docker/Dockerfile.jvm @@ -0,0 +1,95 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./mvnw package +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/osv-ingester-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/osv-ingester-jvm +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/osv-ingester-jvm +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.redhat.io/ubi8/openjdk-17:1.16 + +ENV LANGUAGE='en_US:en' + + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/ +COPY --chown=185 target/quarkus-app/*.jar /deployments/ +COPY --chown=185 target/quarkus-app/app/ /deployments/app/ +COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS="-Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + diff --git a/src/main/docker/Dockerfile.multi-stage b/src/main/docker/Dockerfile.multi-stage new file mode 100644 index 0000000..94b1ba2 --- /dev/null +++ b/src/main/docker/Dockerfile.multi-stage @@ -0,0 +1,35 @@ +## Stage 1 : build with maven builder image with native capabilities +FROM registry.redhat.io/quarkus/mandrel-23-rhel8:23.0 AS build + +COPY --chown=quarkus:quarkus mvnw /code/mvnw +COPY --chown=quarkus:quarkus .mvn /code/.mvn +COPY --chown=quarkus:quarkus pom.xml /code/ + +USER quarkus +WORKDIR /code +RUN ./mvnw -B org.apache.maven.plugins:maven-dependency-plugin:3.6.1:go-offline +COPY --chown=quarkus:quarkus src /code/src +RUN ./mvnw package -B -Pnative -DskipTests=true + +## Stage 2 : create the docker final image +FROM registry.redhat.io/ubi9/ubi-minimal:9.4 + +LABEL description="Red Hat Trusted Profile Analyzer - OSV Ingester Service" +LABEL io.k8s.description="Red Hat Trusted Profile Analyzer - OSV Ingester Service" +LABEL io.k8s.display-name="RHTPA OSV Ingester service" +LABEL io.openshift.tags="rhtpa exhort osv-ingester" +LABEL summary="The RHTPA OSV Ingester Service ingests the OSV public source into the ONGuard Database" + +WORKDIR /work/ +COPY --from=build /code/target/*-runner /work/application + +# set up permissions for user `1001` +RUN chmod 775 /work /work/application \ + && chown -R 1001 /work \ + && chmod -R "g+rwX" /work \ + && chown -R 1001:root /work + +EXPOSE 8080 +USER 1001 + +CMD ["./application"] diff --git a/src/main/docker/Dockerfile.native b/src/main/docker/Dockerfile.native new file mode 100644 index 0000000..3be8c71 --- /dev/null +++ b/src/main/docker/Dockerfile.native @@ -0,0 +1,27 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# +# Before building the container image run: +# +# ./mvnw package -Pnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native -t quarkus/osv-ingester . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/osv-ingester +# +### +FROM registry.redhat.io/ubi9/ubi-minimal:9.3 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +CMD ["./application"] diff --git a/src/main/docker/Dockerfile.native-micro b/src/main/docker/Dockerfile.native-micro new file mode 100644 index 0000000..38bb7d0 --- /dev/null +++ b/src/main/docker/Dockerfile.native-micro @@ -0,0 +1,30 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# It uses a micro base image, tuned for Quarkus native executables. +# It reduces the size of the resulting container image. +# Check https://quarkus.io/guides/quarkus-runtime-base-image for further information about this image. +# +# Before building the container image run: +# +# ./mvnw package -Pnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native-micro -t quarkus/osv-ingester . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/osv-ingester gcs --ecosystem Maven +# +### +FROM quay.io/quarkus/quarkus-micro-image:2.0 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +CMD ["./application"] diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/GCSIngesterCommand.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/GCSIngesterCommand.java new file mode 100644 index 0000000..96c93e8 --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/GCSIngesterCommand.java @@ -0,0 +1,43 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.cli; + +import com.redhat.ecosystemappeng.onguard.osvingester.service.EcosystemLoader; + +import picocli.CommandLine; + +@CommandLine.Command(name = "gcs", description = "Loads Vulnerability data from OSV Google Cloud Storage Bucket") +public class GCSIngesterCommand implements Runnable { + @CommandLine.Option(names = { "--ecosystem" }, description = "Ecosystem to load", defaultValue = "all") + String ecosystem; + + private final EcosystemLoader loader; + + public GCSIngesterCommand(EcosystemLoader loader) { + this.loader = loader; + } + + @Override + public void run() { + if ("all".equalsIgnoreCase(ecosystem)) { + loader.loadAll().collect().asList().await().indefinitely(); + } else { + loader.load(ecosystem).await().indefinitely(); + } + } +} diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/OsvIngesterCommand.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/OsvIngesterCommand.java new file mode 100644 index 0000000..a718055 --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/cli/OsvIngesterCommand.java @@ -0,0 +1,27 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.cli; + +import io.quarkus.picocli.runtime.annotations.TopCommand; +import picocli.CommandLine; + +@TopCommand +@CommandLine.Command(name = "osv-ingester", mixinStandardHelpOptions = true, subcommands = {GCSIngesterCommand.class}) +public class OsvIngesterCommand { + +} diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/model/IngestionReport.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/model/IngestionReport.java new file mode 100644 index 0000000..8a80518 --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/model/IngestionReport.java @@ -0,0 +1,24 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.model; + +import java.util.List; + +public record IngestionReport(String name, int total, List withoutSeverity) { + +} diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/EcosystemLoader.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/EcosystemLoader.java new file mode 100644 index 0000000..51234f2 --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/EcosystemLoader.java @@ -0,0 +1,31 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.service; + +import com.redhat.ecosystemappeng.onguard.osvingester.model.IngestionReport; + +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; + +public interface EcosystemLoader { + + Multi loadAll(); + + Uni load(String ecosystem); + +} diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/VulnerabilityIngester.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/VulnerabilityIngester.java new file mode 100644 index 0000000..f11e013 --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/VulnerabilityIngester.java @@ -0,0 +1,116 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.service; + +import java.io.IOException; + +import org.jboss.logging.Logger; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; + +import io.quarkus.redis.datasource.RedisDataSource; +import io.quarkus.redis.datasource.value.ValueCommands; +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class VulnerabilityIngester { + + final private ValueCommands vulnCommands; + final private ObjectMapper objectMapper; + + private final static Logger LOGGER = Logger.getLogger(VulnerabilityIngester.class); + + VulnerabilityIngester(RedisDataSource ds, ObjectMapper objectMapper) { + this.vulnCommands = ds.value(String.class); + this.objectMapper = objectMapper; + } + + public String save(String source, byte[] content) { + JsonNode obj; + try { + obj = objectMapper.readTree(content); + var id = obj.get("id").asText(); + var hasSeverity = obj.get("severity") != null; + if (hasSeverity) { + vulnCommands.set(vulnKey(id), obj.toString()); + } else { + var existingRaw = vulnCommands.get(vulnKey(id)); + if (existingRaw == null) { + vulnCommands.set(vulnKey(id), obj.toString()); + } else { + var existingHasSeverity = objectMapper.readTree(existingRaw).has("severity"); + if (!existingHasSeverity) { + vulnCommands.set(vulnKey(id), obj.toString()); + } + } + } + if (!hasSeverity) { + return id; + } + return null; + } catch (IOException e) { + LOGGER.errorf("Unable to parse Json vulnerability: %s", source, e); + return null; + } + } + + public String get(String vulnId) { + return vulnCommands.get(vulnKey(vulnId)); + } + + public boolean reconcile(String vulnId) { + var vuln = get(vulnId); + try { + var obj = objectMapper.readTree(vuln); + var aliases = obj.withArray("aliases"); + for (var aliasId : aliases) { + var alias = get(aliasId.asText()); + if (alias != null) { + try { + var aliasObj = objectMapper.readTree(alias); + if (aliasObj.has("severity")) { + LOGGER.debugf("Must reconcile %s and %s", vulnId, aliasId); + var merged = mergeSeverity(obj, aliasObj); + vulnCommands.set(vulnKey(vulnId), merged.toPrettyString()); + return true; + } + } catch (JsonProcessingException e) { + LOGGER.error("Unable to parse Json vulnerability: %s", aliasId, e); + } + } + } + } catch (JsonProcessingException e) { + LOGGER.error("Unable to parse Json vulnerability: %s", vulnId, e); + } + return false; + } + + private String vulnKey(String id) { + return "vuln:" + id; + } + + private JsonNode mergeSeverity(JsonNode base, JsonNode source) { + var merged = JsonNodeFactory.instance.objectNode(); + base.fields().forEachRemaining(entry -> merged.set(entry.getKey(), entry.getValue())); + merged.set("severity", source.get("severity")); + return merged; + } +} diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/googlecloudstorage/GCSLoader.java b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/googlecloudstorage/GCSLoader.java new file mode 100644 index 0000000..6b8730c --- /dev/null +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/osvingester/service/googlecloudstorage/GCSLoader.java @@ -0,0 +1,143 @@ +/* + * Copyright ${year} Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.redhat.ecosystemappeng.onguard.osvingester.service.googlecloudstorage; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.redhat.ecosystemappeng.onguard.osvingester.model.IngestionReport; +import com.redhat.ecosystemappeng.onguard.osvingester.service.EcosystemLoader; +import com.redhat.ecosystemappeng.onguard.osvingester.service.VulnerabilityIngester; + +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.infrastructure.Infrastructure; +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class GCSLoader implements EcosystemLoader { + + private static final Logger LOGGER = Logger.getLogger(GCSLoader.class); + + private final VulnerabilityIngester vulnerabilityIngester; + private final Bucket bucket; + private final ExecutorService executor = Executors.newFixedThreadPool(20); + + GCSLoader(Storage storage, + @ConfigProperty(name = "import.osv-bucket", defaultValue = "osv-vulnerabilities") String osvBucketName, + VulnerabilityIngester vulnerabilityIngester) { + this.bucket = storage.get(osvBucketName); + this.vulnerabilityIngester = vulnerabilityIngester; + } + + private Stream listEcosystems() { + var ecosystems = bucket.get("ecosystems.txt"); + var content = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(ecosystems.getContent()))); + + return content.lines(); + } + + public Multi loadAll() { + return Multi.createFrom() + .items(listEcosystems()) + .onItem() + .transformToUniAndMerge(this::load) + .runSubscriptionOn(executor) + .onCompletion() + .invoke(() -> LOGGER.info("Completed load of all ecosystems")); + } + + public Uni load(String ecosystem) { + return Uni.createFrom().item(bucket.get(ecosystem + "/all.zip")) + .emitOn(Infrastructure.getDefaultWorkerPool()) + .onItem() + .transform(all -> { + LOGGER.infof("Loading [%s]", ecosystem); + return load(ecosystem, new ByteArrayInputStream(all.getContent())); + }) + .onFailure().retry().withBackOff(Duration.ofSeconds(2)).atMost(5) + .onFailure().recoverWithItem(e -> { + LOGGER.error("Failed to load vulnerabilities for: " + ecosystem, e); + return new IngestionReport(ecosystem, 0, Collections.emptyList()); + }) + .onItem() + .transform(report -> { + List missing = new ArrayList<>(); + for (var item : report.withoutSeverity()) { + if (!vulnerabilityIngester.reconcile(item)) { + missing.add(item); + } + } + return new IngestionReport(report.name(), report.total(), missing); + }).onItem() + .invoke(report -> LOGGER.infof("Completed [%s] / Total vulnerabilities [%s] / Without severity [%s]", + report.name(), report.total(), report.withoutSeverity().size())); + } + + private IngestionReport load(String ecosystem, ByteArrayInputStream allFile) { + var count = 0; + List incomplete = new ArrayList<>(); + try (var zis = new ZipInputStream(allFile)) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (!entry.isDirectory() && entry.getName().endsWith(".json")) { + var content = readZipEntryContent(zis); + count++; + var incompleteVuln = vulnerabilityIngester.save(entry.getName(), content); + if (incompleteVuln != null) { + incomplete.add(incompleteVuln); + } + } + } + zis.closeEntry(); + } catch (IOException e) { + LOGGER.error("Unable to process Zip file: " + ecosystem, e); + } + return new IngestionReport(ecosystem, count, incomplete); + } + + // Utility method to read the content of a ZipEntry + private byte[] readZipEntryContent(InputStream zipInputStream) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = zipInputStream.read(buffer)) > 0) { + byteArrayOutputStream.write(buffer, 0, length); + } + return byteArrayOutputStream.toByteArray(); + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..8eb4c6a --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,12 @@ +quarkus.banner.enabled=false +quarkus.http.host-enabled=false + +quarkus.redis.max-pool-waiting=100 + +%prod.quarkus.redis.hosts=redis://${db.redis.host:localhost}:${db.redis.port:6379}/ + +%prod.quarkus.log.level=ERROR +%prod.quarkus.log.category."com.redhat.ecosystemappeng.onguard.osvingester".level=INFO +%prod.quarkus.log.category."com.redhat.ecosystemappeng.onguard.osvingester".handlers=cli-handler +%prod.quarkus.log.handler.console.cli-handler.format=%d{yyyy-MM-dd HH:mm:ss} %m%n +%prod.quarkus.log.category."com.redhat.ecosystemappeng.onguard.osvingester".use-parent-handlers=false \ No newline at end of file