From 801a0ef450e11b54abed5a442af365fc9cc5874f Mon Sep 17 00:00:00 2001 From: Ravi Sundriyal Date: Fri, 13 Sep 2024 20:37:11 -0400 Subject: [PATCH] Jenkins: Add gitguardian stage to test pipeline Also changes the Jenkinsfile from scripted to declarative. --- Jenkinsfile | 282 +++++++++++++++++++++++++++++----------------------- 1 file changed, 155 insertions(+), 127 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 58f2857f87..3b3b1aa924 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,159 +56,187 @@ properties( ] ) -node('default') { - stage('Generate Tarball') { - cleanWs() +pipeline { - checkout scm - - dir(path: 'clamav_documentation') { - git(url: 'https://github.com/Cisco-Talos/clamav-documentation.git', branch: "gh-pages") - } - - dir(path: 'docs/html') { - sh '''# Move the clamav-documentation here. - cp -r ../../clamav_documentation/* . - # Clean-up - rm -rf ../../clamav_documentation - rm -rf .git .nojekyll CNAME Placeholder || true - ''' - } + agent { + label "default" + } - dir(path: 'build') { - sh """# CPack - cmake .. -D VENDOR_DEPENDENCIES=ON \ - -D JSONC_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/json-c" \ - -D JSONC_LIBRARY="$HOME/.mussels/install/host-static/lib/libjson-c.a" \ - -D ENABLE_JSON_SHARED=OFF \ - -D BZIP2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D BZIP2_LIBRARY_RELEASE="$HOME/bzip2-1.0.8-install/lib/libbz2.a" \ - -D OPENSSL_ROOT_DIR="$HOME/.mussels/install/host-static" \ - -D OPENSSL_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D OPENSSL_CRYPTO_LIBRARY="$HOME/.mussels/install/host-static/lib/libcrypto.a" \ - -D OPENSSL_SSL_LIBRARY="$HOME/.mussels/install/host-static/lib/libssl.a" \ - -D LIBXML2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/libxml2" \ - -D LIBXML2_LIBRARY="$HOME/.mussels/install/host-static/lib/libxml2.a" \ - -D PCRE2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D PCRE2_LIBRARY="$HOME/.mussels/install/host-static/lib/libpcre2-8.a" \ - -D CURSES_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D CURSES_LIBRARY="$HOME/.mussels/install/host-static/lib/libncurses.a;$HOME/.mussels/install/host-static/lib/libtinfo.a" \ - -D ZLIB_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D ZLIB_LIBRARY="$HOME/.mussels/install/host-static/lib/libz.a" \ - -D LIBCHECK_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ - -D LIBCHECK_LIBRARY="$HOME/.mussels/install/host-static/lib/libcheck.a" + stages { - cpack --config CPackSourceConfig.cmake """ - archiveArtifacts(artifacts: "clamav-${params.VERSION}*.tar.gz", onlyIfSuccessful: true) + stage('GitGuardian Scan') { + environment { + GITGUARDIAN_API_KEY = credentials('gitguardian-token') + GITGUARDIAN_API_URL = 'https://gitguardian.cisco.com/' + } + agent { label "docker" } + steps { + withDockerContainer(args: "-i --entrypoint=''", image: 'gitguardian/ggshield:latest') { + sh 'ggshield secret scan ci' + } + } } - cleanWs() - } + stage('Generate Tarball') { + steps { + cleanWs() - def buildResult + checkout scm - stage('Build') { - buildResult = build(job: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}", - propagate: true, - wait: true, - parameters: [ - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], - [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], - [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] - ] - ) - echo "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE} #${buildResult.number} succeeded." - } + dir(path: 'clamav_documentation') { + git(url: 'https://github.com/Cisco-Talos/clamav-documentation.git', branch: "gh-pages") + } - stage('Test') { - def tasks = [:] + dir(path: 'docs/html') { + sh """# Move the clamav-documentation here. + cp -r ../../clamav_documentation/ . + # Clean-up + rm -rf ../../clamav_documentation + rm -rf .git .nojekyll CNAME Placeholder || true + """ + } - tasks["package_regular_custom"] = { - def exception = null - try { - stage("Package") { - final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE}", - propagate: true, - wait: true, - parameters: [ - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], - [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"], - [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], - [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], - [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], - [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] - ] - ) - echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} #${regularResult.number} succeeded." + dir(path: 'build') { + sh """# CPack + cmake .. -D VENDOR_DEPENDENCIES=ON \ + -D JSONC_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/json-c" \ + -D JSONC_LIBRARY="$HOME/.mussels/install/host-static/lib/libjson-c.a" \ + -D ENABLE_JSON_SHARED=OFF \ + -D BZIP2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D BZIP2_LIBRARY_RELEASE="$HOME/bzip2-1.0.8-install/lib/libbz2.a" \ + -D OPENSSL_ROOT_DIR="$HOME/.mussels/install/host-static" \ + -D OPENSSL_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D OPENSSL_CRYPTO_LIBRARY="$HOME/.mussels/install/host-static/lib/libcrypto.a" \ + -D OPENSSL_SSL_LIBRARY="$HOME/.mussels/install/host-static/lib/libssl.a" \ + -D LIBXML2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include/libxml2" \ + -D LIBXML2_LIBRARY="$HOME/.mussels/install/host-static/lib/libxml2.a" \ + -D PCRE2_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D PCRE2_LIBRARY="$HOME/.mussels/install/host-static/lib/libpcre2-8.a" \ + -D CURSES_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D CURSES_LIBRARY="$HOME/.mussels/install/host-static/lib/libncurses.a;$HOME/.mussels/install/host-static/lib/libtinfo.a" \ + -D ZLIB_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D ZLIB_LIBRARY="$HOME/.mussels/install/host-static/lib/libz.a" \ + -D LIBCHECK_INCLUDE_DIR="$HOME/.mussels/install/host-static/include" \ + -D LIBCHECK_LIBRARY="$HOME/.mussels/install/host-static/lib/libcheck.a" + + cpack --config CPackSourceConfig.cmake + """ + archiveArtifacts(artifacts: "clamav-${params.VERSION}*.tar.gz", onlyIfSuccessful: true) } - } catch (exc) { - echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} failed." - exception = exc + cleanWs() } + } - try { - stage("Regular From-Source") { - final regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE}", + stage('Build') { + steps { + script{ + buildResult = build(job: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}", propagate: true, wait: true, parameters: [ [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] ] ) - echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} #${regularResult.number} succeeded." + echo "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE} #${buildResult.number} succeeded." } - } catch (exc) { - echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} failed." - exception = exc - } - - stage("Custom From-Source") { - final customResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE}", - propagate: true, - wait: true, - parameters: [ - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_CUSTOM_BRANCH}"], - [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], - [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], - [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] - ] - ) - echo "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE} #${customResult.number} succeeded." - } - if(exception != null) { - echo "Custom Pipeline passed, but prior pipelines failed!" - throw exception } } - tasks["fuzz_regression"] = { - stage("Fuzz Regression") { - final fuzzResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE}", - propagate: true, - wait: true, - parameters: [ - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], - [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], - [$class: 'StringParameterValue', name: 'TESTS_FUZZ_BRANCH', value: "${params.TESTS_FUZZ_BRANCH}"], - [$class: 'StringParameterValue', name: 'FUZZ_CORPUS_BRANCH', value: "${params.FUZZ_CORPUS_BRANCH}"], - [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"] - ] - ) - echo "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE} #${fuzzResult.number} succeeded." + stage('Tests') { + failFast false + parallel { + stage('Pipeline') { + stages{ + stage("Package") { + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + script{ + packageResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE}", + propagate: true, + wait: true, + parameters: [ + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], + [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], + [$class: 'StringParameterValue', name: 'BUILD_JOB_NAME', value: "${params.BUILD_PIPELINES_PATH}/${params.BUILD_PIPELINE}"], + [$class: 'StringParameterValue', name: 'BUILD_JOB_NUMBER', value: "${buildResult.number}"], + [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], + [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], + [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] + ] + ) + echo "${params.TEST_PIPELINES_PATH}/${params.PACKAGE_PIPELINE} #${packageResult.number} succeeded." + } + } + } + } + + stage("Regular From-Source") { + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + script{ + regularResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE}", + propagate: true, + wait: true, + parameters: [ + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], + [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_BRANCH}"], + [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], + [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], + [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] + ] + ) + echo "${params.TEST_PIPELINES_PATH}/${params.REGULAR_PIPELINE} #${regularResult.number} succeeded." + } + } + } + } + + stage("Custom From-Source") { + steps { + script{ + customResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE}", + propagate: true, + wait: true, + parameters: [ + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], + [$class: 'StringParameterValue', name: 'TESTS_BRANCH', value: "${params.TESTS_CUSTOM_BRANCH}"], + [$class: 'StringParameterValue', name: 'FRAMEWORK_BRANCH', value: "${params.FRAMEWORK_BRANCH}"], + [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"], + [$class: 'StringParameterValue', name: 'SHARED_LIB_BRANCH', value: "${params.SHARED_LIB_BRANCH}"] + ] + ) + echo "${params.TEST_PIPELINES_PATH}/${params.CUSTOM_PIPELINE} #${customResult.number} succeeded." + } + } + } + } + } + stage("Fuzz Regression") { + steps { + script{ + fuzzResult = build(job: "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE}", + propagate: true, + wait: true, + parameters: [ + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NAME', value: "${JOB_NAME}"], + [$class: 'StringParameterValue', name: 'CLAMAV_JOB_NUMBER', value: "${BUILD_NUMBER}"], + [$class: 'StringParameterValue', name: 'TESTS_FUZZ_BRANCH', value: "${params.TESTS_FUZZ_BRANCH}"], + [$class: 'StringParameterValue', name: 'FUZZ_CORPUS_BRANCH', value: "${params.FUZZ_CORPUS_BRANCH}"], + [$class: 'StringParameterValue', name: 'VERSION', value: "${params.VERSION}"] + ] + ) + echo "${params.TEST_PIPELINES_PATH}/${params.FUZZ_PIPELINE} #${fuzzResult.number} succeeded." + } + } + } } } - - parallel tasks } -} +} \ No newline at end of file