-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #33 from ONSdigital/develop
Create release for 0.2.0-alpha
- Loading branch information
Showing
40 changed files
with
2,629 additions
and
324 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,69 +1,69 @@ | ||
# See https://pre-commit.com for more information | ||
# See https://pre-commit.com/hooks.html for more hooks | ||
repos: | ||
- repo: https://github.com/kynan/nbstripout | ||
rev: 0.4.0 | ||
hooks: | ||
- id: nbstripout | ||
name: nbstripout - Strip outputs from notebooks (auto-fixes) | ||
args: | ||
- --extra-keys | ||
- "metadata.colab metadata.kernelspec cell.metadata.colab cell.metadata.executionInfo cell.metadata.id cell.metadata.outputId" | ||
- repo: https://github.com/pre-commit/pre-commit-hooks | ||
rev: v4.0.1 | ||
hooks: | ||
- id: check-added-large-files | ||
name: Check for files larger than 5 MB | ||
args: [ "--maxkb=5120" ] | ||
- id: end-of-file-fixer | ||
name: Check for a blank line at the end of scripts (auto-fixes) | ||
exclude: '\.Rd' | ||
- id: trailing-whitespace | ||
name: Check for trailing whitespaces (auto-fixes) | ||
- repo: https://github.com/pycqa/isort | ||
rev: 5.8.0 | ||
hooks: | ||
- id: isort | ||
name: isort - Sort Python imports (auto-fixes) | ||
types: [ cython, pyi, python ] | ||
args: [ "--profile", "black", "--filter-files" ] | ||
- repo: https://github.com/psf/black | ||
rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags | ||
hooks: | ||
- id: black | ||
name: black - consistent Python code formatting (auto-fixes) | ||
language_version: python # Should be a command that runs python3.6+ | ||
- repo: https://github.com/PyCQA/flake8 | ||
rev: 5.0.4 | ||
hooks: | ||
- id: flake8 | ||
name: flake8 - Python linting | ||
- repo: https://github.com/nbQA-dev/nbQA | ||
rev: 0.12.0 | ||
hooks: | ||
- id: nbqa-isort | ||
name: nbqa-isort - Sort Python imports (notebooks; auto-fixes) | ||
args: [ --nbqa-mutate ] | ||
additional_dependencies: [ isort==5.8.0 ] | ||
- id: nbqa-black | ||
name: nbqa-black - consistent Python code formatting (notebooks; auto-fixes) | ||
args: [ --nbqa-mutate ] | ||
additional_dependencies: [ black==21.5b2 ] | ||
# TODO: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook | ||
#- id: nbqa-flake8 | ||
# name: nbqa-flake8 - Python linting (notebooks) | ||
# additional_dependencies: [ flake8==3.9.2 ] | ||
- repo: https://github.com/Yelp/detect-secrets | ||
rev: v1.0.3 | ||
hooks: | ||
- id: detect-secrets | ||
name: detect-secrets - Detect secrets in staged code | ||
args: [ "--baseline", ".secrets.baseline" ] | ||
exclude: .*/tests/.*|^\.cruft\.json$ | ||
- repo: local | ||
hooks: | ||
- id: restricted-filenames | ||
name: Check commits for restricted file extensions | ||
entry: These file extensions are restricted. Data should be removed from the commit | ||
language: fail | ||
files: .*\.(csv|feather|xlsx|zip|hdf5|h5|txt|json|xml|hd|parquet) | ||
- repo: https://github.com/kynan/nbstripout | ||
rev: 0.4.0 | ||
hooks: | ||
- id: nbstripout | ||
name: nbstripout - Strip outputs from notebooks (auto-fixes) | ||
args: | ||
- --extra-keys | ||
- "metadata.colab metadata.kernelspec cell.metadata.colab cell.metadata.executionInfo cell.metadata.id cell.metadata.outputId" | ||
- repo: https://github.com/pre-commit/pre-commit-hooks | ||
rev: v4.0.1 | ||
hooks: | ||
- id: check-added-large-files | ||
name: Check for files larger than 5 MB | ||
args: ["--maxkb=5120"] | ||
- id: end-of-file-fixer | ||
name: Check for a blank line at the end of scripts (auto-fixes) | ||
exclude: '\.Rd' | ||
- id: trailing-whitespace | ||
name: Check for trailing whitespaces (auto-fixes) | ||
- repo: https://github.com/pycqa/isort | ||
rev: 5.8.0 | ||
hooks: | ||
- id: isort | ||
name: isort - Sort Python imports (auto-fixes) | ||
types: [cython, pyi, python] | ||
args: ["--profile", "black", "--filter-files"] | ||
- repo: https://github.com/psf/black | ||
rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags | ||
hooks: | ||
- id: black | ||
name: black - consistent Python code formatting (auto-fixes) | ||
language_version: python # Should be a command that runs python3.6+ | ||
- repo: https://github.com/PyCQA/flake8 | ||
rev: 5.0.4 | ||
hooks: | ||
- id: flake8 | ||
name: flake8 - Python linting | ||
- repo: https://github.com/nbQA-dev/nbQA | ||
rev: 0.12.0 | ||
hooks: | ||
- id: nbqa-isort | ||
name: nbqa-isort - Sort Python imports (notebooks; auto-fixes) | ||
args: [--nbqa-mutate] | ||
additional_dependencies: [isort==5.8.0] | ||
- id: nbqa-black | ||
name: nbqa-black - consistent Python code formatting (notebooks; auto-fixes) | ||
args: [--nbqa-mutate] | ||
additional_dependencies: [black==21.5b2] | ||
# TODO: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook | ||
#- id: nbqa-flake8 | ||
# name: nbqa-flake8 - Python linting (notebooks) | ||
# additional_dependencies: [ flake8==3.9.2 ] | ||
- repo: https://github.com/Yelp/detect-secrets | ||
rev: v1.0.3 | ||
hooks: | ||
- id: detect-secrets | ||
name: detect-secrets - Detect secrets in staged code | ||
args: ["--baseline", ".secrets.baseline"] | ||
exclude: .*/tests/.*|^\.cruft\.json$ | ||
- repo: local | ||
hooks: | ||
- id: restricted-filenames | ||
name: Check commits for restricted file extensions | ||
entry: These file extensions are restricted. Data should be removed from the commit | ||
language: fail | ||
files: .*\.(csv|feather|xlsx|zip|hdf5|h5|json|xml|hd|parquet) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
#!groovy | ||
|
||
// Global scope required for multi-stage persistence | ||
def artifactoryStr = 'art-p-01' | ||
artServer = Artifactory.server "${artifactoryStr}" | ||
buildInfo = Artifactory.newBuildInfo() | ||
def agentPython3Version = 'python_3.6.1' | ||
def artifactVersion | ||
|
||
// Define a function to push packaged code to Artifactory | ||
def pushToPyPiArtifactoryRepo_temp(String projectName, String version, String sourceDistLocation = 'python/dist/*', String artifactoryHost = 'art-p-01') { | ||
withCredentials([usernamePassword(credentialsId: env.ARTIFACTORY_CREDS, usernameVariable: 'ARTIFACTORY_USER', passwordVariable: 'ARTIFACTORY_PASSWORD')]){ | ||
sh "curl -u ${ARTIFACTORY_USER}:\${ARTIFACTORY_PASSWORD} -T ${sourceDistLocation} 'http://${artifactoryHost}/artifactory/${env.ARTIFACTORY_PYPI_REPO}/${projectName}/'" | ||
} | ||
} | ||
|
||
// This section defines the Jenkins pipeline | ||
pipeline { | ||
libraries { | ||
lib('jenkins-pipeline-shared@feature/dap-ci-scripts') | ||
} | ||
|
||
environment { | ||
ARTIFACTORY_CREDS = 's_jenkins_epds' | ||
ARTIFACTORY_PYPI_REPO = 'yr-python' | ||
PROJECT_NAME = 'resdev' | ||
BUILD_BRANCH = '142_jenkinsFile_RAP' // Any commits to this branch will create a build in artifactory | ||
BUILD_TAG = 'v*' // Any commits tagged with this pattern will create a build in artifactory | ||
MIN_COVERAGE_PC = '0' | ||
GITLAB_CREDS = 'epds_gitlab_token' // Credentials used for notifying GitLab of build status | ||
} | ||
|
||
options { | ||
skipDefaultCheckout true | ||
} | ||
|
||
agent any | ||
|
||
stages { | ||
stage('Checkout') { | ||
agent { label 'download.jenkins.slave' } | ||
steps { | ||
onStage() | ||
colourText('info', "Checking out code from source control.") | ||
|
||
checkout scm | ||
|
||
script { | ||
buildInfo.name = "${PROJECT_NAME}" | ||
buildInfo.number = "${BUILD_NUMBER}" | ||
buildInfo.env.collect() | ||
} | ||
colourText('info', "BuildInfo: ${buildInfo.name}-${buildInfo.number}") | ||
stash name: 'Checkout', useDefaultExcludes: false | ||
} | ||
} | ||
|
||
stage('Preparing virtual environment') { | ||
agent { label "test.${agentPython3Version}" } | ||
steps { | ||
onStage() | ||
colourText('info', "Create venv and install dependencies") | ||
unstash name: 'Checkout' | ||
|
||
sh ''' | ||
PATH=$WORKSPACE/venv/bin:/usr/local/bin:$PATH | ||
python3 -m pip install -U pip | ||
pip3 install virtualenv | ||
if [ ! -d "venv" ]; then | ||
virtualenv venv | ||
fi | ||
. venv/bin/activate | ||
python -m pip install -U pip | ||
pip3 install pypandoc==1.7.5 | ||
# Remove pydoop from requirements before it's installed. | ||
awk '!/pydoop.*/' requirements.txt > temp && mv temp requirements.txt | ||
pip3 install -r requirements.txt | ||
pip3 install pyspark==2.4.0 | ||
pip3 freeze | ||
''' | ||
stash name: 'venv', useDefaultExcludes: false | ||
} | ||
} | ||
|
||
|
||
stage('Unit Test and coverage') { | ||
agent { label "test.${agentPython3Version}" } | ||
steps { | ||
onStage() | ||
colourText('info', "Running unit tests and code coverage.") | ||
unstash name: 'Checkout' | ||
unstash name: 'venv' | ||
|
||
// Compatibility for PyArrow with Spark 2.4-legacy IPC format. | ||
sh 'export ARROW_PRE_0_15_IPC_FORMAT=1' | ||
|
||
// Running coverage first runs the tests | ||
sh ''' | ||
. venv/bin/activate | ||
coverage run --branch --source=./${PROJECT_NAME} --omit=src/utils/hdfs_mods.py,src/utils/wrappers.py,src/utils/runlog.py,src/_version.py,src/pipeline.py \ | ||
-m pytest -ra ./tests --ignore=tests/test_utils/test_hdfs_mods.py | ||
''' | ||
/* | ||
// Lines below create a coverage report for on Jenkins. Currently commented out | ||
// as it gives errors when no imports are used in unit tests. import src.main | ||
// causes pre-commit to complain. Easier to leave out for now. | ||
coverage xml -o python_coverage.xml && coverage report -m --fail-under=${MIN_COVERAGE_PC} | ||
cobertura autoUpdateHealth: false, | ||
autoUpdateStability: false, | ||
coberturaReportFile: 'python_coverage.xml', | ||
conditionalCoverageTargets: '70, 0, 0', | ||
failUnhealthy: false, | ||
failUnstable: false, | ||
lineCoverageTargets: '80, 0, 0', | ||
maxNumberOfBuilds: 0, | ||
methodCoverageTargets: '80, 0, 0', | ||
onlyStable: false, | ||
zoomCoverageChart: false */ | ||
} | ||
} | ||
|
||
stage('Build and publish Python Package') { | ||
when { | ||
anyOf{ | ||
branch BUILD_BRANCH | ||
//tag BUILD_TAG | ||
} | ||
beforeAgent true | ||
} | ||
agent { label "test.${agentPython3Version}" } | ||
steps { | ||
onStage() | ||
colourText('info', "Building Python package.") | ||
unstash name: 'Checkout' | ||
unstash name: 'venv' | ||
|
||
sh ''' | ||
. venv/bin/activate | ||
pip3 install wheel==0.29.0 | ||
python3 setup.py build bdist_wheel | ||
''' | ||
|
||
script { | ||
pushToPyPiArtifactoryRepo_temp("${buildInfo.name}", "", "dist/*") | ||
} | ||
} | ||
} | ||
|
||
|
||
|
||
} | ||
|
||
} |
Oops, something went wrong.