Skip to content

Commit

Permalink
Merge pull request #33 from ONSdigital/develop
Browse files Browse the repository at this point in the history
Create release for 0.2.0-alpha
  • Loading branch information
jwestw authored Jun 22, 2023
2 parents e717ec3 + a5f2d13 commit 06d8a72
Show file tree
Hide file tree
Showing 40 changed files with 2,629 additions and 324 deletions.
28 changes: 17 additions & 11 deletions .github/workflows/pytest-action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,30 @@ jobs:

# 1) Checkout the code
- uses: actions/checkout@v3
# 2) Removing PyDoop from the environment yaml
- name: Remove pydoop dependency
# 2) Removing PyDoop from the requirements.txt
- name: Remove pydoop requirements.txt
shell: bash -l {0}
run: |
awk '!/pydoop.*/' environment.yml > temp && mv temp environment.yml
# 3) Use Setup Miniconda github action to setup environment
- uses: conda-incubator/setup-miniconda@v2
awk '!/pydoop.*/' requirements.txt> temp && mv temp requirements.txt
# 3) Set up Python
- name: Set up Python 3.6
uses: actions/setup-python@v4
with:
python-version: 3.6
environment-file: environment.yml
activate-environment: resdev36
# 3) Run pytest to run all tests in the tests folder
python-version: '3.6'

# 4) Install dependencies from requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt
# 5) Run pytest to run all tests in the tests folder!
- name: Use coverage to run pytest
# Specify shell to run the command in
working-directory: ${{ github.workspace }}
shell: bash -l {0}
run: |
coverage run --branch --source=./src -m pytest -ra ./tests --junitxml=junit_result.xml && coverage xml -o python_coverage.xml && coverage report -m --fail-under=10
# 4) Get the coverage report in to the pull request comments
coverage run --branch --source=./src --omit=src/utils/hdfs_mods.py,src/utils/wrappers.py,src/utils/runlog.py,src/_version.py,src/pipeline.py \
-m pytest -ra ./tests --junitxml=junit_result.xml --ignore=tests/test_utils/test_hdfs_mods.py && coverage xml \
-o python_coverage.xml && coverage report -m --fail-under=10
# 6) Get the coverage report in to the pull request comments
- name: Pytest coverage comment
uses: MishaKav/pytest-coverage-comment@main
with:
Expand Down
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ doc/_build/
target/

# Jupyter Notebook
notebooks/*
# notebooks/*
!notebooks/.gitkeep
!notebooks/.README.md
.ipynb_checkpoints
Expand Down Expand Up @@ -883,6 +883,7 @@ data/processed/*
*.json
*.parquet
*.txt
!requirements.txt
*.xlsx
*.xml
*.zip
Expand All @@ -904,3 +905,6 @@ docs/_linkcheck/
logs/*
!logs/.gitkeep
!logs/logs.md

# Utility bat file for ssh key
add_ssh_key.bat
132 changes: 66 additions & 66 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,69 +1,69 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/kynan/nbstripout
rev: 0.4.0
hooks:
- id: nbstripout
name: nbstripout - Strip outputs from notebooks (auto-fixes)
args:
- --extra-keys
- "metadata.colab metadata.kernelspec cell.metadata.colab cell.metadata.executionInfo cell.metadata.id cell.metadata.outputId"
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
- id: check-added-large-files
name: Check for files larger than 5 MB
args: [ "--maxkb=5120" ]
- id: end-of-file-fixer
name: Check for a blank line at the end of scripts (auto-fixes)
exclude: '\.Rd'
- id: trailing-whitespace
name: Check for trailing whitespaces (auto-fixes)
- repo: https://github.com/pycqa/isort
rev: 5.8.0
hooks:
- id: isort
name: isort - Sort Python imports (auto-fixes)
types: [ cython, pyi, python ]
args: [ "--profile", "black", "--filter-files" ]
- repo: https://github.com/psf/black
rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags
hooks:
- id: black
name: black - consistent Python code formatting (auto-fixes)
language_version: python # Should be a command that runs python3.6+
- repo: https://github.com/PyCQA/flake8
rev: 5.0.4
hooks:
- id: flake8
name: flake8 - Python linting
- repo: https://github.com/nbQA-dev/nbQA
rev: 0.12.0
hooks:
- id: nbqa-isort
name: nbqa-isort - Sort Python imports (notebooks; auto-fixes)
args: [ --nbqa-mutate ]
additional_dependencies: [ isort==5.8.0 ]
- id: nbqa-black
name: nbqa-black - consistent Python code formatting (notebooks; auto-fixes)
args: [ --nbqa-mutate ]
additional_dependencies: [ black==21.5b2 ]
# TODO: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook
#- id: nbqa-flake8
# name: nbqa-flake8 - Python linting (notebooks)
# additional_dependencies: [ flake8==3.9.2 ]
- repo: https://github.com/Yelp/detect-secrets
rev: v1.0.3
hooks:
- id: detect-secrets
name: detect-secrets - Detect secrets in staged code
args: [ "--baseline", ".secrets.baseline" ]
exclude: .*/tests/.*|^\.cruft\.json$
- repo: local
hooks:
- id: restricted-filenames
name: Check commits for restricted file extensions
entry: These file extensions are restricted. Data should be removed from the commit
language: fail
files: .*\.(csv|feather|xlsx|zip|hdf5|h5|txt|json|xml|hd|parquet)
- repo: https://github.com/kynan/nbstripout
rev: 0.4.0
hooks:
- id: nbstripout
name: nbstripout - Strip outputs from notebooks (auto-fixes)
args:
- --extra-keys
- "metadata.colab metadata.kernelspec cell.metadata.colab cell.metadata.executionInfo cell.metadata.id cell.metadata.outputId"
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
- id: check-added-large-files
name: Check for files larger than 5 MB
args: ["--maxkb=5120"]
- id: end-of-file-fixer
name: Check for a blank line at the end of scripts (auto-fixes)
exclude: '\.Rd'
- id: trailing-whitespace
name: Check for trailing whitespaces (auto-fixes)
- repo: https://github.com/pycqa/isort
rev: 5.8.0
hooks:
- id: isort
name: isort - Sort Python imports (auto-fixes)
types: [cython, pyi, python]
args: ["--profile", "black", "--filter-files"]
- repo: https://github.com/psf/black
rev: 22.8.0 # Replace by any tag/version: https://github.com/psf/black/tags
hooks:
- id: black
name: black - consistent Python code formatting (auto-fixes)
language_version: python # Should be a command that runs python3.6+
- repo: https://github.com/PyCQA/flake8
rev: 5.0.4
hooks:
- id: flake8
name: flake8 - Python linting
- repo: https://github.com/nbQA-dev/nbQA
rev: 0.12.0
hooks:
- id: nbqa-isort
name: nbqa-isort - Sort Python imports (notebooks; auto-fixes)
args: [--nbqa-mutate]
additional_dependencies: [isort==5.8.0]
- id: nbqa-black
name: nbqa-black - consistent Python code formatting (notebooks; auto-fixes)
args: [--nbqa-mutate]
additional_dependencies: [black==21.5b2]
# TODO: Disabled for now until it's clear how to add noqa to specific cells of a Jupyter notebook
#- id: nbqa-flake8
# name: nbqa-flake8 - Python linting (notebooks)
# additional_dependencies: [ flake8==3.9.2 ]
- repo: https://github.com/Yelp/detect-secrets
rev: v1.0.3
hooks:
- id: detect-secrets
name: detect-secrets - Detect secrets in staged code
args: ["--baseline", ".secrets.baseline"]
exclude: .*/tests/.*|^\.cruft\.json$
- repo: local
hooks:
- id: restricted-filenames
name: Check commits for restricted file extensions
entry: These file extensions are restricted. Data should be removed from the commit
language: fail
files: .*\.(csv|feather|xlsx|zip|hdf5|h5|json|xml|hd|parquet)
163 changes: 163 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
#!groovy

// Global scope required for multi-stage persistence
def artifactoryStr = 'art-p-01'
artServer = Artifactory.server "${artifactoryStr}"
buildInfo = Artifactory.newBuildInfo()
def agentPython3Version = 'python_3.6.1'
def artifactVersion

// Define a function to push packaged code to Artifactory
def pushToPyPiArtifactoryRepo_temp(String projectName, String version, String sourceDistLocation = 'python/dist/*', String artifactoryHost = 'art-p-01') {
withCredentials([usernamePassword(credentialsId: env.ARTIFACTORY_CREDS, usernameVariable: 'ARTIFACTORY_USER', passwordVariable: 'ARTIFACTORY_PASSWORD')]){
sh "curl -u ${ARTIFACTORY_USER}:\${ARTIFACTORY_PASSWORD} -T ${sourceDistLocation} 'http://${artifactoryHost}/artifactory/${env.ARTIFACTORY_PYPI_REPO}/${projectName}/'"
}
}

// This section defines the Jenkins pipeline
pipeline {
libraries {
lib('jenkins-pipeline-shared@feature/dap-ci-scripts')
}

environment {
ARTIFACTORY_CREDS = 's_jenkins_epds'
ARTIFACTORY_PYPI_REPO = 'yr-python'
PROJECT_NAME = 'resdev'
BUILD_BRANCH = '142_jenkinsFile_RAP' // Any commits to this branch will create a build in artifactory
BUILD_TAG = 'v*' // Any commits tagged with this pattern will create a build in artifactory
MIN_COVERAGE_PC = '0'
GITLAB_CREDS = 'epds_gitlab_token' // Credentials used for notifying GitLab of build status
}

options {
skipDefaultCheckout true
}

agent any

stages {
stage('Checkout') {
agent { label 'download.jenkins.slave' }
steps {
onStage()
colourText('info', "Checking out code from source control.")

checkout scm

script {
buildInfo.name = "${PROJECT_NAME}"
buildInfo.number = "${BUILD_NUMBER}"
buildInfo.env.collect()
}
colourText('info', "BuildInfo: ${buildInfo.name}-${buildInfo.number}")
stash name: 'Checkout', useDefaultExcludes: false
}
}

stage('Preparing virtual environment') {
agent { label "test.${agentPython3Version}" }
steps {
onStage()
colourText('info', "Create venv and install dependencies")
unstash name: 'Checkout'

sh '''
PATH=$WORKSPACE/venv/bin:/usr/local/bin:$PATH
python3 -m pip install -U pip
pip3 install virtualenv
if [ ! -d "venv" ]; then
virtualenv venv
fi
. venv/bin/activate
python -m pip install -U pip
pip3 install pypandoc==1.7.5
# Remove pydoop from requirements before it's installed.
awk '!/pydoop.*/' requirements.txt > temp && mv temp requirements.txt
pip3 install -r requirements.txt
pip3 install pyspark==2.4.0
pip3 freeze
'''
stash name: 'venv', useDefaultExcludes: false
}
}


stage('Unit Test and coverage') {
agent { label "test.${agentPython3Version}" }
steps {
onStage()
colourText('info', "Running unit tests and code coverage.")
unstash name: 'Checkout'
unstash name: 'venv'

// Compatibility for PyArrow with Spark 2.4-legacy IPC format.
sh 'export ARROW_PRE_0_15_IPC_FORMAT=1'

// Running coverage first runs the tests
sh '''
. venv/bin/activate
coverage run --branch --source=./${PROJECT_NAME} --omit=src/utils/hdfs_mods.py,src/utils/wrappers.py,src/utils/runlog.py,src/_version.py,src/pipeline.py \
-m pytest -ra ./tests --ignore=tests/test_utils/test_hdfs_mods.py
'''
/*
// Lines below create a coverage report for on Jenkins. Currently commented out
// as it gives errors when no imports are used in unit tests. import src.main
// causes pre-commit to complain. Easier to leave out for now.
coverage xml -o python_coverage.xml && coverage report -m --fail-under=${MIN_COVERAGE_PC}
cobertura autoUpdateHealth: false,
autoUpdateStability: false,
coberturaReportFile: 'python_coverage.xml',
conditionalCoverageTargets: '70, 0, 0',
failUnhealthy: false,
failUnstable: false,
lineCoverageTargets: '80, 0, 0',
maxNumberOfBuilds: 0,
methodCoverageTargets: '80, 0, 0',
onlyStable: false,
zoomCoverageChart: false */
}
}

stage('Build and publish Python Package') {
when {
anyOf{
branch BUILD_BRANCH
//tag BUILD_TAG
}
beforeAgent true
}
agent { label "test.${agentPython3Version}" }
steps {
onStage()
colourText('info', "Building Python package.")
unstash name: 'Checkout'
unstash name: 'venv'

sh '''
. venv/bin/activate
pip3 install wheel==0.29.0
python3 setup.py build bdist_wheel
'''

script {
pushToPyPiArtifactoryRepo_temp("${buildInfo.name}", "", "dist/*")
}
}
}



}

}
Loading

0 comments on commit 06d8a72

Please sign in to comment.