BUILD: release FACET 1.2.0rc1 (#298)

* BUILD: pin Jinja2 version to <3.0 to prevent breaking the sphinx build (#286)

* BUILD: pin Jinja2 version to <3.0 to prevent breaking the sphinx build

* BUILD: update package requirements

* DEV: update conda development environment in environment.yml

* Update version number to 1.2.0

* Update version to 1.0.4

* DEV: pin scipy to 1.5 in environment.yml (#287)

* API: remove FACET 1.0 legacy inspections (#288)

* BUILD: pin jinja2 to ~=2.11 to prevent incompatibility with sphinx (#289)

* BUILD: standardize azure pipeline across BCG Gamma repos (#292)

* BUILD: create individual build tasks per package in single/matrix builds

* DOC: remove comments

* BUILD: use stable versions of pytools and sklearndf in matrix builds

* FIX: revert path to /dist in CopyFiles task

* BUILD: add more packages to conda build recipe

* BUILD: use python 3.8 in conda host environment

* BUILD: update build requirements incl. increased min requirements detail

* BUILD: remove dev/* trigger

* BUILD: downgrade typing_inspect to 0.6 in max build

* BUILD: change numpy requirements syntax; improve upper version bounds

* BUILD: add ipython to min/max matrix test dependencies

* BUILD: use pip version syntax in pyproject.toml

* BUILD: remove indirect package dependencies from default dependencies

* BUILD: update pyproject.toml

* BUILD: relax max build requirement for scipy

* BUILD: use global variables for project and package name

* BUILD: update standardized azure-pipelines.yml

* FIX: re-remove _balancer.py

* BUILD: fix pytools minimum versions in pyproject.toml

* FIX: joblib version in min matrix build

* BUILD: disable nightly builds on 1.1.x branch

* BUILD: tweaks for 1.2.0 release (#295)

* TEST: fix subsample to remove version dependence on random generator

* FIX: de-duplicate sample index before intersecting with test splits

* API: remove deprecated mirror package facet.simulation.partition

* FIX: update nbsphinx to ~=0.8.5 to address new issue with jinja 3.0.0

* BUILD: fix max matrix versions at maximum

* FIX: correct shap version spec in pyproject.toml

* DOC: update RELEASE_NOTES.rst

* BUILD: require pytools ~=1.2 and sklearndf ~= 1.2

* BUILD: require numpy 1.17

* BUILD: remove dev/* from azure pipeline triggers

* FIX: rephrase max numpy requirement due to possible bug in conda-build

* FIX: build sklearndf 1.2.x in azure-pipelines.yml

* BUILD: use python 3.8 in host requirements

* BUILD: remove unneeded trailing .* from pyproject.toml

* BUILD: add missing version requirements to conda run config

* BUILD: set matplotlib to 3.4 in max test

* BUILD: change matplotlib==3 back to matplotlib==3.* for tox

* BUILD: add inherited requirements so we can use them for testing

* FIX: change boruta into boruta_py

* FIX: change boruta_py back into boruta for tox builds (but not conda)

* BUILD: make min requirements more specific to speed up build time

* BUILD: remove conda dependencies with boruta

Co-authored-by: Jörg Schneider <[email protected]>

* BUILD: relax upper bounds of package requirements (#296)

* BUILD: relax upper bounds of package requirements

* BUILD: replace matplotlib with matplotlib-base

* BUILD: use matplotlib-base only for conda

* API: improve parameter checking for StratifiedBootstrapCV

* TEST: add unit test for StratifiedBootstrapCV

* FIX: don't require arg y in non-stratified bootstraps

* DEV: update package requirements for facet-develop conda environment

* API: implement full sample simulation as experimental feature (#297)

* API: add hidden option to simulate single models on the full sample

* DOC: fix an outdated (but hidden) docstring

* TEST: validate partition frequencies

* FIX: correct sort order of imports

* Update __init__.py

* FIX: use lists not tuples to create simulation results data frame

Co-authored-by: Jörg Schneider <[email protected]>

RELEASE_NOTES.rst

@@ -4,14 +4,16 @@ Release Notes
 FACET 1.2
 ---------
 
-FACET 1.2 introduces the ability to run simulations on a subsample of the data used to
+FACET 1.2 adds support for *sklearndf* 1.2 and *scikit-learn* 0.24.
+It also introduces the ability to run simulations on a subsample of the data used to
 fit the underlying crossfit.
 One example where this can be useful is to use only a recent period of a time series as
 the baseline of a simulation.
 
 1.2.0
 ~~~~~
 
+- BUILD: added support for *sklearndf* 1.2 and *scikit-learn* 0.24
 - API: new optional parameter `subsample` in method 
   :meth:`.BaseUnivariateSimulator.simulate_feature` can be used to specify a subsample
   to be used in the simulation (but simulating using a crossfit based on the full

azure-pipelines.yml

@@ -1,11 +1,9 @@
 trigger:
   - 1.2.x
-  - dev/*
   - release/*
 
 pr:
   - 1.2.x
-  - dev/*
   - release/*
 
 # set the build name
@@ -21,11 +19,6 @@ schedules:
 
 resources:
   repositories:
-    - repository: sklearndf
-      type: github
-      endpoint: BCG-Gamma
-      name: BCG-Gamma/sklearndf
-      ref: 1.1.x
     - repository: pytools
       type: github
       endpoint: BCG-Gamma
@@ -40,6 +33,10 @@ variables:
   source_is_release_branch: $[ startsWith(variables['branchName'], 'release') ]
   source_is_develop_branch: $[ or(startsWith(variables['branchName'], 'develop'), startsWith(variables['branchName'], 'dev/')) ]
   is_scheduled: $[ eq(variables['Build.Reason'], 'Schedule') ]
+  project_name: facet
+  project_root: $(project_name)
+  package_name: gamma-facet
+  sshPublicKey: $(sshPublicKey_facet)
 
 stages:
   # check code quality first to fail fast (isort, flake8, black)
@@ -96,6 +93,7 @@ stages:
             inputs:
               targetType: 'inline'
               script: |
+                echo Repo: $(Build.DefinitionName)
                 cd $(System.DefaultWorkingDirectory)
                 files_changed=$(git diff $(Build.SourceVersion)^! --name-only)
                 echo "Files changed since last commit: ${files_changed}"
@@ -132,8 +130,6 @@ stages:
           displayName: 'use Python 3.7'
 
         - checkout: self
-        - checkout: sklearndf
-        - checkout: pytools
 
         - script: dir $(Build.SourcesDirectory)
 
@@ -143,21 +139,12 @@ stages:
             script: |
               set -eux
               eval "$(conda shell.bash hook)"
-              cd $(System.DefaultWorkingDirectory)/facet/
-              export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/
-              export RUN_PACKAGE_VERSION_TEST=facet
-              ENV_YML=$(Agent.TempDirectory)/environment-stripped.yml
-              grep -vE 'gamma-pytools|sklearndf' environment.yml > "$ENV_YML"
-              conda env create -f "$ENV_YML"
-              conda activate facet-develop
-              pip install flit
-              cd $(System.DefaultWorkingDirectory)/pytools/
-              flit install -s
-              cd $(System.DefaultWorkingDirectory)/sklearndf/
-              flit install -s
-              cd $(System.DefaultWorkingDirectory)/facet/
+              export PYTHONPATH=$(System.DefaultWorkingDirectory)/src/
+              export RUN_PACKAGE_VERSION_TEST=$(project_name)
+              conda env create
+              conda activate $(project_name)-develop
               pytest \
-                 --cov facet \
+                 --cov $(project_name) \
                  --cov-config "tox.ini" \
                  --cov-report=xml:coverage.xml --cov-report=html:htmlcov \
                  --junitxml pytest.xml \
@@ -167,15 +154,15 @@ stages:
         - task: PublishTestResults@2
           condition: succeededOrFailed()
           inputs:
-            testResultsFiles: '$(System.DefaultWorkingDirectory)/facet/*.xml'
-            searchFolder: '$(System.DefaultWorkingDirectory)/facet/'
+            testResultsFiles: '$(System.DefaultWorkingDirectory)/*.xml'
+            searchFolder: '$(System.DefaultWorkingDirectory)/'
             testRunTitle: 'Publish test results'
 
         - task: PublishCodeCoverageResults@1
           inputs:
             codeCoverageTool: Cobertura
-            summaryFileLocation: '$(System.DefaultWorkingDirectory)/facet/coverage.xml'
-            reportDirectory: '$(System.DefaultWorkingDirectory)/facet/htmlcov'
+            summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+            reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
 
   # conda env & tox build test
   # testing matrix of python & sklearn versions
@@ -217,14 +204,13 @@ stages:
             versionSpec: '$(FACET_V_PYTHON_BUILD)'
           displayName: 'Use Python $(FACET_V_PYTHON_BUILD)'
 
-        - checkout: self
-        - checkout: sklearndf
         - checkout: pytools
+        - checkout: self
 
         - script: dir $(Build.SourcesDirectory)
 
         - script: |
-            conda install -y -c anaconda conda-build=3.20.5 conda-verify toml=0.10.* flit=3.0.*
+            conda install -y -c anaconda conda-build~=3.20.5 conda-verify toml=0.10.* flit=3.0.*
           displayName: 'Install conda-build, flit, toml'
           condition: eq(variables['BUILD_SYSTEM'], 'conda')
 
@@ -241,21 +227,17 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
+              set -eux
               if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
-              export RUN_PACKAGE_VERSION_TEST=facet
-              cd $(Build.SourcesDirectory)/pytools
-              ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
+              export RUN_PACKAGE_VERSION_TEST=$(project_name)
 
-              cd $(Build.SourcesDirectory)/sklearndf
-              ./make.py sklearndf $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
-
-              cd $(Build.SourcesDirectory)/facet
-              ./make.py facet $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
-          displayName: "build & test conda/pip packages"
+              cd $(Build.SourcesDirectory)/$(project_root)
+              ./make.py $(project_name) $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
+          displayName: "build & test"
 
         - task: CopyFiles@2
           inputs:
-            sourceFolder: $(System.DefaultWorkingDirectory)/facet/dist
+            sourceFolder: $(System.DefaultWorkingDirectory)/$(project_root)/dist
             targetFolder: $(Build.ArtifactStagingDirectory)
 
         - task: PublishBuildArtifacts@1
@@ -309,14 +291,13 @@ stages:
             versionSpec: '$(FACET_V_PYTHON_BUILD)'
           displayName: 'Use Python $(FACET_V_PYTHON_BUILD)'
 
-        - checkout: self
-        - checkout: sklearndf
         - checkout: pytools
+        - checkout: self
 
         - script: dir $(Build.SourcesDirectory)
 
         - script: |
-            conda install -y -c anaconda conda-build=3.20.5 conda-verify toml=0.10.* flit=3.0.*
+            conda install -y -c anaconda conda-build~=3.20.5 conda-verify toml=0.10.* flit=3.0.*
           displayName: 'Install conda-build, flit, toml'
           condition: eq(variables['BUILD_SYSTEM'], 'conda')
 
@@ -335,18 +316,15 @@ stages:
             script: |
               set -eux
               if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
-              export RUN_PACKAGE_VERSION_TEST=facet
-              cd $(Build.SourcesDirectory)/pytools
-              ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
-              cd $(Build.SourcesDirectory)/sklearndf
-              ./make.py sklearndf $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
-              cd $(Build.SourcesDirectory)/facet
-              ./make.py facet $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
-          displayName: "build & test conda/pip packages"
+              export RUN_PACKAGE_VERSION_TEST=$(project_name)
+
+              cd $(Build.SourcesDirectory)/$(project_root)
+              ./make.py $(project_name) $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
+          displayName: "build & test"
 
         - task: CopyFiles@2
           inputs:
-            sourceFolder: $(System.DefaultWorkingDirectory)/facet/dist
+            sourceFolder: $(System.DefaultWorkingDirectory)/$(project_root)/dist
             targetFolder: $(Build.ArtifactStagingDirectory)
 
         - task: PublishBuildArtifacts@1
@@ -382,7 +360,7 @@ stages:
               veracodeAppProfile: 'FACET'
               version: '$(Build.BuildID)'
               filepath: '$(System.DefaultWorkingDirectory)/static_scan'
-              sandboxName: 'facet'
+              sandboxName: '$(project_name)'
               createSandBox: false
               createProfile: false
               failTheBuildIfVeracodeScanDidNotInitiate: false
@@ -409,8 +387,8 @@ stages:
               versionSpec: '3.7.*'
             displayName: 'use Python 3.7'
 
-          - checkout: self
           - checkout: pytools
+          - checkout: self
 
           - task: Bash@3
             env:
@@ -436,7 +414,7 @@ stages:
                 ), "This check should only run on versioned branches – check pipeline."
 
                 branch_version = branch.split("/", maxsplit=1)[1]
-                package_version = ToxBuilder("facet", "default").package_version
+                package_version = ToxBuilder("$(project_name)", "default").package_version
 
                 assert (
                     package_version == branch_version
@@ -474,9 +452,8 @@ stages:
               versionSpec: '3.7.*'
             displayName: 'use Python 3.7'
 
-          - checkout: self
-
           - checkout: pytools
+          - checkout: self
 
           - task: Bash@3
             inputs:
@@ -485,7 +462,7 @@ stages:
                 set -eux
                 echo "Getting version"
                 pip install packaging
-                cd $(System.DefaultWorkingDirectory)/facet/src
+                cd $(System.DefaultWorkingDirectory)/$(project_root)/src
                 export PYTHONPATH=$(System.DefaultWorkingDirectory)/pytools/sphinx/base
                 version=$(python -c "import make_base; print(make_base.get_package_version())")
                 echo "Current version: $version"
@@ -510,7 +487,7 @@ stages:
               eval "$(conda shell.bash hook)"
               conda install -y anaconda-client
               anaconda login --username "${CONDA_USERNAME}" --password "${CONDA_PASSWORD}"
-              anaconda upload --user BCG_Gamma --force $(System.ArtifactsDirectory)/conda_default/conda/noarch/gamma-facet-*.tar.bz2
+              anaconda upload --user BCG_Gamma --force $(System.ArtifactsDirectory)/conda_default/conda/noarch/$(package_name)-*.tar.bz2
               anaconda logout
               echo "##vso[task.setvariable variable=conda_published]True"
             displayName: 'Publish to Anaconda'
@@ -522,7 +499,7 @@ stages:
 
           - script: |
               set -eux
-              cd $(System.DefaultWorkingDirectory)/facet
+              cd $(System.DefaultWorkingDirectory)/$(project_root)
               pip install flit
               flit install -s
               flit publish
@@ -550,24 +527,24 @@ stages:
               target: $(Build.SourceVersion)
               tagSource: userSpecifiedTag
               tag: $(current_version)
-              title: facet $(current_version)
+              title: $(project_name) $(current_version)
               releaseNotesSource: inline
               releaseNotesInline: |
-                This is the $(current_version) release of facet.
+                This is the $(current_version) release of $(package_name).
 
                 You can upgrade your current pip installation via
 
-                    pip install --upgrade gamma-facet
+                    pip install --upgrade $(package_name)
 
                 Your conda package can be upgraded by running
 
-                    conda install -c conda-forge -c bcg_gamma gamma-facet
+                    conda install -c conda-forge -c bcg_gamma $(package_name)
 
               isDraft: false
               isPreRelease: $(is_prerelease)
               assets: |
-                $(System.ArtifactsDirectory)/tox_default/tox/gamma-facet-*.tar.gz
-                $(System.ArtifactsDirectory)/conda_default/conda/noarch/gamma-facet-*.tar.bz2
+                $(System.ArtifactsDirectory)/tox_default/tox/$(package_name)-*.tar.gz
+                $(System.ArtifactsDirectory)/conda_default/conda/noarch/$(package_name)-*.tar.bz2
 
   # render docs and publish to GitHub Pages
   - stage:
@@ -598,13 +575,12 @@ stages:
         - task: InstallSSHKey@0
           inputs:
             knownHostsEntry: $(knownHostsEntry)
-            sshPublicKey: $(sshPublicKey_facet)
-            sshKeySecureFile: 'deploy_docs_facet'
+            sshPublicKey: $(sshPublicKey)
+            sshKeySecureFile: 'deploy_docs_$(project_name)'
           displayName: 'Install the deploy SSH key'
 
-        - checkout: self
-        - checkout: sklearndf
         - checkout: pytools
+        - checkout: self
 
         - script: dir $(Build.SourcesDirectory)
 
@@ -613,7 +589,7 @@ stages:
             targetType: 'inline'
             script: |
               set -eux
-              cd $(System.DefaultWorkingDirectory)/facet
+              cd $(System.DefaultWorkingDirectory)/$(project_root)
               git checkout --track origin/github-pages
               mkdir -p docs
               sudo apt-get install tree
@@ -629,19 +605,14 @@ stages:
             script: |
               set -eux
               eval "$(conda shell.bash hook)"
-              cd $(System.DefaultWorkingDirectory)/facet
+              cd $(System.DefaultWorkingDirectory)/$(project_root)
               echo "Checking out $(branchName)"
               git checkout $(branchName)
               git status
-              export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/
+              export PYTHONPATH=$(System.DefaultWorkingDirectory)/$(project_root)/src/
               conda env create -f environment.yml
-              conda activate facet-develop
-              pip install flit
-              cd $(System.DefaultWorkingDirectory)/pytools/
-              flit install -s
-              cd $(System.DefaultWorkingDirectory)/sklearndf/
-              flit install -s
-              cd $(System.DefaultWorkingDirectory)/facet/
+              conda activate $(project_name)-develop
+              cd $(System.DefaultWorkingDirectory)/$(project_root)/
               python sphinx/make.py html
               echo "Current docs contents:"
               tree docs
@@ -656,11 +627,11 @@ stages:
               cp -r $(Build.ArtifactStagingDirectory)/old_docs/docs .
               echo "Current docs contents:"
               tree docs
-              mkdir -p $(System.DefaultWorkingDirectory)/facet/sphinx/build/
-              cp -R docs/docs-version $(System.DefaultWorkingDirectory)/facet/sphinx/build/
+              mkdir -p $(System.DefaultWorkingDirectory)/$(project_root)/sphinx/build/
+              cp -R docs/docs-version $(System.DefaultWorkingDirectory)/$(project_root)/sphinx/build/
               echo "Building sphinx docs"
-              conda activate facet-develop
-              cd $(System.DefaultWorkingDirectory)/facet
+              conda activate $(project_name)-develop
+              cd $(System.DefaultWorkingDirectory)/$(project_root)
               python sphinx/make.py prepare_docs_deployment
               echo "Current docs contents:"
               tree docs
@@ -674,7 +645,7 @@ stages:
             targetType: 'inline'
             script: |
               set -eux
-              cd $(System.DefaultWorkingDirectory)/facet
+              cd $(System.DefaultWorkingDirectory)/$(project_root)
               echo "Adjusting git credentials"
               git config --global user.name "Azure Pipelines"
               git config --global user.email "[email protected]"