From f07fc053a19c990a4d0fb8de42517add46ee22bd Mon Sep 17 00:00:00 2001 From: Erik Schytt Mannerfelt <33550973+erikmannerfelt@users.noreply.github.com> Date: Thu, 27 May 2021 16:11:30 +0200 Subject: [PATCH] Update dependencies and version (#133) * Changed workflows to only use conda-forge and updated readme+environment accordingly. * Set minimum python version. Removed unused pdal dependency. Ran black for formatting. * Added dynamic docs version. Added manual pypi publish trigger. Bumped version. * Added dev-environment.yml * Updated workflow to use new dev-environment.yml * Changed to test step names that makes more sense. * Added missing requirement and fixed test that fails randomly. * Simplified doc building script to not use subprocess. Will work better on Windows. * Attempt to make readthedocs happy nr 1. * Attempt to make readthedocs happy nr 2. * Attempt to make readthedocs happy nr 3. --- .github/workflows/python-package.yml | 39 +++++++++++++------ .github/workflows/python-publish.yml | 7 ++++ .readthedocs.yml | 21 ++++++++++ README.md | 19 +++++---- dev-environment.yml | 29 ++++++++++++++ docs/source/conf.py | 10 +++-- environment.yml | 14 ++----- setup.py | 58 +++++++++++++++++----------- tests/test_coreg.py | 2 +- tests/test_docs.py | 37 ++++-------------- 10 files changed, 148 insertions(+), 88 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 dev-environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7b8b219d..22ed2bc1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -11,30 +11,45 @@ on: jobs: build: + name: ${{ matrix.os }}, python ${{ matrix.python-version}} + runs-on: ${{ matrix.os }} - runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + os: ["ubuntu-latest"] + python-version: ["3.7", "3.8", "3.9"] + + + # Run all shells using bash (including Windows) + defaults: + run: + shell: bash -l {0} steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + + # Set up the conda-forge environment with mamba + - name: Set up Python ${{ matrix.python-version }} and dependencies + uses: conda-incubator/setup-miniconda@v2 with: + auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Install dependencies + mamba-version: "*" + channels: conda-forge + channel-priority: strict + environment-file: dev-environment.yml + activate-environment: xdem-dev + + - name: Install project run: | - $CONDA/bin/conda install -c conda-forge mamba -y - $CONDA/bin/mamba env update --file environment.yml --name base - $CONDA/bin/pip install -r dev-requirements.txt - $CONDA/bin/pip install . + pip install . --no-dependencies + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - $CONDA/bin/flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - $CONDA/bin/flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - $CONDA/bin/pytest -rA + pytest -rA diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index fa6c15da..495fb970 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -7,6 +7,13 @@ on: release: types: [created] + workflow_dispatch: + inputs: + reason: + description: 'Reason for manual trigger' + required: true + default: 'testing' + jobs: deploy: diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..07df20ac --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,21 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + fail_on_warning: false + +# Optionally build your docs in additional formats such as PDF and ePub +formats: [] + +python: + version: 3.7 + install: + - requirements: dev-requirements.txt + - method: pip + path: . diff --git a/README.md b/README.md index 291bc403..0555bf20 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ More documentation to come! [![build](https://github.com/GlacioHack/xdem/actions/workflows/python-package.yml/badge.svg)](https://github.com/GlacioHack/xdem/actions/workflows/python-package.yml) -## Installation ## +## Installation +Recommended: Use conda for depencency solving. ``` $ git clone https://github.com/GlacioHack/xdem.git $ cd ./xdem @@ -16,19 +17,19 @@ $ conda env create -f environment.yml $ conda activate xdem $ pip install . ``` -or -```bash -pip install git+https://github.com/GlacioHack/xdem.git -``` - -To update, please use the `--force-reinstall` flag for `conda` or `pip` to ensure the latest version is installed (`geoutils` and `xdem` do not yet have proper release schedules as of 2021-05-13). - After installing, we recommend to check that everything is working by running the tests: ``` $ pytest -rA ``` +### Installing with pip +**NOTE**: Setting up GDAL and PROJ may need some extra steps, depending on your operating system and configuration. +```bash +pip install xdem +``` + + ## Structure xdem are for now composed of three libraries: @@ -41,6 +42,8 @@ xdem are for now composed of three libraries: You can find ways to improve the libraries in the [issues](https://github.com/GlacioHack/xdem/issues) section. All contributions are welcome. To avoid conflicts, it is suggested to use separate branches for each implementation. All changes must then be submitted to the dev branch using pull requests. Each PR must be reviewed by at least one other person. +Please see our [contribution page](CONTRIBUTING.md) for more detailed instructions. + ### Documentation See the documentation at https://xdem.readthedocs.io diff --git a/dev-environment.yml b/dev-environment.yml new file mode 100644 index 00000000..8f6df652 --- /dev/null +++ b/dev-environment.yml @@ -0,0 +1,29 @@ +name: xdem-dev +channels: + - conda-forge +dependencies: + - python>=3.7 + - proj>=7.2 + - geopandas + - numba + - matplotlib + - opencv + - pyproj + - rasterio + - scipy + - tqdm + - scikit-image + - proj-data + - scikit-gstat + - pytransform3d + - geoutils + + # Development-specific + - pytest + - pytest-xdist + - sphinx + - sphinx_rtd_theme + - numpydoc + - sphinxcontrib-programoutput + - flake8 + - sphinx-autodoc-typehints diff --git a/docs/source/conf.py b/docs/source/conf.py index d42fc0ff..9342866f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,10 +13,10 @@ import os import sys -#import xdem.version - # Allow conf.py to find the xdem module -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../")) +sys.path.insert(0, os.path.abspath("../xdem/'")) +import xdem.version + # -- Project information ----------------------------------------------------- project = 'xdem' @@ -25,7 +25,7 @@ author = 'xdem contributors' # The full version, including alpha/beta/rc tags -release = "0.0.1" +release = xdem.version.version os.environ["PYTHON"] = sys.executable @@ -46,6 +46,8 @@ "sphinxcontrib.programoutput" ] +autosummary_generate = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/environment.yml b/environment.yml index f0b4eba9..fd870b6d 100644 --- a/environment.yml +++ b/environment.yml @@ -2,24 +2,18 @@ name: xdem channels: - conda-forge dependencies: - - python>=3 + - python>=3.7 - proj>=7.2 - geopandas - - ipython - numba - matplotlib - - pdal - opencv - - pytest - pyproj - rasterio - scipy - tqdm - scikit-image - - pip - proj-data - - pip: - - richdem - - scikit-gstat - - pytransform3d - - https://github.com/GlacioHack/GeoUtils/tarball/main + - scikit-gstat + - pytransform3d + - geoutils diff --git a/setup.py b/setup.py index 7f75dbf6..ed579dbb 100644 --- a/setup.py +++ b/setup.py @@ -5,31 +5,44 @@ from setuptools import setup from setuptools.command.install import install -FULLVERSION = '0.0.2-2' +FULLVERSION = "0.0.3" VERSION = FULLVERSION with open(os.path.join(os.path.dirname(__file__), "README.md")) as infile: LONG_DESCRIPTION = infile.read() -setup(name='xdem', - version=FULLVERSION, - description='Set of tools to manipulate Digital Elevation Models (DEMs) ', - long_description=LONG_DESCRIPTION, - long_description_content_type="text/markdown", - url='https://github.com/GlacioHack/xdem', - author='The GlacioHack Team', - author_email="this-is-not-an-email@a.com", # This is needed for PyPi unfortunately. - license='BSD-3', - packages=['xdem'], - install_requires=[ - 'numpy', 'scipy', 'rasterio', 'geopandas', - 'pyproj', 'tqdm', 'scikit-gstat', 'scikit-image', - "geoutils @ https://github.com/GlacioHack/geoutils/tarball/main" - ], - extras_require={'rioxarray': ['rioxarray'], 'richdem': ['richdem'], 'pdal': [ - 'pdal'], 'opencv': ['opencv'], "pytransform3d": ["pytransform3d"]}, - scripts=[], - zip_safe=False) +setup( + name="xdem", + version=FULLVERSION, + description="Set of tools to manipulate Digital Elevation Models (DEMs) ", + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + url="https://github.com/GlacioHack/xdem", + author="The GlacioHack Team", + author_email="this-is-not-an-email@a.com", # This is needed for PyPi unfortunately. + license="BSD-3", + packages=["xdem"], + install_requires=[ + "numpy", + "scipy", + "rasterio", + "geopandas", + "pyproj", + "tqdm", + "scikit-gstat", + "scikit-image", + "geoutils @ https://github.com/GlacioHack/geoutils/tarball/main", + ], + extras_require={ + "rioxarray": ["rioxarray"], + "richdem": ["richdem"], + "opencv": ["opencv"], + "pytransform3d": ["pytransform3d"], + }, + python_requires=">=3.7", + scripts=[], + zip_safe=False, +) write_version = True @@ -40,10 +53,9 @@ def write_version_py(filename=None): short_version = '%s' """ if not filename: - filename = os.path.join(os.path.dirname(__file__), 'xdem', - 'version.py') + filename = os.path.join(os.path.dirname(__file__), "xdem", "version.py") - a = open(filename, 'w') + a = open(filename, "w") try: a.write(cnt % (FULLVERSION, VERSION)) finally: diff --git a/tests/test_coreg.py b/tests/test_coreg.py index 413a60a3..602d6dab 100644 --- a/tests/test_coreg.py +++ b/tests/test_coreg.py @@ -150,7 +150,7 @@ def test_error_method(self): # Create random noise and see if the standard deviation is equal (it should) dem3 = dem1 + np.random.random(size=dem1.size).reshape(dem1.shape) - assert biascorr.error(dem1, dem3, transform=affine, error_type="std") == np.std(dem3) + assert abs(biascorr.error(dem1, dem3, transform=affine, error_type="std") - np.std(dem3)) < 1e-6 diff --git a/tests/test_docs.py b/tests/test_docs.py index 0b0b1d8c..2bd74a3b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -4,6 +4,7 @@ import sys import warnings +import sphinx.cmd.build class TestDocs: docs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../", "docs/") @@ -28,36 +29,12 @@ def test_example_code(self): def test_build(self): """Try building the docs and see if it works.""" - current_dir = os.getcwd() - # Change into the docs directory. - os.chdir(self.docs_dir) - # Remove the build directory if it exists. - if os.path.isdir("build/"): - shutil.rmtree("build/") + if os.path.isdir(os.path.join(self.docs_dir, "build/")): + shutil.rmtree(os.path.join(self.docs_dir, "build/")) - # Copy the environment and set the SPHINXBUILD variable to call the module. - # This is for it to work properly with GitHub Workflows - env = os.environ.copy() - env["SPHINXBUILD"] = f"{sys.executable} -m sphinx" - - # Run the makefile - build_commands = ["make", "html"] - result = subprocess.run( - build_commands, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - env=env - ) - - # Raise an error if the string "error" is in the stderr. - if "error" in str(result.stderr).lower(): - raise RuntimeError(result.stderr) - - # If "error" is not in the stderr string but it exists, show it as a warning. - if len(result.stderr) > 0: - warnings.warn(result.stderr) + sphinx.cmd.build.main([ + os.path.join(self.docs_dir, "source/"), + os.path.join(self.docs_dir, "build/") + ]) - os.chdir(current_dir)