diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index dc61a18bd4..155b27add4 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -21,17 +21,17 @@ env: # nomkl: make sure numpy w/out mkl # setuptools_scm: needed for versioning to work CONDA_DEFAULT_DEPENDENCIES: python-build nomkl setuptools_scm - LATEST_SUPPORTED_PYTHON_VERSION: "3.11" + LATEST_SUPPORTED_PYTHON_VERSION: "3.12" jobs: check-syntax-errors: name: Check for syntax errors runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }} @@ -46,13 +46,13 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide python -m flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - check-history-rst-syntax: - name: Check HISTORY RST syntax + check-rst-syntax: + name: Check RST syntax runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 name: Set up python with: python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }} @@ -63,7 +63,7 @@ jobs: - name: Lint with doc8 run: | # Skip line-too-long errors (D001) - python -m doc8 --ignore D001 HISTORY.rst + python -m doc8 --ignore D001 HISTORY.rst README_PYTHON.rst run-model-tests: name: Run model tests @@ -72,14 +72,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] - os: [windows-latest, macos-latest] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] + os: [windows-latest, macos-13] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch complete history for accurate versioning - # NOTE: It takes twice as long to save the sample data cache # as it does to do a fresh clone (almost 5 minutes vs. 2.5 minutes) # Test data is way, way faster by contrast (on the order of a few @@ -94,7 +93,7 @@ jobs: uses: ./.github/actions/setup_env with: python-version: ${{ matrix.python-version }} - requirements-files: requirements.txt requirements-dev.txt + requirements-files: requirements.txt requirements-dev.txt constraints_tests.txt requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} - name: Download previous conda environment.yml @@ -128,13 +127,13 @@ jobs: run: make test - name: Upload wheel artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Wheel for ${{ matrix.os }} ${{ matrix.python-version }} path: dist - name: Upload conda env artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: Conda Env for ${{ matrix.os }} ${{ matrix.python-version }} @@ -142,13 +141,13 @@ jobs: - name: Authenticate GCP if: github.event_name != 'pull_request' - uses: google-github-actions/auth@v0 + uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GOOGLE_SERVICE_ACC_KEY }} - name: Set up GCP if: github.event_name != 'pull_request' - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v2 - name: Deploy artifacts to GCS if: github.event_name != 'pull_request' @@ -160,16 +159,18 @@ jobs: needs: check-syntax-errors strategy: matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] - os: [windows-latest, macos-latest] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] + os: [windows-latest, macos-13] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch complete history for accurate versioning - uses: ./.github/actions/setup_env with: python-version: ${{ matrix.python-version }} requirements-files: requirements.txt - requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} + requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} twine - name: Build source distribution run: | @@ -177,12 +178,14 @@ jobs: # computer is guaranteed to have cython available at build # time. Thus, it is no longer necessary to distribute the # .cpp files in addition to the .pyx files. - python -m build --sdist + # + # Elevating any python warnings to errors to catch build issues ASAP. + python -W error -m build --sdist - name: Install from source distribution run : | # Install natcap.invest from the sdist in dist/ - pip install $(find dist -name "natcap.invest*") + pip install $(find dist -name "natcap[._-]invest*") # Model tests should cover model functionality, we just want # to be sure that we can import `natcap.invest` here. @@ -190,9 +193,12 @@ jobs: # natcap.invest from source and that it imports. python -c "from natcap.invest import *" - - uses: actions/upload-artifact@v3 + - name: Check long description with twine + run: twine check $(find dist -name "natcap[._-]invest*") + + - uses: actions/upload-artifact@v4 with: - name: Source distribution + name: Source distribution ${{ matrix.os }} ${{ matrix.python-version }} path: dist # Secrets not available in PR so don't use GCP. @@ -200,17 +206,17 @@ jobs: # overwrite artifacts or have duplicates (mac/windows sdists have # different extensions) - name: Authenticate GCP - if: github.event_name != 'pull_request' && matrix.os == 'macos-latest' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION - uses: google-github-actions/auth@v0 + if: github.event_name != 'pull_request' && matrix.os == 'macos-13' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION + uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GOOGLE_SERVICE_ACC_KEY }} - name: Set up GCP - if: github.event_name != 'pull_request' && matrix.os == 'macos-latest' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION - uses: google-github-actions/setup-gcloud@v0 + if: github.event_name != 'pull_request' && matrix.os == 'macos-13' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION + uses: google-github-actions/setup-gcloud@v2 - name: Deploy artifacts to GCS - if: github.event_name != 'pull_request' && matrix.os == 'macos-latest' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION + if: github.event_name != 'pull_request' && matrix.os == 'macos-13' && matrix.python-version == env.LATEST_SUPPORTED_PYTHON_VERSION run: make deploy validate-resources: @@ -218,7 +224,7 @@ jobs: runs-on: windows-latest needs: check-syntax-errors steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch complete history for accurate versioning @@ -245,11 +251,11 @@ jobs: fail-fast: false max-parallel: 4 matrix: - os: [windows-latest, macos-latest] + os: [windows-latest, macos-13] steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch complete history for accurate versioning @@ -273,7 +279,7 @@ jobs: uses: actions/cache@v3 with: path: workbench/node_modules - key: ${{ runner.os }}-${{ hashFiles('workbench/yarn.lock') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('workbench/yarn.lock') }} - name: Install workbench dependencies if: steps.nodemodules-cache.outputs.cache-hit != 'true' @@ -295,9 +301,9 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [macos-13, windows-latest] include: - - os: macos-latest + - os: macos-13 puppeteer-log: ~/Library/Logs/invest-workbench/ workspace-path: InVEST-failed-mac-workspace.tar binary-extension: dmg @@ -306,7 +312,7 @@ jobs: workspace-path: ${{ github.workspace }} binary-extension: exe steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch complete history for accurate versioning @@ -314,7 +320,11 @@ jobs: uses: ./.github/actions/setup_env with: python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }} - requirements-files: requirements.txt requirements-dev.txt requirements-docs.txt + requirements-files: | + requirements.txt + requirements-dev.txt + requirements-docs.txt + constraints_tests.txt requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} pandoc - name: Make install @@ -348,7 +358,7 @@ jobs: uses: actions/cache@v3 with: path: workbench/node_modules - key: ${{ runner.os }}-${{ hashFiles('workbench/yarn.lock') }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('workbench/yarn.lock') }} - name: Install Workbench Dependencies if: steps.nodemodules-cache.outputs.cache-hit != 'true' @@ -359,13 +369,13 @@ jobs: - name: Authenticate GCP if: github.event_name != 'pull_request' - uses: google-github-actions/auth@v0 + uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GOOGLE_SERVICE_ACC_KEY }} - name: Set up GCP if: github.event_name != 'pull_request' - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v2 - name: Build Workbench (PRs) if: github.event_name == 'pull_request' @@ -379,7 +389,7 @@ jobs: yarn run dist - name: Build Workbench (macOS) - if: github.event_name != 'pull_request' && matrix.os == 'macos-latest' # secrets not available in PR + if: github.event_name != 'pull_request' && matrix.os == 'macos-13' # secrets not available in PR working-directory: workbench env: GH_TOKEN: env.GITHUB_TOKEN @@ -397,10 +407,10 @@ jobs: env: GH_TOKEN: env.GITHUB_TOKEN DEBUG: electron-builder - CSC_LINK: Stanford-natcap-code-signing-cert-expires-2024-01-26.p12 - CSC_KEY_PASSWORD: ${{ secrets.WINDOWS_CODESIGN_CERT_PASS }} + #CSC_LINK: Stanford-natcap-code-signing-cert-expires-2024-01-26.p12 + #CSC_KEY_PASSWORD: ${{ secrets.WINDOWS_CODESIGN_CERT_PASS }} run: | - gsutil cp gs://stanford_cert/$CSC_LINK $CSC_LINK + #gsutil cp gs://stanford_cert/$CSC_LINK $CSC_LINK yarn run build yarn run dist @@ -409,7 +419,7 @@ jobs: run: npx cross-env CI=true yarn run test-electron-app - name: Sign binaries (macOS) - if: github.event_name != 'pull_request' && matrix.os == 'macos-latest' # secrets not available in PR + if: github.event_name != 'pull_request' && matrix.os == 'macos-13' # secrets not available in PR env: CERT_FILE: 2025-01-16-Expiry-AppStore-App.p12 CERT_PASS: ${{ secrets.MACOS_CODESIGN_CERT_PASS }} @@ -417,16 +427,16 @@ jobs: WORKBENCH_BINARY=$(find "$(pwd)/workbench/dist" -type f -name 'invest_*.dmg' | head -n 1) make WORKBENCH_BIN_TO_SIGN="$WORKBENCH_BINARY" codesign_mac - - name: Sign binaries (Windows) - if: github.event_name != 'pull_request' && matrix.os == 'windows-latest' # secrets not available in PR - env: - CERT_FILE: Stanford-natcap-code-signing-cert-expires-2024-01-26.p12 - CERT_PASS: ${{ secrets.WINDOWS_CODESIGN_CERT_PASS }} - run: | - # figure out the path to signtool.exe (it keeps changing with SDK updates) - SIGNTOOL_PATH=$(find 'C:\\Program Files (x86)\\Windows Kits\\10' -type f -name 'signtool.exe*' | head -n 1) - WORKBENCH_BINARY=$(find "$(pwd)/workbench/dist" -type f -name 'invest_*.exe' | head -n 1) - make WORKBENCH_BIN_TO_SIGN="$WORKBENCH_BINARY" SIGNTOOL="$SIGNTOOL_PATH" codesign_windows + #- name: Sign binaries (Windows) + # if: github.event_name != 'pull_request' && matrix.os == 'windows-latest' # secrets not available in PR + # env: + # CERT_FILE: Stanford-natcap-code-signing-cert-expires-2024-01-26.p12 + # CERT_PASS: ${{ secrets.WINDOWS_CODESIGN_CERT_PASS }} + # run: | + # # figure out the path to signtool.exe (it keeps changing with SDK updates) + # SIGNTOOL_PATH=$(find 'C:\\Program Files (x86)\\Windows Kits\\10' -type f -name 'signtool.exe*' | head -n 1) + # WORKBENCH_BINARY=$(find "$(pwd)/workbench/dist" -type f -name 'invest_*.exe' | head -n 1) + # make WORKBENCH_BIN_TO_SIGN="$WORKBENCH_BINARY" SIGNTOOL="$SIGNTOOL_PATH" codesign_windows - name: Deploy artifacts to GCS if: github.event_name != 'pull_request' @@ -434,20 +444,20 @@ jobs: - name: Upload workbench binary artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Workbench-${{ runner.os }}-binary path: workbench/dist/*.${{ matrix.binary-extension }} - name: Upload user's guide artifact (Windows) if: matrix.os == 'windows-latest' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: InVEST-user-guide path: dist/InVEST_*_userguide.zip - name: Upload workbench logging from puppeteer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: ${{ runner.os }}_puppeteer_log.zip' @@ -461,12 +471,12 @@ jobs: run: make invest_autotest - name: Tar the workspace to preserve permissions (macOS) - if: failure() && matrix.os == 'macos-latest' + if: failure() && matrix.os == 'macos-13' run: tar -cvf ${{ matrix.workspace-path}} ${{ github.workspace }} - name: Upload workspace on failure if: ${{ failure() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: InVEST-failed-${{ runner.os }}-workspace path: ${{ matrix.workspace-path}} @@ -476,11 +486,11 @@ jobs: needs: check-syntax-errors runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch complete history for accurate versioning - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }} @@ -491,20 +501,20 @@ jobs: - run: make sampledata sampledata_single - name: Upload sample data artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: InVEST-sample-data path: dist/*.zip - name: Authenticate GCP if: github.event_name != 'pull_request' - uses: google-github-actions/auth@v0 + uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GOOGLE_SERVICE_ACC_KEY }} - name: Set up GCP if: github.event_name != 'pull_request' - uses: google-github-actions/setup-gcloud@v0 + uses: google-github-actions/setup-gcloud@v2 - name: Deploy artifacts to GCS if: github.event_name != 'pull_request' diff --git a/.readthedocs.yml b/.readthedocs.yml index eba6cf0b4c..e3f13fa6aa 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -18,5 +18,9 @@ build: tools: python: "mambaforge-4.10" jobs: + post_create_environment: + - pip install --upgrade-strategy=only-if-needed -r requirements.txt + - pip install --upgrade-strategy=only-if-needed -r requirements-dev.txt + - pip install --upgrade-strategy=only-if-needed -r requirements-docs.txt post_install: - make install diff --git a/.readthedocs_environment.yml b/.readthedocs_environment.yml index 1d1681aa5f..2f2a2de840 100644 --- a/.readthedocs_environment.yml +++ b/.readthedocs_environment.yml @@ -11,9 +11,5 @@ channels: - nodefaults dependencies: - python=3.11 -- gdal>=3.4.2,<3.6.0 +- gdal>=3.4.2 - pip -- pip: - - -r requirements.txt - - -r requirements-dev.txt - - -r requirements-docs.txt diff --git a/HISTORY.rst b/HISTORY.rst index 439ca6d11c..dfa49b3025 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,10 +35,26 @@ .. :changelog: - Unreleased Changes ------------------ +* Workbench + * Several small updates to the model input form UI to improve usability + and visual consistency (https://github.com/natcap/invest/issues/912) + * Fixed a bug that caused the application to crash when attempting to + open a workspace without a valid logfile + (https://github.com/natcap/invest/issues/1598) + * Fixed a bug that was allowing readonly workspace directories on Windows + (https://github.com/natcap/invest/issues/1599) + * Fixed a bug that, in certain scenarios, caused a datastack to be saved + with relative paths when the Relative Paths checkbox was left unchecked + (https://github.com/natcap/invest/issues/1609) + +3.14.2 (2024-05-29) +------------------- * General + * Validation now covers file paths contained in CSVs. CSV validation + will fail if the files listed in a CSV fail to validate. + https://github.com/natcap/invest/issues/327 * We have updated validation in several ways that will improve the developer experience of working with InVEST models, and we hope will also improve the user experience: @@ -59,15 +75,52 @@ Unreleased Changes versions of InVEST would skip these parameters' type-specific validation. Now, these parameters will be validated with their type-specific validation checks. - + * Add support for latest GDAL versions; remove test-specific constraint on + GDAL versions from invest requirements. + https://github.com/natcap/invest/issues/916 + * Updated to Cython 3 (https://github.com/natcap/invest/issues/556) * Annual Water Yield * Added the results_suffix to a few intermediate files where it was missing. https://github.com/natcap/invest/issues/1517 +* Coastal Blue Carbon + * Updated model validation to prevent the case where a user provides only + one snapshot year and no analysis year + (`#1534 `_). + Also enforces that the analysis year, if provided, is greater than the + latest snapshot year. An analysis year equal to the latest snapshot year + is no longer allowed. * Coastal Vulnerability * Fixed a bug in handling ``nan`` as the nodata value of the bathymetry raster. ``nan`` pixels will now be propertly ignored before calculating mean depths along fetch rays. https://github.com/natcap/invest/issues/1528 +* HRA + * Fixed a bug where habitat and stressor vectors were not being rasterized + with the `ALL_TOUCHED=TRUE` setting. +* Scenic Quality + * Fixed an issue with viewshed calculations where some slight numerical + error was introduced on M1 Macs, but not on x86-based computers. This + numerical error was leading to slightly different visibility results. + https://github.com/natcap/invest/issues/1562 +* SDR + * Fixed an issue encountered in the sediment deposition function where + rasters with more than 2^32 pixels would raise a cryptic error relating + to negative dimensions. https://github.com/natcap/invest/issues/1431 + * Optimized the creation of the summary vector by minimizing the number of + times the target vector needs to be rasterized. +* Seasonal Water Yield + * Fixed an issue with the precip directory units. Units for these input + rasters are now correctly stated as mm/month. + https://github.com/natcap/invest/issues/1571 + * Fixed an issue where the monthly quickflow values were being summed over + a block area and not summed pixelwise. This caused the quickflow + output ``QF.tif`` to have malformed values. + https://github.com/natcap/invest/issues/1541 +* Wind Energy + * Fixed a bug where some number inputs were not being properly cast to + ``float`` or ``int`` types. If the inputs happened to be passed as + a ``str`` this caused unintended side effects such as a concatenation + error. (https://github.com/natcap/invest/issues/1498) * Urban Nature Access * Fixed a ``NameError`` that occurred when running the model using search radii defined per population group with an exponential search @@ -91,12 +144,6 @@ Unreleased Changes * Fixed an issue where an LULC raster without a nodata value would always raise in exception during reclassification. https://github.com/natcap/invest/issues/1539 -* SDR - * Fixed an issue encountered in the sediment deposition function where - rasters with more than 2^32 pixels would raise a cryptic error relating - to negative dimensions. https://github.com/natcap/invest/issues/1431 - * Optimized the creation of the summary vector by minimizing the number of - times the target vector needs to be rasterized. 3.14.1 (2023-12-18) ------------------- diff --git a/Makefile b/Makefile index 707f356fe8..234e814bbf 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,15 @@ DATA_DIR := data GIT_SAMPLE_DATA_REPO := https://bitbucket.org/natcap/invest-sample-data.git GIT_SAMPLE_DATA_REPO_PATH := $(DATA_DIR)/invest-sample-data -GIT_SAMPLE_DATA_REPO_REV := 2e7cd618c661ec3f3b2a3bddfd2ce7d4704abc05 +GIT_SAMPLE_DATA_REPO_REV := ab8c74a62a93fd0019de2bca064abc0a5a07afab GIT_TEST_DATA_REPO := https://bitbucket.org/natcap/invest-test-data.git GIT_TEST_DATA_REPO_PATH := $(DATA_DIR)/invest-test-data -GIT_TEST_DATA_REPO_REV := da013683e80ea094fbb2309197e2488c02794da8 +GIT_TEST_DATA_REPO_REV := 324abde73e1d770ad75921466ecafd1ec6297752 GIT_UG_REPO := https://github.com/natcap/invest.users-guide GIT_UG_REPO_PATH := doc/users-guide -GIT_UG_REPO_REV := fa6b181d49136089dce56d4ff8f3dcaf12eb4ced +GIT_UG_REPO_REV := 0404bc5d4d43085cdc58f50f8fc29944b10cefb1 ENV = "./env" ifeq ($(OS),Windows_NT) @@ -251,6 +251,7 @@ $(INVEST_BINARIES_DIR): | $(DIST_DIR) $(BUILD_DIR) -$(RMDIR) $(INVEST_BINARIES_DIR) $(PYTHON) -m PyInstaller --workpath $(BUILD_DIR)/pyi-build --clean --distpath $(DIST_DIR) exe/invest.spec $(CONDA) list > $(INVEST_BINARIES_DIR)/package_versions.txt + $(PYTHON) -m pip list >> $(INVEST_BINARIES_DIR)/package_versions.txt $(INVEST_BINARIES_DIR)/invest list # Documentation. diff --git a/constraints_tests.txt b/constraints_tests.txt new file mode 100644 index 0000000000..299273430e --- /dev/null +++ b/constraints_tests.txt @@ -0,0 +1,15 @@ +# This file contains package constraints needed to run the invest test suite. +# It follows the pip constraints file format: +# https://pip.pypa.io/en/stable/user_guide/#constraints-files + +# A gdal bug caused our test suite to fail, but this issue is unlikely to +# occur with regular use of invest. https://github.com/OSGeo/gdal/issues/8497 +GDAL!=3.6.*,!=3.7.* + +# https://github.com/natcap/pygeoprocessing/issues/387 +GDAL<3.8.5 + +# Pyinstaller 6.10 breaks our windows builds. Until we can figure out the +# root cause, let's cap the versions to those that work. +# https://github.com/natcap/invest/issues/1622 +#pyinstaller<6.10 diff --git a/docker/Dockerfile b/docker/Dockerfile index ab38b0a139..496948c370 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,11 +13,15 @@ RUN cd / && \ # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:1.5.0-bookworm-slim +# Python version should match the version used in stage 1. +# If we update the stage 1 debian version, also update this python version +ARG PYTHON_VERSION="3.11" COPY --from=build /invest/dist/*.whl /tmp/ # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml -RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ +RUN micromamba install -y -n base -c conda-forge python==${PYTHON_VERSION} && \ + micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ /opt/conda/bin/python -m pip install /tmp/*.whl && \ /opt/conda/bin/python -m pip cache purge && \ diff --git a/exe/hooks/rthook.py b/exe/hooks/rthook.py index 53a9af14b6..ec0b3264a8 100644 --- a/exe/hooks/rthook.py +++ b/exe/hooks/rthook.py @@ -1,9 +1,6 @@ -import sys import os -import multiprocessing import platform - -multiprocessing.freeze_support() +import sys os.environ['PROJ_LIB'] = os.path.join(sys._MEIPASS, 'proj') diff --git a/exe/invest.spec b/exe/invest.spec index d019f413be..b0f3085ec7 100644 --- a/exe/invest.spec +++ b/exe/invest.spec @@ -30,6 +30,9 @@ kwargs = { 'pkg_resources.py2_warn', 'cmath', 'charset_normalizer', + 'scipy.special._cdflib', + 'scipy.special._special_ufuncs', + 'scipy._lib.array_api_compat.numpy.fft', ], 'datas': [proj_datas], 'cipher': block_cipher, diff --git a/pyproject.toml b/pyproject.toml index fed1937a6c..fadad29609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "natcap.invest" description = "InVEST Ecosystem Service models" readme = "README_PYTHON.rst" -requires-python = ">=3.8,<3.12" +requires-python = ">=3.8" license = {file = "LICENSE.txt"} maintainers = [ {name = "Natural Capital Project Software Team"} @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Cython", "License :: OSI Approved :: BSD License", "Topic :: Scientific/Engineering :: GIS" @@ -44,7 +45,7 @@ invest = "natcap.invest.cli:main" # that we can provide a much easier build experience so long as GDAL is # available at runtime. requires = [ - 'setuptools>=61', 'wheel', 'setuptools_scm>=8.0', 'cython', 'babel', + 'setuptools>=61', 'wheel', 'setuptools_scm>=8.0', 'cython>=3.0.0', 'babel', 'oldest-supported-numpy' ] build-backend = "setuptools.build_meta" @@ -78,4 +79,11 @@ where = ["src"] [tool.pytest.ini_options] # raise warnings to errors, except for deprecation warnings -filterwarnings = ["error", "default::DeprecationWarning"] +filterwarnings = [ + "error", + "default::DeprecationWarning", + "default::FutureWarning", + # don't error on a specific runtime warning coming from a shapely + # issue on M1: https://github.com/natcap/invest/issues/1562 + "default:invalid value encountered in intersection:RuntimeWarning", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 5f828011dc..6325f1f62d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,15 +9,18 @@ # Any lines with "# pip-only" at the end will be processed by # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. +# Sometimes conda-forge does not install the latest available version, +# pip-only can be a workaround for that. -Cython<3.0.0 virtualenv>=12.0.1 pytest pytest-subtests wheel>=0.27.0 pypiwin32; sys_platform == 'win32' # pip-only -setuptools>=8.0,<60.7.0 # https://github.com/pyinstaller/pyinstaller/issues/6564 -PyInstaller>=4.10 + +# 60.7.0 exception because of https://github.com/pyinstaller/pyinstaller/issues/6564 +setuptools>=8.0,!=60.7.0 +PyInstaller>=4.10 # pip-only setuptools_scm>=6.4.0 requests coverage diff --git a/requirements.txt b/requirements.txt index cad4fa0e7f..cf2d3aeb76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,10 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2,<3.6.0 +GDAL>=3.4.2 Pyro4==4.77 # pip-only pandas>=1.2.1 -numpy>=1.11.0,!=1.16.0 +numpy>=1.11.0,!=1.16.0,<2.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0,!=1.12.* diff --git a/setup.py b/setup.py index a05b74314d..ea852f7e5b 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,8 @@ import platform import subprocess -import Cython.Build import numpy +from Cython.Build import cythonize from setuptools import setup from setuptools.command.build_py import build_py as _build_py from setuptools.extension import Extension @@ -46,25 +46,34 @@ def run(self): setup( install_requires=_REQUIREMENTS, - ext_modules=[ + ext_modules=cythonize([ Extension( name=f'natcap.invest.{package}.{module}', sources=[f'src/natcap/invest/{package}/{module}.pyx'], - include_dirs=[numpy.get_include()], - extra_compile_args=compiler_and_linker_args, + extra_compile_args=compiler_args + compiler_and_linker_args, extra_link_args=compiler_and_linker_args, - language='c++' - ) for package, module in [ - ('delineateit', 'delineateit_core'), - ('recreation', 'out_of_core_quadtree'), - ('scenic_quality', 'viewshed'), - ('ndr', 'ndr_core'), - ('sdr', 'sdr_core'), - ('seasonal_water_yield', 'seasonal_water_yield_core') + language='c++', + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")] + ) for package, module, compiler_args in [ + ('delineateit', 'delineateit_core', []), + ('recreation', 'out_of_core_quadtree', []), + # clang-14 defaults to -ffp-contract=on, which causes the + # arithmetic of A*B+C to be implemented using a contraction, which + # causes an unexpected change in the precision in some viewshed + # tests on ARM64 (mac M1). See these issues for more details: + # * https://github.com/llvm/llvm-project/issues/91824 + # * https://github.com/natcap/invest/issues/1562 + # * https://github.com/natcap/invest/pull/1564/files + # Using this flag on gcc and on all versions of clang should work + # as expected, with consistent results. + ('scenic_quality', 'viewshed', ['-ffp-contract=off']), + ('ndr', 'ndr_core', []), + ('sdr', 'sdr_core', []), + ('seasonal_water_yield', 'seasonal_water_yield_core', []) ] - ], + ], compiler_directives={'language_level': '3'}), + include_dirs=[numpy.get_include()], cmdclass={ - 'build_ext': Cython.Build.build_ext, 'build_py': build_py } ) diff --git a/src/natcap/invest/__main__.py b/src/natcap/invest/__main__.py index 1f68c51913..167e0fc170 100644 --- a/src/natcap/invest/__main__.py +++ b/src/natcap/invest/__main__.py @@ -1,5 +1,11 @@ +import multiprocessing import sys +# We want to guarantee that this is called BEFORE any other processes start, +# which could happen at import time. +if __name__ == '__main__': + multiprocessing.freeze_support() + from . import cli if __name__ == '__main__': diff --git a/src/natcap/invest/coastal_blue_carbon/coastal_blue_carbon.py b/src/natcap/invest/coastal_blue_carbon/coastal_blue_carbon.py index f1f7fc67a0..7e118cdd5f 100644 --- a/src/natcap/invest/coastal_blue_carbon/coastal_blue_carbon.py +++ b/src/natcap/invest/coastal_blue_carbon/coastal_blue_carbon.py @@ -114,10 +114,10 @@ LOGGER = logging.getLogger(__name__) INVALID_ANALYSIS_YEAR_MSG = gettext( - "Analysis year {analysis_year} must be >= the latest snapshot year " + "Analysis year ({analysis_year}) must be greater than the latest snapshot year " "({latest_year})") -INVALID_SNAPSHOT_RASTER_MSG = gettext( - "Raster for snapshot {snapshot_year} could not be validated.") +MISSING_ANALYSIS_YEAR_MSG = gettext( + "Analysis year is required if only one snapshot year is provided.") INVALID_TRANSITION_VALUES_MSG = gettext( "The transition table expects values of {model_transitions} but found " "values of {transition_values}.") @@ -2166,7 +2166,6 @@ def validate(args, limit_to=None): """ validation_warnings = validation.validate( args, MODEL_SPEC['args']) - sufficient_keys = validation.get_sufficient_keys(args) invalid_keys = validation.get_invalid_keys(validation_warnings) @@ -2177,19 +2176,14 @@ def validate(args, limit_to=None): **MODEL_SPEC['args']['landcover_snapshot_csv'] )['raster_path'].to_dict() - for snapshot_year, snapshot_raster_path in snapshots.items(): - raster_error_message = validation.check_raster( - snapshot_raster_path) - if raster_error_message: - validation_warnings.append(( - ['landcover_snapshot_csv'], - INVALID_SNAPSHOT_RASTER_MSG.format( - snapshot_year=snapshot_year - ) + ' ' + raster_error_message)) + snapshot_years = set(snapshots.keys()) + if len(snapshot_years) == 1 and "analysis_year" not in sufficient_keys: + validation_warnings.append( + (['analysis_year'], MISSING_ANALYSIS_YEAR_MSG)) if ("analysis_year" not in invalid_keys and "analysis_year" in sufficient_keys): - if max(set(snapshots.keys())) > int(args['analysis_year']): + if max(snapshot_years) >= int(args['analysis_year']): validation_warnings.append(( ['analysis_year'], INVALID_ANALYSIS_YEAR_MSG.format( diff --git a/src/natcap/invest/coastal_vulnerability.py b/src/natcap/invest/coastal_vulnerability.py index 8ebf032988..ff30d3a24e 100644 --- a/src/natcap/invest/coastal_vulnerability.py +++ b/src/natcap/invest/coastal_vulnerability.py @@ -240,7 +240,7 @@ "type": "freestyle_string", "about": gettext("Unique name for the habitat. No spaces allowed.")}, "path": { - "type": {"vector", "raster"}, + "type": {"raster", "vector"}, "fields": {}, "geometries": {"POLYGON", "MULTIPOLYGON"}, "bands": {1: {"type": "number", "units": u.none}}, @@ -771,8 +771,6 @@ def execute(args): None """ - _validate_habitat_table_paths(args['habitat_table_path']) - output_dir = os.path.join(args['workspace_dir']) intermediate_dir = os.path.join( args['workspace_dir'], 'intermediate') @@ -3450,36 +3448,6 @@ def logger_callback(proportion_complete): return logger_callback -def _validate_habitat_table_paths(habitat_table_path): - """Validate paths to vectors within the habitat CSV can be opened. - - Args: - habitat_table_path (str): typically args['habitat_table_path'] - - Returns: - None - - Raises: - ValueError if any vector in the ``path`` column cannot be opened. - """ - habitat_dataframe = validation.get_validated_dataframe( - habitat_table_path, **MODEL_SPEC['args']['habitat_table_path']) - bad_paths = [] - for habitat_row in habitat_dataframe.itertuples(): - try: - gis_type = pygeoprocessing.get_gis_type(habitat_row.path) - if not gis_type: - # Treating an unknown GIS type the same as a bad filepath - bad_paths.append(habitat_row.path) - except ValueError: - bad_paths.append(habitat_row.path) - - if bad_paths: - raise ValueError( - f'Could not open these datasets referenced in {habitat_table_path}:' - + ' | '.join(bad_paths)) - - @validation.invest_validator def validate(args, limit_to=None): """Validate args to ensure they conform to ``execute``'s contract. diff --git a/src/natcap/invest/delineateit/delineateit_core.pyx b/src/natcap/invest/delineateit/delineateit_core.pyx index 979423e626..0b229a814f 100644 --- a/src/natcap/invest/delineateit/delineateit_core.pyx +++ b/src/natcap/invest/delineateit/delineateit_core.pyx @@ -1,4 +1,3 @@ -# cython: language_level=3 import numpy import pygeoprocessing cimport numpy @@ -105,5 +104,3 @@ cpdef cset[cpair[double, double]] calculate_pour_point_array( # return set of (x, y) coordinates referenced to the same coordinate system # as the original raster return pour_points - - diff --git a/src/natcap/invest/habitat_quality.py b/src/natcap/invest/habitat_quality.py index 75c203294d..2bf024fe46 100644 --- a/src/natcap/invest/habitat_quality.py +++ b/src/natcap/invest/habitat_quality.py @@ -1040,23 +1040,13 @@ def _validate_threat_path(threat_path, lulc_key): """ # Checking threat path exists to control custom error messages # for user readability. - try: - threat_gis_type = pygeoprocessing.get_gis_type(threat_path) - if threat_gis_type != pygeoprocessing.RASTER_TYPE: - # Raise a value error with custom message to help users - # debug threat raster issues - if lulc_key != '_b': - return "error" - # it's OK to have no threat raster w/ baseline scenario - else: - return None - else: - return threat_path - except ValueError: - if lulc_key != '_b': - return "error" - else: + if threat_path: + return threat_path + else: + if lulc_key == '_b': return None + else: + return 'error' @validation.invest_validator diff --git a/src/natcap/invest/hra.py b/src/natcap/invest/hra.py index 5d5c0d477c..e8db4ce113 100644 --- a/src/natcap/invest/hra.py +++ b/src/natcap/invest/hra.py @@ -70,7 +70,7 @@ "names must match the habitat and stressor names in " "the Criteria Scores Table.")}, "path": { - "type": {"vector", "raster"}, + "type": {"raster", "vector"}, "bands": {1: { "type": "number", "units": u.none, @@ -80,13 +80,13 @@ "values besides 0 or 1 will be treated as 0.") }}, "fields": {}, - "geometries": spec_utils.POLYGONS, + "geometries": spec_utils.ALL_GEOMS, "about": gettext( "Map of where the habitat or stressor exists. For " "rasters, a pixel value of 1 indicates presence of " "the habitat or stressor. 0 (or any other value) " "indicates absence of the habitat or stressor. For " - "vectors, a polygon indicates an area where the " + "vectors, a geometry indicates an area where the " "habitat or stressor is present.") }, "type": { @@ -633,10 +633,6 @@ def execute(args): # If the input is a vector, reproject to the AOI SRS and simplify. # Rasterization happens in the alignment step. elif gis_type == pygeoprocessing.VECTOR_TYPE: - # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE - if name in habitats_info or name in stressors_info: - habitat_stressor_vectors.add(source_filepath) - # Using Shapefile here because its driver appears to not raise a # warning if a MultiPolygon geometry is inserted into a Polygon # layer, which was happening on a real-world sample dataset while @@ -680,6 +676,10 @@ def execute(args): dependent_task_list=[reprojected_vector_task] )) + # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE + if name in habitats_info or name in stressors_info: + habitat_stressor_vectors.add(target_simplified_vector) + # Later operations make use of the habitats rasters or the stressors # rasters, so it's useful to collect those here now. if name in habitats_info: @@ -1648,7 +1648,6 @@ def _simplify(source_vector_path, tolerance, target_vector_path, for source_feature in source_layer: target_feature = ogr.Feature(target_layer_defn) source_geom = source_feature.GetGeometryRef() - simplified_geom = source_geom.SimplifyPreserveTopology(tolerance) if simplified_geom is not None: target_geom = simplified_geom @@ -1785,6 +1784,8 @@ def _parse_info_table(info_table_path): except ValueError as err: if 'Index has duplicate keys' in str(err): raise ValueError("Habitat and stressor names may not overlap.") + else: + raise err table = table.rename(columns={'stressor buffer (meters)': 'buffer'}) diff --git a/src/natcap/invest/ndr/ndr_core.pyx b/src/natcap/invest/ndr/ndr_core.pyx index b56ba40d12..d921b1bc59 100644 --- a/src/natcap/invest/ndr/ndr_core.pyx +++ b/src/natcap/invest/ndr/ndr_core.pyx @@ -1,5 +1,3 @@ -# cython: profile=False -# cython: language_level=2 import tempfile import logging import os @@ -129,9 +127,9 @@ cdef class _ManagedRaster: self.block_xbits = numpy.log2(self.block_xsize) self.block_ybits = numpy.log2(self.block_ysize) self.block_nx = ( - self.raster_x_size + (self.block_xsize) - 1) / self.block_xsize + self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize self.block_ny = ( - self.raster_y_size + (self.block_ysize) - 1) / self.block_ysize + self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS) self.raster_path = raster_path @@ -197,7 +195,7 @@ cdef class _ManagedRaster: if dirty_itr != self.dirty_blocks.end(): self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for # cached array @@ -261,7 +259,7 @@ cdef class _ManagedRaster: cdef void _load_block(self, int block_index) except *: cdef int block_xi = block_index % self.block_nx - cdef int block_yi = block_index / self.block_nx + cdef int block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for cached array cdef int xoff = block_xi << self.block_xbits @@ -322,7 +320,7 @@ cdef class _ManagedRaster: self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx xoff = block_xi << self.block_xbits yoff = block_yi << self.block_ybits @@ -415,9 +413,9 @@ def ndr_eff_calculation( # create direction raster in bytes def _mfd_to_flow_dir_op(mfd_array): - result = numpy.zeros(mfd_array.shape, dtype=numpy.int8) + result = numpy.zeros(mfd_array.shape, dtype=numpy.uint8) for i in range(8): - result[:] |= (((mfd_array >> (i*4)) & 0xF) > 0) << i + result[:] |= ((((mfd_array >> (i*4)) & 0xF) > 0) << i).astype(numpy.uint8) return result pygeoprocessing.raster_calculator( @@ -488,7 +486,7 @@ def ndr_eff_calculation( # hasn't already been set for processing. flat_index = processing_stack.top() processing_stack.pop() - global_row = flat_index / n_cols + global_row = flat_index // n_cols global_col = flat_index % n_cols crit_len = crit_len_raster.get(global_col, global_row) diff --git a/src/natcap/invest/recreation/out_of_core_quadtree.pyx b/src/natcap/invest/recreation/out_of_core_quadtree.pyx index 89e5a2f950..bc5369617f 100644 --- a/src/natcap/invest/recreation/out_of_core_quadtree.pyx +++ b/src/natcap/invest/recreation/out_of_core_quadtree.pyx @@ -1,5 +1,3 @@ -# cython: profile=True -# cython: language_level=2 """A hierarchical spatial index for fast culling of points in 2D space.""" import os @@ -22,7 +20,7 @@ from osgeo import osr cimport numpy MAX_BYTES_TO_BUFFER = 2**27 # buffer a little over 128 megabytes -import buffered_numpy_disk_map +from natcap.invest.recreation import buffered_numpy_disk_map _ARRAY_TUPLE_TYPE = ( buffered_numpy_disk_map.BufferedNumpyDiskMap._ARRAY_TUPLE_TYPE) diff --git a/src/natcap/invest/scenic_quality/viewshed.pyx b/src/natcap/invest/scenic_quality/viewshed.pyx index 2f125a4077..51abf495b5 100644 --- a/src/natcap/invest/scenic_quality/viewshed.pyx +++ b/src/natcap/invest/scenic_quality/viewshed.pyx @@ -1,5 +1,3 @@ -# coding=UTF-8 -# cython: language_level=2 """ Implements the Wang et al (2000) viewshed based on reference planes. @@ -307,9 +305,9 @@ cdef class _ManagedRaster: self.block_xbits = numpy.log2(self.block_xsize) self.block_ybits = numpy.log2(self.block_ysize) self.block_nx = ( - self.raster_x_size + (self.block_xsize) - 1) / self.block_xsize + self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize self.block_ny = ( - self.raster_y_size + (self.block_ysize) - 1) / self.block_ysize + self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS) self.raster_path = raster_path @@ -375,7 +373,7 @@ cdef class _ManagedRaster: if dirty_itr != self.dirty_blocks.end(): self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for # cached array @@ -439,7 +437,7 @@ cdef class _ManagedRaster: cdef void _load_block(self, int block_index) except *: cdef int block_xi = block_index % self.block_nx - cdef int block_yi = block_index / self.block_nx + cdef int block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for cached array cdef int xoff = block_xi << self.block_xbits @@ -500,7 +498,7 @@ cdef class _ManagedRaster: self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx xoff = block_xi << self.block_xbits yoff = block_yi << self.block_ybits @@ -815,8 +813,7 @@ def viewshed(dem_raster_path_band, if target_distance > max_visible_radius: break - z = (((previous_height-r_v)/slope_distance) * - target_distance + r_v) + z = (((previous_height-r_v)/slope_distance) * target_distance) + r_v # add on refractivity/curvature-of-earth calculations. adjustment = 0.0 # increase in required height due to curvature diff --git a/src/natcap/invest/sdr/sdr_core.pyx b/src/natcap/invest/sdr/sdr_core.pyx index 45b71b25b1..69fc7500e4 100644 --- a/src/natcap/invest/sdr/sdr_core.pyx +++ b/src/natcap/invest/sdr/sdr_core.pyx @@ -1,5 +1,3 @@ -# cython: profile=False -# cython: language_level=3 import logging import os @@ -196,7 +194,7 @@ cdef class _ManagedRaster: if dirty_itr != self.dirty_blocks.end(): self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for # cached array diff --git a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py index 4f8d7396ac..25228b3db4 100644 --- a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py +++ b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py @@ -70,7 +70,14 @@ "contents": { # monthly precipitation maps, each file ending in a number 1-12 "[MONTH]": { - **spec_utils.PRECIP, + "type": "raster", + "bands": { + 1: { + "type": "number", + "units": u.millimeter/u.month, + }, + }, + "name": gettext("precipitation"), "about": gettext( "Twelve files, one for each month. File names must " "end with the month number (1-12). For example, " @@ -894,7 +901,7 @@ def execute(args): ], dependent_task_list=[ align_task, flow_dir_task, stream_threshold_task, - fill_pit_task, qf_task] + quick_flow_task_list, + fill_pit_task] + quick_flow_task_list, task_name='calculate local recharge') # calculate Qb as the sum of local_recharge_avail over the AOI, Eq [9] @@ -966,7 +973,7 @@ def execute(args): # raster_map equation: sum the monthly qfis -def qfi_sum_op(*qf_values): return numpy.sum(qf_values) +def qfi_sum_op(*qf_values): return numpy.sum(qf_values, axis=0) def _calculate_l_avail(l_path, gamma, target_l_avail_path): diff --git a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx index 6c6aad9eae..7902c3243b 100644 --- a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx +++ b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx @@ -1,5 +1,3 @@ -# cython: profile=False -# cython: language_level=2 import logging import os import collections @@ -146,9 +144,9 @@ cdef class _ManagedRaster: self.block_xbits = numpy.log2(self.block_xsize) self.block_ybits = numpy.log2(self.block_ysize) self.block_nx = ( - self.raster_x_size + (self.block_xsize) - 1) / self.block_xsize + self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize self.block_ny = ( - self.raster_y_size + (self.block_ysize) - 1) / self.block_ysize + self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS) self.raster_path = raster_path @@ -214,7 +212,7 @@ cdef class _ManagedRaster: if dirty_itr != self.dirty_blocks.end(): self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for # cached array @@ -278,7 +276,7 @@ cdef class _ManagedRaster: cdef void _load_block(self, int block_index) except *: cdef int block_xi = block_index % self.block_nx - cdef int block_yi = block_index / self.block_nx + cdef int block_yi = block_index // self.block_nx # we need the offsets to subtract from global indexes for cached array cdef int xoff = block_xi << self.block_xbits @@ -339,7 +337,7 @@ cdef class _ManagedRaster: self.dirty_blocks.erase(dirty_itr) block_xi = block_index % self.block_nx - block_yi = block_index / self.block_nx + block_yi = block_index // self.block_nx xoff = block_xi << self.block_xbits yoff = block_yi << self.block_ybits diff --git a/src/natcap/invest/utils.py b/src/natcap/invest/utils.py index f3a076d505..f34c25d9e5 100644 --- a/src/natcap/invest/utils.py +++ b/src/natcap/invest/utils.py @@ -10,6 +10,7 @@ import time from datetime import datetime +import natcap.invest import numpy import pandas import pygeoprocessing @@ -169,7 +170,7 @@ def prepare_workspace( LOGGER.info('Elapsed time: %s', _format_time(round(time.time() - start_time, 2))) logging.captureWarnings(False) - LOGGER.info('Execution finished') + LOGGER.info(f'Execution finished; version: {natcap.invest.__version__}') class ThreadFilter(logging.Filter): diff --git a/src/natcap/invest/validation.py b/src/natcap/invest/validation.py index 113fa7f304..01d22526bf 100644 --- a/src/natcap/invest/validation.py +++ b/src/natcap/invest/validation.py @@ -65,7 +65,9 @@ 'BBOX_NOT_INTERSECT': gettext( 'Not all of the spatial layers overlap each ' 'other. All bounding boxes must intersect: {bboxes}'), - 'NEED_PERMISSION': gettext( + 'NEED_PERMISSION_DIRECTORY': gettext( + 'You must have {permission} access to this directory'), + 'NEED_PERMISSION_FILE': gettext( 'You must have {permission} access to this file'), 'WRONG_GEOM_TYPE': gettext('Geometry type must be one of {allowed}') } @@ -164,9 +166,8 @@ def check_directory(dirpath, must_exist=True, permissions='rx', **kwargs): must_exist=True (bool): If ``True``, the directory at ``dirpath`` must already exist on the filesystem. permissions='rx' (string): A string that includes the lowercase - characters ``r``, ``w`` and/or ``x`` indicating required - permissions for this folder . See ``check_permissions`` for - details. + characters ``r``, ``w`` and/or ``x``, indicating read, write, and + execute permissions (respectively) required for this directory. Returns: A string error message if an error was found. ``None`` otherwise. @@ -191,9 +192,33 @@ def check_directory(dirpath, must_exist=True, permissions='rx', **kwargs): dirpath = parent break - permissions_warning = check_permissions(dirpath, permissions) - if permissions_warning: - return permissions_warning + MESSAGE_KEY = 'NEED_PERMISSION_DIRECTORY' + + if 'r' in permissions: + try: + os.scandir(dirpath).close() + except OSError: + return MESSAGES[MESSAGE_KEY].format(permission='read') + + # Check for x access before checking for w, + # since w operations to a dir are dependent on x access + if 'x' in permissions: + try: + cwd = os.getcwd() + os.chdir(dirpath) + except OSError: + return MESSAGES[MESSAGE_KEY].format(permission='execute') + finally: + os.chdir(cwd) + + if 'w' in permissions: + try: + temp_path = os.path.join(dirpath, 'temp__workspace_validation.txt') + with open(temp_path, 'w') as temp: + temp.close() + os.remove(temp_path) + except OSError: + return MESSAGES[MESSAGE_KEY].format(permission='write') def check_file(filepath, permissions='r', **kwargs): @@ -202,9 +227,8 @@ def check_file(filepath, permissions='r', **kwargs): Args: filepath (string): The filepath to validate. permissions='r' (string): A string that includes the lowercase - characters ``r``, ``w`` and/or ``x`` indicating required - permissions for this file. See ``check_permissions`` for - details. + characters ``r``, ``w`` and/or ``x``, indicating read, write, and + execute permissions (respectively) required for this file. Returns: A string error message if an error was found. ``None`` otherwise. @@ -213,33 +237,12 @@ def check_file(filepath, permissions='r', **kwargs): if not os.path.exists(filepath): return MESSAGES['FILE_NOT_FOUND'] - permissions_warning = check_permissions(filepath, permissions) - if permissions_warning: - return permissions_warning - - -def check_permissions(path, permissions): - """Validate permissions on a filesystem object. - - This function uses ``os.access`` to determine permissions access. - - Args: - path (string): The path to examine for permissions. - permissions (string): a string including the characters ``r``, ``w`` - and/or ``x`` (lowercase), indicating read, write, and execute - permissions (respectively) that the filesystem object at ``path`` - must have. - - Returns: - A string error message if an error was found. ``None`` otherwise. - - """ for letter, mode, descriptor in ( ('r', os.R_OK, 'read'), ('w', os.W_OK, 'write'), ('x', os.X_OK, 'execute')): - if letter in permissions and not os.access(path, mode): - return MESSAGES['NEED_PERMISSION'].format(permission=letter) + if letter in permissions and not os.access(filepath, mode): + return MESSAGES['NEED_PERMISSION_FILE'].format(permission=descriptor) def _check_projection(srs, projected, projection_units): @@ -424,6 +427,27 @@ def check_vector(filepath, geometries, fields=None, projected=False, return projection_warning +def check_raster_or_vector(filepath, **kwargs): + """Validate an input that may be a raster or vector. + + Args: + filepath (string): The path to the raster or vector. + **kwargs: kwargs of the raster and vector spec. Will be + passed to ``check_raster`` or ``check_vector``. + + Returns: + A string error message if an error was found. ``None`` otherwise. + """ + try: + gis_type = pygeoprocessing.get_gis_type(filepath) + except ValueError as err: + return str(err) + if gis_type == pygeoprocessing.RASTER_TYPE: + return check_raster(filepath, **kwargs) + else: + return check_vector(filepath, **kwargs) + + def check_freestyle_string(value, regexp=None, **kwargs): """Validate an arbitrary string. @@ -639,7 +663,7 @@ def get_validated_dataframe( for col in matching_cols: try: # frozenset needed to make the set hashable. A frozenset and set with the same members are equal. - if col_spec['type'] in {'csv', 'directory', 'file', 'raster', 'vector', frozenset({'vector', 'raster'})}: + if col_spec['type'] in {'csv', 'directory', 'file', 'raster', 'vector', frozenset({'raster', 'vector'})}: df[col] = df[col].apply( lambda p: p if pandas.isna(p) else utils.expand_path(str(p).strip(), csv_path)) df[col] = df[col].astype(pandas.StringDtype()) @@ -660,6 +684,20 @@ def get_validated_dataframe( f'Value(s) in the "{col}" column could not be interpreted ' f'as {col_spec["type"]}s. Original error: {err}') + col_type = col_spec['type'] + if isinstance(col_type, set): + col_type = frozenset(col_type) + if col_type in {'raster', 'vector', frozenset({'raster', 'vector'})}: + # recursively validate the files within the column + def check_value(value): + if pandas.isna(value): + return + err_msg = _VALIDATION_FUNCS[col_type](value, **col_spec) + if err_msg: + raise ValueError( + f'Error in {axis} "{col}", value "{value}": {err_msg}') + df[col].apply(check_value) + if any(df.columns.duplicated()): duplicated_columns = df.columns[df.columns.duplicated] return MESSAGES['DUPLICATE_HEADER'].format( @@ -881,6 +919,7 @@ def get_headers_to_validate(spec): 'option_string': check_option_string, 'raster': functools.partial(timeout, check_raster), 'vector': functools.partial(timeout, check_vector), + frozenset({'raster', 'vector'}): functools.partial(timeout, check_raster_or_vector), 'other': None, # Up to the user to define their validate() } @@ -965,13 +1004,16 @@ def validate(args, spec, spatial_overlap_opts=None): LOGGER.debug(f'Provided key {key} does not exist in MODEL_SPEC') continue + param_type = parameter_spec['type'] + if isinstance(param_type, set): + param_type = frozenset(param_type) # rewrite parameter_spec for any nested, conditional validity axis_keys = None - if parameter_spec['type'] == 'csv': + if param_type == 'csv': axis_keys = ['columns', 'rows'] - elif parameter_spec['type'] == 'vector': + elif param_type == 'vector' or 'vector' in param_type: axis_keys = ['fields'] - elif parameter_spec['type'] == 'directory': + elif param_type == 'directory': axis_keys = ['contents'] if axis_keys: @@ -985,7 +1027,7 @@ def validate(args, spec, spatial_overlap_opts=None): bool(_evaluate_expression( nested_spec['required'], expression_values))) - type_validation_func = _VALIDATION_FUNCS[parameter_spec['type']] + type_validation_func = _VALIDATION_FUNCS[param_type] if type_validation_func is None: # Validation for 'other' type must be performed by the user. @@ -1127,6 +1169,8 @@ def _wrapped_validate_func(args, limit_to=None): # need to validate it. if args_value not in ('', None): input_type = args_key_spec['type'] + if isinstance(input_type, set): + input_type = frozenset(input_type) validator_func = _VALIDATION_FUNCS[input_type] error_msg = validator_func(args_value, **args_key_spec) diff --git a/src/natcap/invest/wind_energy.py b/src/natcap/invest/wind_energy.py index e3947dbf09..252e4811db 100644 --- a/src/natcap/invest/wind_energy.py +++ b/src/natcap/invest/wind_energy.py @@ -1289,10 +1289,15 @@ def execute(args): levelized_raster_path = os.path.join( out_dir, 'levelized_cost_price_per_kWh%s.tif' % suffix) + # Include foundation_cost, discount_rate, number_of_turbines with + # parameters_dict to pass for NPV calculation + for key in ['foundation_cost', 'discount_rate', 'number_of_turbines']: + parameters_dict[key] = float(args[key]) + task_graph.add_task( func=_calculate_npv_levelized_rasters, args=(harvested_masked_path, final_dist_raster_path, npv_raster_path, - levelized_raster_path, parameters_dict, args, price_list), + levelized_raster_path, parameters_dict, price_list), target_path_list=[npv_raster_path, levelized_raster_path], task_name='calculate_npv_levelized_rasters', dependent_task_list=[final_dist_task]) @@ -1321,7 +1326,7 @@ def execute(args): def _calculate_npv_levelized_rasters( base_harvested_raster_path, base_dist_raster_path, target_npv_raster_path, target_levelized_raster_path, - parameters_dict, args, price_list): + parameters_dict, price_list): """Calculate NPV and levelized rasters from harvested and dist rasters. Args: @@ -1341,9 +1346,6 @@ def _calculate_npv_levelized_rasters( parameters_dict (dict): a dictionary of the turbine and biophysical global parameters. - args (dict): a dictionary that contains information on - ``foundation_cost``, ``discount_rate``, ``number_of_turbines``. - price_list (list): a list of wind energy prices for a period of time. @@ -1375,7 +1377,7 @@ def _calculate_npv_levelized_rasters( # The cost of infield cable in currency units per km infield_cost = parameters_dict['infield_cable_cost'] # The cost of the foundation in currency units - foundation_cost = args['foundation_cost'] + foundation_cost = parameters_dict['foundation_cost'] # The cost of each turbine unit in currency units unit_cost = parameters_dict['turbine_cost'] # The installation cost as a decimal @@ -1385,7 +1387,7 @@ def _calculate_npv_levelized_rasters( # The operations and maintenance costs as a decimal factor of capex_arr op_maint_cost = parameters_dict['operation_maintenance_cost'] # The discount rate as a decimal - discount_rate = args['discount_rate'] + discount_rate = parameters_dict['discount_rate'] # The cost to decommission the farm as a decimal factor of capex_arr decom = parameters_dict['decommission_cost'] # The mega watt value for the turbines in MW @@ -1401,16 +1403,15 @@ def _calculate_npv_levelized_rasters( # The total mega watt capacity of the wind farm where mega watt is the # turbines rated power - total_mega_watt = mega_watt * int(args['number_of_turbines']) + number_of_turbines = int(parameters_dict['number_of_turbines']) + total_mega_watt = mega_watt * number_of_turbines # Total infield cable cost - infield_cable_cost = infield_length * infield_cost * int( - args['number_of_turbines']) + infield_cable_cost = infield_length * infield_cost * number_of_turbines LOGGER.debug('infield_cable_cost : %s', infield_cable_cost) # Total foundation cost - total_foundation_cost = (foundation_cost + unit_cost) * int( - args['number_of_turbines']) + total_foundation_cost = (foundation_cost + unit_cost) * number_of_turbines LOGGER.debug('total_foundation_cost : %s', total_foundation_cost) # Nominal Capital Cost (CAP) minus the cost of cable which needs distances diff --git a/tests/test_coastal_blue_carbon.py b/tests/test_coastal_blue_carbon.py index 5af17b57ef..10c983144f 100644 --- a/tests/test_coastal_blue_carbon.py +++ b/tests/test_coastal_blue_carbon.py @@ -856,6 +856,20 @@ def test_validation(self): self.assertEqual([], validation_warnings) # Now work through the extra validation warnings. + # test validation: invalid analysis year + prior_snapshots = validation.get_validated_dataframe( + args['landcover_snapshot_csv'], + **coastal_blue_carbon.MODEL_SPEC['args']['landcover_snapshot_csv'] + )['raster_path'].to_dict() + baseline_year = min(prior_snapshots) + # analysis year must be >= the last transition year. + args['analysis_year'] = baseline_year + validation_warnings = coastal_blue_carbon.validate(args) + self.assertIn( + coastal_blue_carbon.INVALID_ANALYSIS_YEAR_MSG.format( + analysis_year=2000, latest_year=2020), + validation_warnings[0][1]) + # Create an invalid transitions table. invalid_raster_path = os.path.join(self.workspace_dir, 'invalid_raster.tif') @@ -863,11 +877,6 @@ def test_validation(self): raster.write('not a raster') # Write over the landcover snapshot CSV - prior_snapshots = validation.get_validated_dataframe( - args['landcover_snapshot_csv'], - **coastal_blue_carbon.MODEL_SPEC['args']['landcover_snapshot_csv'] - )['raster_path'].to_dict() - baseline_year = min(prior_snapshots) with open(args['landcover_snapshot_csv'], 'w') as snapshot_table: snapshot_table.write('snapshot_year,raster_path\n') snapshot_table.write( @@ -875,9 +884,6 @@ def test_validation(self): snapshot_table.write( f"{baseline_year + 10},{invalid_raster_path}") - # analysis year must be >= the last transition year. - args['analysis_year'] = baseline_year - # Write invalid entries to landcover transition table with open(args['landcover_transitions_table'], 'w') as transition_table: transition_table.write('lulc-class,Developed,Forest,Water\n') @@ -887,22 +893,16 @@ def test_validation(self): transition_options = [ 'accum', 'high-impact-disturb', 'med-impact-disturb', 'low-impact-disturb', 'ncc'] - validation_warnings = coastal_blue_carbon.validate(args) - self.assertEqual(len(validation_warnings), 3) + self.assertEqual(len(validation_warnings), 2) self.assertIn( - coastal_blue_carbon.INVALID_SNAPSHOT_RASTER_MSG.format( - snapshot_year=baseline_year + 10), + 'File could not be opened as a GDAL raster', validation_warnings[0][1]) - self.assertIn( - coastal_blue_carbon.INVALID_ANALYSIS_YEAR_MSG.format( - analysis_year=2000, latest_year=2010), - validation_warnings[1][1]) self.assertIn( coastal_blue_carbon.INVALID_TRANSITION_VALUES_MSG.format( model_transitions=transition_options, transition_values=['disturb', 'invalid']), - validation_warnings[2][1]) + validation_warnings[1][1]) def test_track_first_disturbance(self): """CBC: Track disturbances over time.""" @@ -1034,3 +1034,28 @@ def test_track_later_disturbance(self): expected_year_of_disturbance) finally: raster = None + + def test_validate_required_analysis_year(self): + """CBC: analysis year validation (regression test for #1534).""" + from natcap.invest.coastal_blue_carbon import coastal_blue_carbon + + args = TestCBC2._create_model_args(self.workspace_dir) + args['workspace_dir'] = self.workspace_dir + args['analysis_year'] = None + # truncate the CSV so that it has only one snapshot year + with open(args['landcover_snapshot_csv'], 'r') as file: + lines = file.readlines() + with open(args['landcover_snapshot_csv'], 'w') as file: + file.writelines(lines[:2]) + validation_warnings = coastal_blue_carbon.validate(args) + self.assertEqual( + validation_warnings, + [(['analysis_year'], coastal_blue_carbon.MISSING_ANALYSIS_YEAR_MSG)]) + + args['analysis_year'] = 2000 # set analysis year equal to snapshot year + validation_warnings = coastal_blue_carbon.validate(args) + self.assertEqual( + validation_warnings, + [(['analysis_year'], + coastal_blue_carbon.INVALID_ANALYSIS_YEAR_MSG.format( + analysis_year=2000, latest_year=2000))]) diff --git a/tests/test_coastal_vulnerability.py b/tests/test_coastal_vulnerability.py index 8d8a6a37d2..0b3be853a2 100644 --- a/tests/test_coastal_vulnerability.py +++ b/tests/test_coastal_vulnerability.py @@ -1550,21 +1550,6 @@ def test_clip_project_vector_on_invalid_geometry(self): n_actual_features = target_layer.GetFeatureCount() self.assertTrue(n_actual_features == n_features - 1) - def test_invalid_habitat_table_paths(self): - """CV: test ValueError raised on invalid filepaths inside table.""" - from natcap.invest import coastal_vulnerability - habitat_table_path = os.path.join(self.workspace_dir, 'bad_table.csv') - df = pandas.DataFrame( - columns=['id', 'path', 'rank', 'protection distance (m)'], - data=[['foo', 'bad_path.shp', 3, 400]]) - df.to_csv(habitat_table_path) - with self.assertRaises(ValueError) as cm: - coastal_vulnerability._validate_habitat_table_paths( - habitat_table_path) - actual_message = str(cm.exception) - expected_message = 'Could not open these datasets referenced in' - self.assertTrue(expected_message in actual_message) - def test_polygon_to_lines(self): """CV: test a helper function that converts polygons to linestrings.""" from natcap.invest import coastal_vulnerability diff --git a/tests/test_datastack_modules/archive_extraction.py b/tests/test_datastack_modules/archive_extraction.py index 8f48ce666e..36a95a5f24 100644 --- a/tests/test_datastack_modules/archive_extraction.py +++ b/tests/test_datastack_modules/archive_extraction.py @@ -14,7 +14,11 @@ 'type': 'csv', 'columns': { 'ID': {'type': 'integer'}, - 'path': {'type': {'raster', 'vector'}}, + 'path': { + 'type': {'raster', 'vector'}, + 'geometries': {'POINT', 'POLYGON'}, + 'bands': {1: {'type': 'number'}} + } } } } diff --git a/tests/test_habitat_quality.py b/tests/test_habitat_quality.py index 24f7a7a17e..0924dc76c8 100644 --- a/tests/test_habitat_quality.py +++ b/tests/test_habitat_quality.py @@ -957,8 +957,7 @@ def test_habitat_quality_bad_rasters(self): actual_message = str(cm.exception) self.assertIn( - 'There was an Error locating a threat raster from ' - 'the path in CSV for column: cur_path and threat: threat_1', + 'File could not be opened as a GDAL raster', actual_message) def test_habitat_quality_lulc_current_only(self): @@ -1392,14 +1391,7 @@ def test_habitat_quality_validation_bad_threat_path(self): self.assertTrue( validate_result, "expected failed validations instead didn't get any.") - self.assertEqual( - habitat_quality.MISSING_THREAT_RASTER_MSG.format( - threat_list=[ - ('threat_1', 'cur_path'), - ('threat_2', 'cur_path'), - ('threat_1', 'fut_path'), - ('threat_2', 'fut_path')]), - validate_result[0][1]) + self.assertIn('File not found', validate_result[0][1]) def test_habitat_quality_missing_cur_threat_path(self): """Habitat Quality: test for missing threat paths in current.""" @@ -1555,10 +1547,7 @@ def test_habitat_quality_misspelled_cur_threat_path(self): habitat_quality.execute(args) actual_message = str(cm.exception) - self.assertIn( - 'There was an Error locating a threat raster from ' - 'the path in CSV for column: cur_path and threat: threat_1', - actual_message) + self.assertIn('File not found', actual_message) def test_habitat_quality_validate_missing_cur_threat_path(self): """Habitat Quality: test validate for missing threat paths in cur.""" @@ -1666,59 +1655,6 @@ def test_habitat_quality_validate_missing_fut_threat_path(self): threat_list=[('threat_1', 'fut_path')]), validate_result[0][1]) - def test_habitat_quality_validate_misspelled_cur_threat_path(self): - """Habitat Quality: test validate for a misspelled cur threat path.""" - from natcap.invest import habitat_quality - - args = { - 'half_saturation_constant': '0.5', - 'results_suffix': 'regression', - 'workspace_dir': self.workspace_dir, - 'n_workers': -1, - } - - args['access_vector_path'] = os.path.join( - args['workspace_dir'], 'access_samp.shp') - make_access_shp(args['access_vector_path']) - - scenarios = ['_bas_', '_cur_', '_fut_'] - for lulc_val, scenario in enumerate(scenarios, start=1): - lulc_array = numpy.ones((100, 100), dtype=numpy.int8) - lulc_array[50:, :] = lulc_val - args['lulc' + scenario + 'path'] = os.path.join( - args['workspace_dir'], 'lc_samp' + scenario + 'b.tif') - make_raster_from_array( - lulc_array, args['lulc' + scenario + 'path']) - - args['sensitivity_table_path'] = os.path.join( - args['workspace_dir'], 'sensitivity_samp.csv') - make_sensitivity_samp_csv(args['sensitivity_table_path']) - - make_threats_raster( - args['workspace_dir'], threat_values=[1, 1], - dtype=numpy.int8, gdal_type=gdal.GDT_Int32) - - args['threats_table_path'] = os.path.join( - args['workspace_dir'], 'threats_samp.csv') - - with open(args['threats_table_path'], 'w') as open_table: - open_table.write( - 'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n') - open_table.write( - '0.04,0.7,threat_1,linear,,threat_1_cur.tif,threat_1_c.tif\n') - open_table.write( - '0.07,1.0,threat_2,exponential,,threat_2_c.tif,' - 'threat_2_f.tif\n') - - validate_result = habitat_quality.validate(args, limit_to=None) - self.assertTrue( - validate_result, - "expected failed validations instead didn't get any.") - self.assertEqual( - habitat_quality.MISSING_THREAT_RASTER_MSG.format( - threat_list=[('threat_1', 'cur_path')]), - validate_result[0][1], validate_result[0][1]) - def test_habitat_quality_validate_duplicate_threat_path(self): """Habitat Quality: test validate for duplicate threat paths.""" from natcap.invest import habitat_quality diff --git a/tests/test_hra.py b/tests/test_hra.py index 455704e989..f0581261c5 100644 --- a/tests/test_hra.py +++ b/tests/test_hra.py @@ -195,6 +195,17 @@ def test_info_table_parsing(self): """HRA: check info table parsing w/ case sensitivity.""" from natcap.invest import hra + corals_path = 'habitat/corals.shp' + oil_path = 'stressors/oil.shp' + transport_path = 'stressors/transport.shp' + geoms = [shapely.geometry.Point(ORIGIN).buffer(100)] + os.makedirs(os.path.join(self.workspace_dir, 'habitat')) + os.makedirs(os.path.join(self.workspace_dir, 'stressors')) + for path in [corals_path, oil_path, transport_path]: + pygeoprocessing.shapely_geometry_to_vector( + geoms, os.path.join(self.workspace_dir, path), + SRS_WKT, 'ESRI Shapefile') + info_table_path = os.path.join(self.workspace_dir, 'info_table.csv') with open(info_table_path, 'w') as info_table: info_table.write( @@ -202,13 +213,11 @@ def test_info_table_parsing(self): # This leading backslash is important for dedent to parse # the right number of leading spaces from the following # rows. - # The paths don't actually need to exist for this test - - # this function is merely parsing the table contents. - """\ + f"""\ NAME,PATH,TYPE,STRESSOR BUFFER (meters) - Corals,habitat/corals.shp,habitat, - Oil,stressors/oil.shp,stressor,1000 - Transportation,stressors/transport.shp,stressor,100""" + Corals,{corals_path},habitat, + Oil,{oil_path},stressor,1000 + Transportation,{transport_path},stressor,100""" )) habitats, stressors = hra._parse_info_table(info_table_path) @@ -236,6 +245,19 @@ def test_info_table_overlapping_habs_stressors(self): """HRA: error when info table has overlapping habitats, stressors.""" from natcap.invest import hra + corals_habitat_path = 'habitat/corals.shp' + oil_path = 'stressors/oil.shp' + corals_stressor_path = 'stressors/corals.shp' + transport_path = 'stressors/transport.shp' + geoms = [shapely.geometry.Point(ORIGIN).buffer(100)] + os.makedirs(os.path.join(self.workspace_dir, 'habitat')) + os.makedirs(os.path.join(self.workspace_dir, 'stressors')) + for path in [corals_habitat_path, oil_path, + corals_stressor_path, transport_path]: + pygeoprocessing.shapely_geometry_to_vector( + geoms, os.path.join(self.workspace_dir, path), + SRS_WKT, 'ESRI Shapefile') + info_table_path = os.path.join(self.workspace_dir, 'info_table.csv') with open(info_table_path, 'w') as info_table: info_table.write( @@ -245,12 +267,12 @@ def test_info_table_overlapping_habs_stressors(self): # rows. # The paths don't actually need to exist for this test - # this function is merely parsing the table contents. - """\ + f"""\ NAME,PATH,TYPE,STRESSOR BUFFER (meters) - corals,habitat/corals.shp,habitat, - oil,stressors/oil.shp,stressor,1000 - corals,stressors/corals.shp,stressor,1000 - transportation,stressors/transport.shp,stressor,100""" + corals,{corals_habitat_path},habitat, + oil,{oil_path},stressor,1000 + corals,{corals_stressor_path},stressor,1000 + transportation,{transport_path},stressor,100""" )) with self.assertRaises(ValueError) as cm: @@ -1328,14 +1350,16 @@ def test_model_habitat_mismatch(self): """HRA: check errors when habitats are mismatched.""" from natcap.invest import hra + eelgrass_conn_path = os.path.join( + self.workspace_dir, 'eelgrass_connectivity.shp') criteria_table_path = os.path.join(self.workspace_dir, 'criteria.csv') with open(criteria_table_path, 'w') as criteria_table: criteria_table.write(textwrap.dedent( - """\ + f"""\ HABITAT NAME,eelgrass,,,hardbottom,,,CRITERIA TYPE HABITAT RESILIENCE ATTRIBUTES,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C recruitment rate,2,2,2,2,2,2,C - connectivity rate,eelgrass_connectivity.shp,2,2,2,2,2,C + connectivity rate,{eelgrass_conn_path},2,2,2,2,2,C ,,,,,,, HABITAT STRESSOR OVERLAP PROPERTIES,,,,,,, oil,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C @@ -1348,20 +1372,26 @@ def test_model_habitat_mismatch(self): """ )) - eelgrass_path = os.path.join( - self.workspace_dir, 'eelgrass_connectivity.shp') + corals_path = 'habitat/corals.shp' + oil_path = 'stressors/oil.shp' + transport_path = 'stressors/transport.shp' geoms = [shapely.geometry.Point(ORIGIN).buffer(100)] - pygeoprocessing.shapely_geometry_to_vector( - geoms, eelgrass_path, SRS_WKT, 'ESRI Shapefile') + os.makedirs(os.path.join(self.workspace_dir, 'habitat')) + os.makedirs(os.path.join(self.workspace_dir, 'stressors')) + for path in [ + eelgrass_conn_path, corals_path, oil_path, transport_path]: + pygeoprocessing.shapely_geometry_to_vector( + geoms, os.path.join(self.workspace_dir, path), + SRS_WKT, 'ESRI Shapefile') info_table_path = os.path.join(self.workspace_dir, 'info.csv') with open(info_table_path, 'w') as info_table: info_table.write(textwrap.dedent( - """\ + f"""\ NAME,PATH,TYPE,STRESSOR BUFFER (meters) - corals,habitat/corals.shp,habitat, - oil,stressors/oil.shp,stressor,1000 - transportation,stressors/transport.shp,stressor,100""")) + corals,{corals_path},habitat, + oil,{oil_path},stressor,1000 + transportation,{transport_path},stressor,100""")) args = { 'workspace_dir': os.path.join(self.workspace_dir, 'workspace'), @@ -1393,14 +1423,15 @@ def test_model_stressor_mismatch(self): """HRA: check stressor mismatch.""" from natcap.invest import hra + eelgrass_conn_path = 'eelgrass_connectivity.shp' criteria_table_path = os.path.join(self.workspace_dir, 'criteria.csv') with open(criteria_table_path, 'w') as criteria_table: criteria_table.write(textwrap.dedent( - """\ + f"""\ HABITAT NAME,eelgrass,,,hardbottom,,,CRITERIA TYPE HABITAT RESILIENCE ATTRIBUTES,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C recruitment rate,2,2,2,2,2,2,C - connectivity rate,eelgrass_connectivity.shp,2,2,2,2,2,C + connectivity rate,{eelgrass_conn_path},2,2,2,2,2,C ,,,,,,, HABITAT STRESSOR OVERLAP PROPERTIES,,,,,,, oil,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C @@ -1413,21 +1444,29 @@ def test_model_stressor_mismatch(self): """ )) - eelgrass_path = os.path.join( - self.workspace_dir, 'eelgrass_connectivity.shp') + eelgrass_path = 'habitat/eelgrass.shp' + hardbottom_path = 'habitat/hardbottom.shp' + oil_path = 'stressors/oil.shp' + transport_path = 'stressors/transport.shp' geoms = [shapely.geometry.Point(ORIGIN).buffer(100)] - pygeoprocessing.shapely_geometry_to_vector( - geoms, eelgrass_path, SRS_WKT, 'ESRI Shapefile') + os.makedirs(os.path.join(self.workspace_dir, 'habitat')) + os.makedirs(os.path.join(self.workspace_dir, 'stressors')) + for path in [ + eelgrass_conn_path, eelgrass_path, hardbottom_path, + oil_path, transport_path]: + pygeoprocessing.shapely_geometry_to_vector( + geoms, os.path.join(self.workspace_dir, path), + SRS_WKT, 'ESRI Shapefile') info_table_path = os.path.join(self.workspace_dir, 'info.csv') with open(info_table_path, 'w') as info_table: info_table.write(textwrap.dedent( - """\ + f"""\ NAME,PATH,TYPE,STRESSOR BUFFER (meters) - eelgrass,habitat/eelgrass.shp,habitat, - hardbottom,habitat/hardbottom.shp,habitat, - oil,stressors/oil.shp,stressor,1000 - transportation,stressors/transport.shp,stressor,100""")) + eelgrass,{eelgrass_path},habitat, + hardbottom,{hardbottom_path},habitat, + oil,{oil_path},stressor,1000 + transportation,{transport_path},stressor,100""")) args = { 'workspace_dir': os.path.join(self.workspace_dir, 'workspace'), @@ -1448,7 +1487,6 @@ def test_model_stressor_mismatch(self): with self.assertRaises(ValueError) as cm: hra.execute(args) - self.assertIn('stressors', str(cm.exception)) self.assertIn("Missing from info table: fishing", str(cm.exception)) diff --git a/tests/test_ndr.py b/tests/test_ndr.py index d4e806f31f..5dc1bd456e 100644 --- a/tests/test_ndr.py +++ b/tests/test_ndr.py @@ -380,7 +380,7 @@ def test_masking_invalid_geometry(self): # bowtie geometry is invalid; verify we can still create a mask. coordinates = [] for pixel_x_offset, pixel_y_offset in [ - (0, 0), (0, 1), (1, 0), (1, 1), (0, 0)]: + (0, 0), (0, 1), (1, 0.25), (1, 0.75), (0, 0)]: coordinates.append(( default_origin[0] + default_pixel_size[0] * pixel_x_offset, default_origin[1] + default_pixel_size[1] * pixel_y_offset diff --git a/tests/test_recreation.py b/tests/test_recreation.py index 21771479d9..5992d55bc5 100644 --- a/tests/test_recreation.py +++ b/tests/test_recreation.py @@ -636,17 +636,6 @@ def tearDown(self): """Delete workspace.""" shutil.rmtree(self.workspace_dir) - def test_data_missing_in_predictors(self): - """Recreation can validate if predictor data missing.""" - from natcap.invest.recreation import recmodel_client - - response_vector_path = os.path.join(SAMPLE_DATA, 'andros_aoi.shp') - table_path = os.path.join( - SAMPLE_DATA, 'predictors_data_missing.csv') - - self.assertIsNotNone(recmodel_client._validate_same_projection( - response_vector_path, table_path)) - def test_data_different_projection(self): """Recreation can validate if data in different projection.""" from natcap.invest.recreation import recmodel_client @@ -654,21 +643,21 @@ def test_data_different_projection(self): response_vector_path = os.path.join(SAMPLE_DATA, 'andros_aoi.shp') table_path = os.path.join( SAMPLE_DATA, 'predictors_wrong_projection.csv') - - self.assertIsNotNone(recmodel_client._validate_same_projection( - response_vector_path, table_path)) + msg = recmodel_client._validate_same_projection( + response_vector_path, table_path) + self.assertIn('did not match the projection', msg) def test_different_tables(self): """Recreation can validate if scenario ids different than predictor.""" from natcap.invest.recreation import recmodel_client base_table_path = os.path.join( - SAMPLE_DATA, 'predictors_data_missing.csv') + SAMPLE_DATA, 'predictors_all.csv') scenario_table_path = os.path.join( - SAMPLE_DATA, 'predictors_wrong_projection.csv') - self.assertIsNotNone( - recmodel_client._validate_same_ids_and_types( - base_table_path, scenario_table_path)) + SAMPLE_DATA, 'predictors.csv') + msg = recmodel_client._validate_same_ids_and_types( + base_table_path, scenario_table_path) + self.assertIn('table pairs unequal', msg) def test_delay_op(self): """Recreation coverage of delay op function.""" diff --git a/tests/test_utils.py b/tests/test_utils.py index 4ebfdecab6..6025e699b6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1199,7 +1199,7 @@ def test_os_path_normalization_linux(self): relative_to = os.path.join(self.workspace_dir, 'test.csv') expected_path = os.path.join(self.workspace_dir, "foo/bar.shp") path = utils.expand_path(rel_path, relative_to) - self.assertEquals(path, expected_path) + self.assertEqual(path, expected_path) @unittest.skipIf(platform.system() != 'Windows', "Function behavior differs across systems.") @@ -1213,7 +1213,7 @@ def test_os_path_normalization_windows(self): relative_to = os.path.join(self.workspace_dir, 'test.csv') expected_path = os.path.join(self.workspace_dir, "foo\\bar.shp") path = utils.expand_path(rel_path, relative_to) - self.assertEquals(path, expected_path) + self.assertEqual(path, expected_path) def test_falsey(self): """Utils: test return None when falsey.""" diff --git a/tests/test_validation.py b/tests/test_validation.py index 9d6d0d45d9..f5ad094f50 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,11 +1,10 @@ """Testing module for validation.""" -import codecs -import collections import functools import os -import platform import shutil +import stat import string +import sys import tempfile import textwrap import time @@ -330,6 +329,91 @@ def test_workspace_not_exists(self): new_dir, must_exist=False, permissions='rwx')) +@unittest.skipIf( + sys.platform.startswith('win'), + 'requires support for os.chmod(), which is unreliable on Windows') +class DirectoryValidationMacOnly(unittest.TestCase): + """Test Directory Permissions Validation.""" + + def test_invalid_permissions_r(self): + """Validation: when a folder must have read/write/execute + permissions but is missing write and execute permissions.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IREAD) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='execute')) + + def test_invalid_permissions_w(self): + """Validation: when a folder must have read/write/execute + permissions but is missing read and execute permissions.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IWRITE) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='read')) + + def test_invalid_permissions_x(self): + """Validation: when a folder must have read/write/execute + permissions but is missing read and write permissions.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IEXEC) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='read')) + + def test_invalid_permissions_rw(self): + """Validation: when a folder must have read/write/execute + permissions but is missing execute permission.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IREAD | stat.S_IWRITE) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='execute')) + + def test_invalid_permissions_rx(self): + """Validation: when a folder must have read/write/execute + permissions but is missing write permission.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IREAD | stat.S_IEXEC) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='write')) + + def test_invalid_permissions_wx(self): + """Validation: when a folder must have read/write/execute + permissions but is missing read permission.""" + from natcap.invest import validation + + with tempfile.TemporaryDirectory() as tempdir: + os.chmod(tempdir, stat.S_IWRITE | stat.S_IEXEC) + validation_warning = validation.check_directory(tempdir, + permissions='rwx') + self.assertEqual( + validation_warning, + validation.MESSAGES['NEED_PERMISSION_DIRECTORY'].format(permission='read')) + + class FileValidation(unittest.TestCase): """Test File Validator.""" @@ -539,7 +623,6 @@ def test_vector_projected_in_m(self): def test_wrong_geom_type(self): """Validation: checks that the vector's geometry type is correct.""" - from natcap.invest import spec_utils from natcap.invest import validation driver = gdal.GetDriverByName('GPKG') filepath = os.path.join(self.workspace_dir, 'vector.gpkg') @@ -1098,7 +1181,7 @@ def test_integer_type_columns(self): csv_file, columns={ 'id': {'type': 'integer'}, - 'header': {'type': 'integer', 'na_allowed': True}}) + 'header': {'type': 'integer'}}) self.assertIsInstance(df['header'][0], numpy.int64) self.assertIsInstance(df['header'][1], numpy.int64) # empty values are returned as pandas.NA @@ -1113,7 +1196,7 @@ def test_float_type_columns(self): """\ h1,h2,h3 5,0.5,.4 - -1,-.3, + -1,.3, """ )) df = validation.get_validated_dataframe( @@ -1121,7 +1204,7 @@ def test_float_type_columns(self): columns={ 'h1': {'type': 'number'}, 'h2': {'type': 'ratio'}, - 'h3': {'type': 'percent', 'na_allowed': True}, + 'h3': {'type': 'percent'}, }) self.assertEqual(df['h1'].dtype, float) self.assertEqual(df['h2'].dtype, float) @@ -1145,7 +1228,7 @@ def test_string_type_columns(self): csv_file, columns={ 'h1': {'type': 'freestyle_string'}, - 'h2': {'type': 'option_string'}, + 'h2': {'type': 'option_string', 'options': ['a', 'b']}, 'h3': {'type': 'freestyle_string'}, }) self.assertEqual(df['h1'][0], '1') @@ -1170,7 +1253,7 @@ def test_boolean_type_columns(self): csv_file, columns={ 'index': {'type': 'freestyle_string'}, - 'h1': {'type': 'boolean', 'na_allowed': True}}) + 'h1': {'type': 'boolean'}}) self.assertEqual(df['h1'][0], True) self.assertEqual(df['h1'][1], False) # empty values are returned as pandas.NA @@ -1180,15 +1263,18 @@ def test_expand_path_columns(self): """validation: test values in path columns are expanded.""" from natcap.invest import validation csv_file = os.path.join(self.workspace_dir, 'csv.csv') + # create files so that validation will pass + open(os.path.join(self.workspace_dir, 'foo.txt'), 'w').close() + os.mkdir(os.path.join(self.workspace_dir, 'foo')) + open(os.path.join(self.workspace_dir, 'foo', 'bar.txt'), 'w').close() with open(csv_file, 'w') as file_obj: file_obj.write(textwrap.dedent( f"""\ bar,path 1,foo.txt 2,foo/bar.txt - 3,foo\\bar.txt - 4,{self.workspace_dir}/foo.txt - 5, + 3,{self.workspace_dir}/foo.txt + 4, """ )) df = validation.get_validated_dataframe( @@ -1203,23 +1289,11 @@ def test_expand_path_columns(self): self.assertEqual( f'{self.workspace_dir}{os.sep}foo{os.sep}bar.txt', df['path'][1]) - - # utils.expand_path() will convert Windows path separators to linux if - # we're on mac/linux - if platform.system() == 'Windows': - self.assertEqual( - f'{self.workspace_dir}{os.sep}foo\\bar.txt', - df['path'][2]) - else: - self.assertEqual( - f'{self.workspace_dir}{os.sep}foo/bar.txt', - df['path'][2]) - self.assertEqual( f'{self.workspace_dir}{os.sep}foo.txt', - df['path'][3]) + df['path'][2]) # empty values are returned as empty strings - self.assertTrue(pandas.isna(df['path'][4])) + self.assertTrue(pandas.isna(df['path'][3])) def test_other_kwarg(self): """validation: any other kwarg should be passed to pandas.read_csv""" @@ -1349,7 +1423,6 @@ def test_rows(self): 'row1': {'type': 'freestyle_string'}, 'row2': {'type': 'number'}, }) - print(df) # header should have no leading / trailing whitespace self.assertEqual(list(df.columns), ['row1', 'row2']) @@ -1359,6 +1432,125 @@ def test_rows(self): self.assertEqual(df['row2'][1], 3) self.assertEqual(df['row2'].dtype, float) + def test_csv_raster_validation_missing_file(self): + """validation: validate missing raster within csv column""" + from natcap.invest import validation + + csv_path = os.path.join(self.workspace_dir, 'csv.csv') + raster_path = os.path.join(self.workspace_dir, 'foo.tif') + + with open(csv_path, 'w') as file_obj: + file_obj.write('col1,col2\n') + file_obj.write(f'1,{raster_path}\n') + + with self.assertRaises(ValueError) as cm: + validation.get_validated_dataframe( + csv_path, + columns={ + 'col1': {'type': 'number'}, + 'col2': {'type': 'raster'} + }) + self.assertIn('File not found', str(cm.exception)) + + def test_csv_raster_validation_not_projected(self): + """validation: validate unprojected raster within csv column""" + from natcap.invest import validation + # create a non-linear projected raster and validate it + driver = gdal.GetDriverByName('GTiff') + csv_path = os.path.join(self.workspace_dir, 'csv.csv') + raster_path = os.path.join(self.workspace_dir, 'foo.tif') + raster = driver.Create(raster_path, 3, 3, 1, gdal.GDT_Int32) + wgs84_srs = osr.SpatialReference() + wgs84_srs.ImportFromEPSG(4326) + raster.SetProjection(wgs84_srs.ExportToWkt()) + raster = None + + with open(csv_path, 'w') as file_obj: + file_obj.write('col1,col2\n') + file_obj.write(f'1,{raster_path}\n') + + with self.assertRaises(ValueError) as cm: + validation.get_validated_dataframe( + csv_path, + columns={ + 'col1': {'type': 'number'}, + 'col2': {'type': 'raster', 'projected': True} + }) + self.assertIn('must be projected', str(cm.exception)) + + def test_csv_vector_validation_missing_field(self): + """validation: validate vector missing field in csv column""" + from natcap.invest import validation + import pygeoprocessing + from shapely.geometry import Point + + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + projection_wkt = srs.ExportToWkt() + csv_path = os.path.join(self.workspace_dir, 'csv.csv') + vector_path = os.path.join(self.workspace_dir, 'test.gpkg') + pygeoprocessing.shapely_geometry_to_vector( + [Point(0.0, 0.0)], vector_path, projection_wkt, 'GPKG', + fields={'b': ogr.OFTInteger}, + attribute_list=[{'b': 0}], + ogr_geom_type=ogr.wkbPoint) + + with open(csv_path, 'w') as file_obj: + file_obj.write('col1,col2\n') + file_obj.write(f'1,{vector_path}\n') + + with self.assertRaises(ValueError) as cm: + validation.get_validated_dataframe( + csv_path, + columns={ + 'col1': {'type': 'number'}, + 'col2': { + 'type': 'vector', + 'fields': { + 'a': {'type': 'integer'}, + 'b': {'type': 'integer'} + }, + 'geometries': ['POINT'] + } + }) + self.assertIn( + 'Expected the field "a" but did not find it', + str(cm.exception)) + + def test_csv_raster_or_vector_validation(self): + """validation: validate vector in raster-or-vector csv column""" + from natcap.invest import validation + import pygeoprocessing + from shapely.geometry import Point + + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + projection_wkt = srs.ExportToWkt() + csv_path = os.path.join(self.workspace_dir, 'csv.csv') + vector_path = os.path.join(self.workspace_dir, 'test.gpkg') + pygeoprocessing.shapely_geometry_to_vector( + [Point(0.0, 0.0)], vector_path, projection_wkt, 'GPKG', + ogr_geom_type=ogr.wkbPoint) + + with open(csv_path, 'w') as file_obj: + file_obj.write('col1,col2\n') + file_obj.write(f'1,{vector_path}\n') + + with self.assertRaises(ValueError) as cm: + validation.get_validated_dataframe( + csv_path, + columns={ + 'col1': {'type': 'number'}, + 'col2': { + 'type': {'raster', 'vector'}, + 'fields': {}, + 'geometries': ['POLYGON'] + } + }) + self.assertIn( + "Geometry type must be one of ['POLYGON']", + str(cm.exception)) + class TestValidationFromSpec(unittest.TestCase): """Test Validation From Spec.""" @@ -2002,6 +2194,7 @@ def test_spatial_overlap_error(self): layer = vector.CreateLayer('layer', vector_srs, ogr.wkbPoint) new_feature = ogr.Feature(layer.GetLayerDefn()) new_feature.SetGeometry(ogr.CreateGeometryFromWkt('POINT (1 1)')) + layer.CreateFeature(new_feature) new_feature = None layer = None diff --git a/tests/test_wind_energy.py b/tests/test_wind_energy.py index c2b732fb0e..209da2f814 100644 --- a/tests/test_wind_energy.py +++ b/tests/test_wind_energy.py @@ -338,13 +338,12 @@ def test_calculate_npv_levelized_rasters(self): 'air_density_coefficient': 1.19E-04, 'loss_parameter': 0.05, 'turbine_cost': 10000, - 'turbine_rated_pwr': 5 - } - args = { + 'turbine_rated_pwr': 5, 'foundation_cost': 1000000, 'discount_rate': 0.01, 'number_of_turbines': 10 } + price_list = [0.10, 0.10, 0.10, 0.10, 0.10] srs = osr.SpatialReference() @@ -381,7 +380,7 @@ def test_calculate_npv_levelized_rasters(self): wind_energy._calculate_npv_levelized_rasters( base_harvest_path, base_distance_path, target_npv_raster_path, target_levelized_raster_path, - val_parameters_dict, args, price_list) + val_parameters_dict, price_list) # Compare the results that were "eye" tested. desired_npv_array = numpy.array( @@ -428,8 +427,8 @@ def generate_base_args(workspace_dir): SAMPLE_DATA, 'global_wind_energy_parameters.csv'), 'turbine_parameters_path': os.path.join( SAMPLE_DATA, '3_6_turbine.csv'), - 'number_of_turbines': 80, - 'min_depth': 3, + 'number_of_turbines': '80', # pass str to test casting + 'min_depth': '3', # pass str to test casting 'max_depth': 180, 'n_workers': -1 } @@ -534,13 +533,13 @@ def test_val_gridpts_windprice(self): args['max_distance'] = 200000 args['valuation_container'] = True args['foundation_cost'] = 2000000 - args['discount_rate'] = 0.07 + args['discount_rate'] = '0.07' # pass str to test casting # Test that only grid points are provided in grid_points_path args['grid_points_path'] = os.path.join( SAMPLE_DATA, 'resampled_grid_pts.csv') args['price_table'] = False args['wind_price'] = 0.187 - args['rate_change'] = 0.2 + args['rate_change'] = '0.2' # pass str to test casting wind_energy.execute(args) diff --git a/workbench/src/main/ipcMainChannels.js b/workbench/src/main/ipcMainChannels.js index de7ca13c19..f908d7c33b 100644 --- a/workbench/src/main/ipcMainChannels.js +++ b/workbench/src/main/ipcMainChannels.js @@ -14,6 +14,7 @@ export const ipcMainChannels = { IS_FIRST_RUN: 'is-first-run', LOGGER: 'logger', OPEN_EXTERNAL_URL: 'open-external-url', + OPEN_PATH: 'open-path', OPEN_LOCAL_HTML: 'open-local-html', SET_SETTING: 'set-setting', SHOW_ITEM_IN_FOLDER: 'show-item-in-folder', diff --git a/workbench/src/main/setupDialogs.js b/workbench/src/main/setupDialogs.js index cecc003a90..3e79df45ba 100644 --- a/workbench/src/main/setupDialogs.js +++ b/workbench/src/main/setupDialogs.js @@ -26,4 +26,10 @@ export default function setupDialogs() { shell.showItemInFolder(filepath); } ); + + ipcMain.handle( + ipcMainChannels.OPEN_PATH, async (event, dirpath) => { + return await shell.openPath(dirpath); + } + ); } diff --git a/workbench/src/renderer/components/InvestTab/index.jsx b/workbench/src/renderer/components/InvestTab/index.jsx index 0c250a4e3a..7e26369c8b 100644 --- a/workbench/src/renderer/components/InvestTab/index.jsx +++ b/workbench/src/renderer/components/InvestTab/index.jsx @@ -7,6 +7,8 @@ import TabContainer from 'react-bootstrap/TabContainer'; import Nav from 'react-bootstrap/Nav'; import Row from 'react-bootstrap/Row'; import Col from 'react-bootstrap/Col'; +import Modal from 'react-bootstrap/Modal'; +import Button from 'react-bootstrap/Button'; import { MdKeyboardArrowRight, } from 'react-icons/md'; @@ -45,10 +47,6 @@ async function investGetSpec(modelName) { return undefined; } -function handleOpenWorkspace(logfile) { - ipcRenderer.send(ipcMainChannels.SHOW_ITEM_IN_FOLDER, logfile); -} - /** * Render an invest model setup form, log display, etc. * Manage launching of an invest model in a child process. @@ -64,6 +62,7 @@ class InvestTab extends React.Component { uiSpec: null, userTerminated: false, executeClicked: false, + showErrorModal: false, }; this.investExecute = this.investExecute.bind(this); @@ -71,6 +70,8 @@ class InvestTab extends React.Component { this.terminateInvestProcess = this.terminateInvestProcess.bind(this); this.investLogfileCallback = this.investLogfileCallback.bind(this); this.investExitCallback = this.investExitCallback.bind(this); + this.handleOpenWorkspace = this.handleOpenWorkspace.bind(this); + this.showErrorModal = this.showErrorModal.bind(this); } async componentDidMount() { @@ -154,6 +155,7 @@ class InvestTab extends React.Component { updateJobProperties(tabID, { argsValues: args, status: undefined, // in case of re-run, clear an old status + logfile: undefined, // in case of re-run where logfile may no longer exist, clear old logfile path }); ipcRenderer.send( @@ -186,6 +188,22 @@ class InvestTab extends React.Component { ); } + async handleOpenWorkspace(workspace_dir) { + if (workspace_dir) { + const error = await ipcRenderer.invoke(ipcMainChannels.OPEN_PATH, workspace_dir); + if (error) { + logger.error(`Error opening workspace (${workspace_dir}). ${error}`); + this.showErrorModal(true); + } + } + } + + showErrorModal(shouldShow) { + this.setState({ + showErrorModal: shouldShow, + }); + } + render() { const { activeTab, @@ -193,6 +211,7 @@ class InvestTab extends React.Component { argsSpec, uiSpec, executeClicked, + showErrorModal, } = this.state; const { status, @@ -213,88 +232,99 @@ class InvestTab extends React.Component { const sidebarFooterElementId = `sidebar-footer-${tabID}`; return ( - - - - -
-
- -
-
+ + + - { - status - ? ( - handleOpenWorkspace(logfile)} - terminateInvestProcess={this.terminateInvestProcess} - /> - ) - : null - } -
- - - - - + {t('Setup')} + + + + {t('Log')} + + + +
+
+ - - +
- - - - - - + { + status + ? ( + this.handleOpenWorkspace(argsValues?.workspace_dir)} + terminateInvestProcess={this.terminateInvestProcess} + /> + ) + : null + } +
+ + + + + + + + + + + + + + this.showErrorModal(false)} aria-labelledby="error-modal-title"> + + {t('Error opening workspace')} + + {t('Failed to open workspace directory. Make sure the directory exists and that you have write access to it.')} + + + + + ); } } diff --git a/workbench/src/renderer/components/SaveAsModal/index.jsx b/workbench/src/renderer/components/SaveAsModal/index.jsx index 40c9d35c4a..06707a846d 100644 --- a/workbench/src/renderer/components/SaveAsModal/index.jsx +++ b/workbench/src/renderer/components/SaveAsModal/index.jsx @@ -67,7 +67,10 @@ class SaveAsModal extends React.Component { } handleShow() { - this.setState({ show: true }); + this.setState({ + relativePaths: false, + show: true, + }); } handleChange(event) { diff --git a/workbench/src/renderer/components/SetupTab/ArgInput/index.jsx b/workbench/src/renderer/components/SetupTab/ArgInput/index.jsx index 5cf8294ac7..64516d8515 100644 --- a/workbench/src/renderer/components/SetupTab/ArgInput/index.jsx +++ b/workbench/src/renderer/components/SetupTab/ArgInput/index.jsx @@ -41,23 +41,28 @@ function filterSpatialOverlapFeedback(message, filepath) { function FormLabel(props) { const { - argkey, argname, required, units, + argkey, argname, argtype, required, units, } = props; + const userFriendlyArgType = parseArgType(argtype); + const optional = typeof required === 'boolean' && !required; + const includeComma = userFriendlyArgType && optional; + return ( - - {argname} - - - { - (typeof required === 'boolean' && !required) - ? ({i18n.t('optional')}) - : - } - {/* display units at the end of the arg name, if applicable */} - { (units && units !== 'unitless') ? ` (${units})` : } - + {argname} + { + (userFriendlyArgType || optional) && + + ( + {userFriendlyArgType} + {includeComma && ', '} + {optional && {i18n.t('optional')}} + ) + + } + {/* display units at the end of the arg name, if applicable */} + { (units && units !== 'unitless') ? ({units}) : } ); } @@ -65,6 +70,7 @@ function FormLabel(props) { FormLabel.propTypes = { argkey: PropTypes.string.isRequired, argname: PropTypes.string.isRequired, + argtype: PropTypes.string.isRequired, required: PropTypes.oneOfType( [PropTypes.string, PropTypes.bool] ), @@ -72,22 +78,7 @@ FormLabel.propTypes = { }; function Feedback(props) { - const { argkey, argtype, message } = props; - const { t } = useTranslation(); - const argTypeDisplayNames = { - boolean: t('boolean'), - integer: t('integer'), - csv: t('csv'), - directory: t('directory'), - file: t('file'), - freestyle_string: t('freestyle_string'), - number: t('number'), - option_string: t('option_string'), - percent: t('percent'), - raster: t('raster'), - ratio: t('ratio'), - vector: t('vector'), - }; + const { argkey, message } = props; return ( // d-block class is needed because of a bootstrap bug // https://github.com/twbs/bootstrap/issues/29439 @@ -96,13 +87,12 @@ function Feedback(props) { type="invalid" id={`${argkey}-feedback`} > - {`${argTypeDisplayNames[argtype]} : ${(message)}`} + {message} ); } Feedback.propTypes = { argkey: PropTypes.string.isRequired, - argtype: PropTypes.string.isRequired, message: PropTypes.string, }; Feedback.defaultProps = { @@ -142,6 +132,30 @@ function dragLeavingHandler(event) { event.currentTarget.classList.remove('input-dragging'); } +function parseArgType(argtype) { + const { t, i18n } = useTranslation(); + // These types benefit from more descriptive placeholder text. + let userFriendlyArgType; + switch (argtype) { + case 'freestyle_string': + userFriendlyArgType = t('text'); + break; + case 'percent': + userFriendlyArgType = t('percent: a number from 0 - 100'); + break; + case 'ratio': + userFriendlyArgType = t('ratio: a decimal from 0 - 1'); + break; + case 'boolean': + case 'option_string': + userFriendlyArgType = ''; + break; + default: + userFriendlyArgType = t(argtype); + } + return userFriendlyArgType; +} + export default function ArgInput(props) { const inputRef = useRef(); @@ -162,24 +176,6 @@ export default function ArgInput(props) { } = props; let { validationMessage } = props; - const { t } = useTranslation(); - - // Some types benefit from more descriptive placeholder text. - const argTypeDisplayNames = { - boolean: t('boolean'), - integer: t('integer'), - csv: t('csv'), - directory: t('directory'), - file: t('file'), - freestyle_string: t('text'), - number: t('number'), - option_string: t('option_string'), - percent: t('percent: a number from 0 - 100'), - raster: t('raster'), - ratio: t('ratio: a decimal from 0 - 1'), - vector: t('vector'), - }; - // Occasionaly we want to force a scroll to the end of input fields // so that the most important part of a filepath is visible. // scrollEventCount changes on drop events and on use of the browse button. @@ -237,6 +233,9 @@ export default function ArgInput(props) { ); } + // These types benefit from more descriptive placeholder text. + const placeholderText = parseArgType(argSpec.type); + let form; if (argSpec.type === 'boolean') { form = ( @@ -261,6 +260,8 @@ export default function ArgInput(props) { onChange={handleChange} onFocus={handleChange} disabled={!enabled} + isValid={enabled && isValid} + custom > { Array.isArray(dropdownOptions) ? @@ -281,7 +282,7 @@ export default function ArgInput(props) { id={argkey} name={argkey} type="text" - placeholder={argTypeDisplayNames[argSpec.type]} + placeholder={placeholderText} value={value || ''} // empty string is handled better than `undefined` onChange={handleChange} onFocus={handleFocus} @@ -293,6 +294,7 @@ export default function ArgInput(props) { onDragOver={dragOverHandler} onDragEnter={dragEnterHandler} onDragLeave={dragLeavingHandler} + aria-describedby={`${argkey}-feedback`} /> {fileSelector} @@ -309,6 +311,7 @@ export default function ArgInput(props) { diff --git a/workbench/src/renderer/styles/style.css b/workbench/src/renderer/styles/style.css index 523d9ceb17..a335cabd6f 100644 --- a/workbench/src/renderer/styles/style.css +++ b/workbench/src/renderer/styles/style.css @@ -483,23 +483,33 @@ exceed 100% of window.*/ } .args-form .form-group { - align-items: center; + align-items: flex-start; } -#argname { +.argname { text-transform: capitalize; font-weight: 600; } -.args-form .form-control { +.args-form .form-control, +.custom-select { font-family: monospace; - font-size:1.3rem; + font-size: 1.3rem; +} + +.args-form { + --form-field-padding-right: 2em; +} + +.args-form .form-label { + padding-top: 0; + padding-bottom: 0; } .args-form .form-control[type=text] { /*always hold space for a validation mark so the rightmost text is never hidden by the mark when it appears.*/ - padding-right: 2em; + padding-right: var(--form-field-padding-right); } input.input-dragging { @@ -522,14 +532,24 @@ input[type=text]::placeholder { font-size: 0.9rem; font-family: monospace; white-space: pre-wrap; - padding-left: 3rem; - text-indent: -3rem; + padding-left: calc(3.5rem + 2px); } .args-form svg { font-size: 1.5rem; } +.custom-select, +.custom-select.is-valid { + --caret-width: 1.875rem; + padding-right: var(--form-field-padding-right); + /* Custom dropdown icon is react-icons/md/MdKeyboardArrowDown, as a URL-encoded SVG */ + background-image: url('data:image/svg+xml,%3Csvg xmlns="http://www.w3.org/2000/svg" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" height="200px" width="200px"%3E%3Cpath fill="none" d="M0 0h24v24H0V0z"%3E%3C/path%3E%3Cpath d="M7.41 8.59 12 13.17l4.59-4.58L18 10l-6 6-6-6 1.41-1.41z"%3E%3C/path%3E%3C/svg%3E'); + background-repeat: no-repeat; + background-position: center right calc((var(--form-field-padding-right) - var(--caret-width)) / 2); + background-size: var(--caret-width); +} + /* InVEST Log Tab */ #log-display { overflow: auto; diff --git a/workbench/tests/binary_tests/puppet.test.js b/workbench/tests/binary_tests/puppet.test.js index 1a6175a09e..54d8d74945 100644 --- a/workbench/tests/binary_tests/puppet.test.js +++ b/workbench/tests/binary_tests/puppet.test.js @@ -38,7 +38,7 @@ const TMP_AOI_PATH = path.join(TMP_DIR, 'aoi.geojson'); if (process.platform === 'darwin') { // https://github.com/electron-userland/electron-builder/issues/2724#issuecomment-375850150 - [BINARY_PATH] = glob.sync('./dist/mac/*.app/Contents/MacOS/InVEST*'); + [BINARY_PATH] = glob.sync('./dist/mac*/*.app/Contents/MacOS/InVEST*'); SCREENSHOT_PREFIX = path.join( os.homedir(), 'Library/Logs', pkg.name, 'invest-workbench-' ); @@ -183,16 +183,16 @@ test('Run a real invest model', async () => { const argsForm = await page.waitForSelector('.args-form'); const typeDelay = 10; const workspace = await argsForm.waitForSelector( - 'aria/[name="Workspace"][role="textbox"]'); + 'aria/[name="Workspace (directory)"][role="textbox"]'); await workspace.type(TMP_DIR, { delay: typeDelay }); const aoi = await argsForm.waitForSelector( - 'aria/[name="Area Of Interest"][role="textbox"]'); + 'aria/[name="Area Of Interest (vector)"][role="textbox"]'); await aoi.type(TMP_AOI_PATH, { delay: typeDelay }); const startYear = await argsForm.waitForSelector( - 'aria/[name="Start Year"][role="textbox"]'); + 'aria/[name="Start Year (number)"][role="textbox"]'); await startYear.type('2008', { delay: typeDelay }); const endYear = await argsForm.waitForSelector( - 'aria/[name="End Year"][role="textbox"]'); + 'aria/[name="End Year (number)"][role="textbox"]'); await endYear.type('2012', { delay: typeDelay }); await page.screenshot({ path: `${SCREENSHOT_PREFIX}4-complete-setup-form.png` }); diff --git a/workbench/tests/main/main.test.js b/workbench/tests/main/main.test.js index a65e806ff6..2b479b6882 100644 --- a/workbench/tests/main/main.test.js +++ b/workbench/tests/main/main.test.js @@ -187,6 +187,7 @@ describe('createWindow', () => { ipcMainChannels.GET_N_CPUS, ipcMainChannels.INVEST_VERSION, ipcMainChannels.IS_FIRST_RUN, + ipcMainChannels.OPEN_PATH, ipcMainChannels.SHOW_OPEN_DIALOG, ipcMainChannels.SHOW_SAVE_DIALOG, ]; diff --git a/workbench/tests/renderer/app.test.js b/workbench/tests/renderer/app.test.js index 60e5f6c894..558379ea38 100644 --- a/workbench/tests/renderer/app.test.js +++ b/workbench/tests/renderer/app.test.js @@ -122,7 +122,7 @@ describe('Various ways to open and close InVEST models', () => { expect(setupTab.classList.contains('active')).toBeTruthy(); // Expect some arg values that were loaded from the saved job: - const input = await findByLabelText(SAMPLE_SPEC.args.workspace_dir.name); + const input = await findByLabelText((content) => content.startsWith(SAMPLE_SPEC.args.workspace_dir.name)); expect(input).toHaveValue( argsValues.workspace_dir ); @@ -155,7 +155,7 @@ describe('Various ways to open and close InVEST models', () => { expect(executeButton).toBeDisabled(); const setupTab = await findByText('Setup'); const input = await findByLabelText( - SAMPLE_SPEC.args.carbon_pools_path.name + (content) => content.startsWith(SAMPLE_SPEC.args.carbon_pools_path.name) ); expect(setupTab.classList.contains('active')).toBeTruthy(); expect(input).toHaveValue(mockDatastack.args.carbon_pools_path); diff --git a/workbench/tests/renderer/downloadmodal.test.js b/workbench/tests/renderer/downloadmodal.test.js index 99c5014a23..3543ecfc45 100644 --- a/workbench/tests/renderer/downloadmodal.test.js +++ b/workbench/tests/renderer/downloadmodal.test.js @@ -41,7 +41,7 @@ describe('Sample Data Download Form', () => { }); test('Modal does not display when app has been run before', async () => { - const { findByText, queryByText } = render(); + const { findByText, queryByText } = render(); await findByText('InVEST'); // wait for page to load before querying const modalTitle = queryByText('Download InVEST sample data'); expect(modalTitle).toBeNull(); diff --git a/workbench/tests/renderer/invest_subprocess.test.js b/workbench/tests/renderer/invest_subprocess.test.js index 9c12e18600..56030999af 100644 --- a/workbench/tests/renderer/invest_subprocess.test.js +++ b/workbench/tests/renderer/invest_subprocess.test.js @@ -145,7 +145,7 @@ describe('InVEST subprocess testing', () => { ); await userEvent.click(carbon); const workspaceInput = await findByLabelText( - `${spec.args.workspace_dir.name}` + (content) => content.startsWith(spec.args.workspace_dir.name) ); await userEvent.type(workspaceInput, fakeWorkspace); const execute = await findByRole('button', { name: /Run/ }); @@ -195,7 +195,7 @@ describe('InVEST subprocess testing', () => { ); await userEvent.click(carbon); const workspaceInput = await findByLabelText( - `${spec.args.workspace_dir.name}` + (content) => content.startsWith(spec.args.workspace_dir.name) ); await userEvent.type(workspaceInput, fakeWorkspace); @@ -250,7 +250,7 @@ describe('InVEST subprocess testing', () => { ); await userEvent.click(carbon); const workspaceInput = await findByLabelText( - `${spec.args.workspace_dir.name}` + (content) => content.startsWith(spec.args.workspace_dir.name) ); await userEvent.type(workspaceInput, fakeWorkspace); @@ -301,7 +301,7 @@ describe('InVEST subprocess testing', () => { ); await userEvent.click(carbon); const workspaceInput = await findByLabelText( - `${spec.args.workspace_dir.name}` + (content) => content.startsWith(spec.args.workspace_dir.name) ); await userEvent.type(workspaceInput, fakeWorkspace); diff --git a/workbench/tests/renderer/investtab.test.js b/workbench/tests/renderer/investtab.test.js index 52b6225f20..2a125fbddc 100644 --- a/workbench/tests/renderer/investtab.test.js +++ b/workbench/tests/renderer/investtab.test.js @@ -118,9 +118,71 @@ describe('Run status Alert renders with status from a recent run', () => { }); const { findByRole } = renderInvestTab(job); - const openWorkspace = await findByRole('button', { name: 'Open Workspace' }) - openWorkspace.click(); - expect(shell.showItemInFolder).toHaveBeenCalledTimes(1); + const openWorkspaceBtn = await findByRole('button', { name: 'Open Workspace' }); + expect(openWorkspaceBtn).toBeTruthy(); + }); +}); + +describe('Open Workspace button', () => { + const spec = { + args: {}, + }; + + const baseJob = { + ...DEFAULT_JOB, + status: 'success', + }; + + beforeEach(() => { + getSpec.mockResolvedValue(spec); + fetchValidation.mockResolvedValue([]); + uiConfig.UI_SPEC = mockUISpec(spec); + setupDialogs(); + }); + + afterEach(() => { + removeIpcMainListeners(); + }); + + test('should open workspace', async () => { + const job = { + ...baseJob, + argsValues: { + workspace_dir: '/workspace', + }, + }; + + jest.spyOn(ipcRenderer, 'invoke'); + + const { findByRole } = renderInvestTab(job); + const openWorkspaceBtn = await findByRole('button', { name: 'Open Workspace' }) + openWorkspaceBtn.click(); + + expect(ipcRenderer.invoke).toHaveBeenCalledTimes(1); + expect(ipcRenderer.invoke).toHaveBeenCalledWith(ipcMainChannels.OPEN_PATH, job.argsValues.workspace_dir); + }); + + test('should present an error message to the user if workspace cannot be opened (e.g., if it does not exist)', async () => { + const job = { + ...baseJob, + status: 'error', + argsValues: { + workspace_dir: '/nonexistent-workspace', + }, + }; + + jest.spyOn(ipcRenderer, 'invoke'); + ipcRenderer.invoke.mockResolvedValue('Error opening workspace'); + + const { findByRole } = renderInvestTab(job); + const openWorkspaceBtn = await findByRole('button', { name: 'Open Workspace' }); + openWorkspaceBtn.click(); + + expect(ipcRenderer.invoke).toHaveBeenCalledTimes(1); + expect(ipcRenderer.invoke).toHaveBeenCalledWith(ipcMainChannels.OPEN_PATH, job.argsValues.workspace_dir); + + const errorModal = await findByRole('dialog', { name: 'Error opening workspace'}); + expect(errorModal).toBeTruthy(); }); }); @@ -299,9 +361,9 @@ describe('Sidebar Buttons', () => { const setupTab = await findByRole('tab', { name: 'Setup' }); expect(setupTab.classList.contains('active')).toBeTruthy(); - const input1 = await findByLabelText(spec.args.workspace.name); + const input1 = await findByLabelText((content) => content.startsWith(spec.args.workspace.name)); expect(input1).toHaveValue(mockDatastack.args.workspace); - const input2 = await findByLabelText(spec.args.port.name); + const input2 = await findByLabelText((content) => content.startsWith(spec.args.port.name)); expect(input2).toHaveValue(mockDatastack.args.port); }); @@ -427,8 +489,8 @@ describe('InVEST Run Button', () => { const runButton = await findByRole('button', { name: /Run/ }); expect(runButton).toBeDisabled(); - const a = await findByLabelText(`${spec.args.a.name}`); - const b = await findByLabelText(`${spec.args.b.name}`); + const a = await findByLabelText((content) => content.startsWith(spec.args.a.name)); + const b = await findByLabelText((content) => content.startsWith(spec.args.b.name)); expect(a).toHaveClass('is-invalid'); expect(b).toHaveClass('is-invalid'); diff --git a/workbench/tests/renderer/setuptab.test.js b/workbench/tests/renderer/setuptab.test.js index 4806ad0b2e..22ab595097 100644 --- a/workbench/tests/renderer/setuptab.test.js +++ b/workbench/tests/renderer/setuptab.test.js @@ -114,14 +114,14 @@ describe('Arguments form input types', () => { ])('render a text input for a %s', async (type) => { const spec = baseArgsSpec(type); const { findByLabelText } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(RegExp(`^${spec.args.arg.name}$`)); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); expect(input).toHaveAttribute('type', 'text'); }); test('render a text input with unit label for a number', async () => { const spec = baseArgsSpec('number'); const { findByLabelText } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name} (${spec.args.arg.units})`); + const input = await findByLabelText(`${spec.args.arg.name} (number) (${spec.args.arg.units})`); expect(input).toHaveAttribute('type', 'text'); }); @@ -174,7 +174,7 @@ describe('Arguments form input types', () => { }; const { findByLabelText, queryByText } = renderSetupFromSpec(spec, UI_SPEC, initArgs); - const input = await findByLabelText(`${spec.args.arg.name} (${spec.args.arg.units})`); + const input = await findByLabelText(`${spec.args.arg.name} (number) (${spec.args.arg.units})`); await waitFor(() => expect(input).toHaveValue(displayedValue)); expect(queryByText(missingValue)).toBeNull(); }); @@ -198,7 +198,7 @@ describe('Arguments form interactions', () => { findByRole, findByLabelText, } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name}`); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); expect(input).toHaveAttribute('type', 'text'); expect(await findByRole('button', { name: /browse for/ })) .toBeInTheDocument(); @@ -234,7 +234,7 @@ describe('Arguments form interactions', () => { // Click on a target element nested within the button to make // sure the handler still works correctly. await userEvent.click(btn.querySelector('svg')); - expect(await findByLabelText(`${spec.args.arg.name}`)) + expect(await findByLabelText((content) => content.startsWith(spec.args.arg.name))) .toHaveValue(filepath); }); @@ -245,7 +245,7 @@ describe('Arguments form interactions', () => { findByText, findByLabelText, queryByText, } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name}`); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); // A required input with no value is invalid (red X), but // feedback does not display until the input has been touched. @@ -274,7 +274,7 @@ describe('Arguments form interactions', () => { spec.args.arg.required = true; const { findByLabelText } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name}`); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); spy.mockClear(); // it was already called once on render // Fast typing, expect only 1 validation call @@ -290,7 +290,7 @@ describe('Arguments form interactions', () => { spec.args.arg.required = true; const { findByLabelText } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name}`); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); spy.mockClear(); // it was already called once on render // Slow typing, expect validation call after each character @@ -308,7 +308,7 @@ describe('Arguments form interactions', () => { findByText, findByLabelText, queryByText, } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(spec.args.arg.name); + const input = await findByLabelText((content) => content.startsWith(spec.args.arg.name)); expect(input).toHaveClass('is-invalid'); expect(queryByText(RegExp(VALIDATION_MESSAGE))).toBeNull(); @@ -326,7 +326,7 @@ describe('Arguments form interactions', () => { fetchValidation.mockResolvedValue([]); const { findByLabelText } = renderSetupFromSpec(spec, UI_SPEC); - const input = await findByLabelText(`${spec.args.arg.name} (optional)`); + const input = await findByLabelText((content) => content.includes('optional')); // An optional input with no value is valid, but green check // does not display until the input has been touched. @@ -400,10 +400,10 @@ describe('UI spec functionality', () => { }; const { findByLabelText } = renderSetupFromSpec(spec, uiSpec); - const arg1 = await findByLabelText(`${spec.args.arg1.name}`); - const arg2 = await findByLabelText(`${spec.args.arg2.name}`); - const arg3 = await findByLabelText(`${spec.args.arg3.name}`); - const arg4 = await findByLabelText(`${spec.args.arg4.name}`); + const arg1 = await findByLabelText((content) => content.startsWith(spec.args.arg1.name)); + const arg2 = await findByLabelText((content) => content.startsWith(spec.args.arg2.name)); + const arg3 = await findByLabelText((content) => content.startsWith(spec.args.arg3.name)); + const arg4 = await findByLabelText((content) => content.startsWith(spec.args.arg4.name)); await waitFor(() => { // Boolean Radios should default to "false" when a spec is loaded, @@ -466,7 +466,7 @@ describe('UI spec functionality', () => { const { findByLabelText, findByText, queryByText, } = renderSetupFromSpec(spec, uiSpec); - const arg1 = await findByLabelText(`${spec.args.arg1.name}`); + const arg1 = await findByLabelText((content) => content.startsWith(spec.args.arg1.name)); let option = await queryByText('Field1'); expect(option).toBeNull(); @@ -600,7 +600,7 @@ describe('Misc form validation stuff', () => { fetchValidation.mockResolvedValue([[Object.keys(spec.args), message]]); const { findByLabelText } = renderSetupFromSpec(spec, uiSpec); - const vectorInput = await findByLabelText(spec.args.vector.name); + const vectorInput = await findByLabelText((content) => content.startsWith(spec.args.vector.name)); const rasterInput = await findByLabelText(RegExp(`^${spec.args.raster.name}`)); await userEvent.type(vectorInput, vectorValue); await userEvent.type(rasterInput, rasterValue); @@ -682,9 +682,9 @@ describe('Form drag-and-drop', () => { }); fireEvent(setupForm, fileDropEvent); - expect(await findByLabelText(`${spec.args.arg1.name}`)) + expect(await findByLabelText((content) => content.startsWith(spec.args.arg1.name))) .toHaveValue(mockDatastack.args.arg1); - expect(await findByLabelText(`${spec.args.arg2.name}`)) + expect(await findByLabelText((content) => content.startsWith(spec.args.arg2.name))) .toHaveValue(mockDatastack.args.arg2); }); @@ -743,9 +743,9 @@ describe('Form drag-and-drop', () => { }); fireEvent(setupForm, fileDropEvent); - expect(await findByLabelText(`${spec.args.arg1.name}`)) + expect(await findByLabelText((content) => content.startsWith(spec.args.arg1.name))) .toHaveValue(mockDatastack.args.arg1); - expect(await findByLabelText(`${spec.args.arg2.name}`)) + expect(await findByLabelText((content) => content.startsWith(spec.args.arg2.name))) .toHaveValue(mockDatastack.args.arg2); expect(setupForm).not.toHaveClass('dragging'); }); @@ -817,7 +817,7 @@ describe('Form drag-and-drop', () => { findByLabelText, findByTestId, } = renderSetupFromSpec(spec, uiSpec); const setupForm = await findByTestId('setup-form'); - const setupInput = await findByLabelText(`${spec.args.arg1.name}`); + const setupInput = await findByLabelText((content) => content.startsWith(spec.args.arg1.name)); const fileDragEvent = createEvent.dragEnter(setupInput); // `dataTransfer.files` normally returns a `FileList` object. Since we are @@ -867,7 +867,7 @@ describe('Form drag-and-drop', () => { ); const { findByLabelText } = renderSetupFromSpec(spec, uiSpec); - const setupInput = await findByLabelText(`${spec.args.arg1.name}`); + const setupInput = await findByLabelText((content) => content.startsWith(spec.args.arg1.name)); const fileDragEnterEvent = createEvent.dragEnter(setupInput); // `dataTransfer.files` normally returns a `FileList` object. Since we are @@ -920,7 +920,7 @@ describe('Form drag-and-drop', () => { ); const { findByLabelText } = renderSetupFromSpec(spec, uiSpec); - const setupInput = await findByLabelText(`${spec.args.arg2.name}`); + const setupInput = await findByLabelText((content) => content.startsWith(spec.args.arg2.name)); const fileDragEnterEvent = createEvent.dragEnter(setupInput); // `dataTransfer.files` normally returns a `FileList` object. Since we are