diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml new file mode 100644 index 0000000000..fa6ab4f6ef --- /dev/null +++ b/.github/actions/build-docs/action.yml @@ -0,0 +1,11 @@ +name: Build Sphinx docs +description: Requires the setup-idaes action to be run before this +# TODO add options as inputs as needed +runs: + using: "composite" + steps: + - name: Build Sphinx docs (HTML) + shell: bash + run: | + cd docs/ + python build.py diff --git a/.github/actions/pylint/action.yml b/.github/actions/pylint/action.yml new file mode 100644 index 0000000000..a047dc52ed --- /dev/null +++ b/.github/actions/pylint/action.yml @@ -0,0 +1,19 @@ +name: Run pylint +description: Run static analysis using pylint +# TODO add inputs (options) as needed +runs: + using: "composite" + steps: + - name: Install pylint dependencies + shell: bash + run: | + PIP_INSTALL="pip --no-cache-dir install --progress-bar off" + $PIP_INSTALL --upgrade pip setuptools wheel + # TODO the pylint version will have to be pinned + # TODO installing idaes is necessary in our case because we import some idaes code in pylint plugins + $PIP_INSTALL pylint -r requirements.txt + # don't think we need to install the extensions, though + - name: Run pylint (errors only) + shell: bash + run: | + pylint -E --ignore-patterns="test_.*" idaes || true diff --git a/.github/actions/setup-idaes/action.yml b/.github/actions/setup-idaes/action.yml new file mode 100644 index 0000000000..aa4d045855 --- /dev/null +++ b/.github/actions/setup-idaes/action.yml @@ -0,0 +1,36 @@ +name: Set up IDAES +description: Install IDAES and extensions +inputs: + install-target: + description: 'Command-line arguments and options to pass to the install command, e.g. pip install' + required: true + install-command: + description: 'Command to use to install `install-target`' + required: false + default: pip --no-cache-dir install --progress-bar off +runs: + using: "composite" + steps: + - name: Update pip and other packaging tools + shell: bash + run: | + ${{ inputs.install-command }} pip setuptools wheel + - name: Install idaes and dependencies + shell: bash + run: | + ${{ inputs.install-command }} ${{ inputs.install-target}} + idaes --version + - name: Install extensions + shell: bash + run: | + idaes get-extensions --verbose + # add bin directory to $PATH (only valid for subsequent steps) + echo "$(idaes bin-directory)" >> $GITHUB_PATH + - name: Test access to executables + shell: bash + run: | + if [ "$RUNNER_OS" == "Windows" ]; then + ipopt.exe -v + else + ipopt -v + fi diff --git a/.github/scripts/countApproved.js b/.github/scripts/countApproved.js new file mode 100644 index 0000000000..ac3c40d727 --- /dev/null +++ b/.github/scripts/countApproved.js @@ -0,0 +1,32 @@ +const REVIEW_STATE = { + dismissed: "DISMISSED", + approved: "APPROVED", +}; + +function selectLatestPerUser(reviews) { + const latestByUser = {}; + reviews.forEach((r) => { + // the reviews are in chronological order (earliest to latest) + // so to get the latest for each user we can use an Object as a map and loop over all reviews + // at each iteration, a more recent review for that user will replace an earlier one set before it + latestByUser[r.user.login] = r; + }); + return Object.values(latestByUser); +} + +module.exports = async ({ github, owner, repo, pullNumber }) => { + const { data: reviews } = await github.pulls.listReviews({ + owner: owner, + repo: repo, + pull_number: pullNumber, + }); + + const latestReviews = selectLatestPerUser(reviews); + console.log(latestReviews); + const countApproved = latestReviews.filter( + (r) => r.state === REVIEW_STATE.approved + ).length; + console.log(`${countApproved} approving reviews`); + + return countApproved; +}; diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 2c25f87d38..0000000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Integration tests - -on: - # for the moment, this workflow needs to be triggered manually - workflow_dispatch: - inputs: - git-ref: - description: Git hash (optional) - required: false - repository_dispatch: - # to run this, send a POST API call at repos/IDAES/idaes-pse/dispatches with the specified event_type - # e.g. `gh repos/IDAES/idaes-pse/dispatches -F event_type=integration_tests_requested` - types: [integration_tests_requested] - schedule: - # run daily at 5:00 am UTC (12 am ET/9 pm PT) - - cron: '0 5 * * *' - # TODO add PR when "draft" status is switched off (or absent?) - -defaults: - run: - shell: bash - -jobs: - integration: - name: Integration tests (py=${{ matrix.python-version }}, os=${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - # if fail-fast == true (the default), jobs for the remaining values in the matrix are cancelled - fail-fast: false - matrix: - python-version: - - '3.6' - - '3.7' - os: - - ubuntu-18.04 - - windows-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install idaes and dependencies - run: | - python -m pip --no-cache-dir install --progress-bar off .[dev] - - name: Install extensions - run: | - idaes get-extensions --verbose - find $(idaes data-directory) -ls - # add bin directory to $PATH (only valid for subsequent steps) - echo "$(idaes bin-directory)" >> $GITHUB_PATH - - name: Test access to executables - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - ipopt.exe -v - else - ipopt -v - fi - - name: Run idaes-pse integration tests - run: | - pytest -m integration idaes/ - - name: Fetch examples-pse - run: | - git clone https://github.com/IDAES/examples-pse.git - - name: Run examples-pse tests - run: | - cd examples-pse - pytest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..d05f2fff6c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,102 @@ +name: Main + +on: + push: + branches: + # TODO this could also be run when pushing to main, but could end up clogging the CI when merging multiple PRs + - rel_* + schedule: + # run daily at 5:00 am UTC (12 am ET/9 pm PT) + - cron: '0 5 * * *' + repository_dispatch: + # to run this, send a POST API call at repos/IDAES/idaes-pse/dispatches with the specified event_type + # e.g. `gh repos/IDAES/idaes-pse/dispatches -F event_type=run_tests` + types: [run_tests] + workflow_dispatch: + inputs: + git-ref: + description: Git hash (optional) + required: false + +jobs: + pytest: + name: All tests (py${{ matrix.python-version }}/${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: + - '3.6' + - '3.7' + os: + - ubuntu-18.04 + - windows-latest + steps: + - name: Display debug info + run: | + echo '${{ toJSON(matrix) }}' + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: '.' + - name: Run pytest (all) + run: | + pytest + build-docs: + name: Build Sphinx docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: -r requirements-dev.txt + - name: Build Sphinx docs + uses: ./.github/actions/build-docs + - name: Publish built docs + uses: actions/upload-artifact@v2 + with: + name: idaes-pse-docs-html + path: docs/build/html/ + retention-days: 7 + pylint: + name: pylint (errors only) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Run pylint + uses: ./.github/actions/pylint + pytest-coverage: + name: Run pytest with coverage report + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: -r requirements-dev.txt + - name: Run pytest with cov + run: | + pytest --cov + - name: Upload coverage report to Codecov + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: | + bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/pr-approved.yml b/.github/workflows/pr-approved.yml new file mode 100644 index 0000000000..ca20f3ad4f --- /dev/null +++ b/.github/workflows/pr-approved.yml @@ -0,0 +1,68 @@ +name: PR (integration) + +on: + pull_request: + types: [labeled] + +jobs: + check-skip: + name: Check if integration tests should run + # NOTE: the name of the special label is hardcoded here + # it would be better to extract it to a more global location, e.g. the workflow-level env context, + # but the env context is not available in job-level if expressions (only step-level ones) + if: contains(github.event.label.name, 'ci:approved') + runs-on: ubuntu-latest + steps: + - name: Notify + run: echo "The integration tests will run" + pytest: + name: Integration tests (py${{ matrix.python-version }}/${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: [check-skip] + strategy: + fail-fast: false + matrix: + python-version: + - '3.6' + - '3.7' + os: + - ubuntu-18.04 + - windows-latest + steps: + - name: Display debug info + run: | + echo '${{ toJSON(matrix) }}' + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: '.' + - name: Run pytest (not integration) + run: | + pytest -m 'integration' + pytest-coverage: + name: Run pytest (complete) with coverage report + runs-on: ubuntu-latest + needs: [check-skip] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: -r requirements-dev.txt + - name: Run pytest (complete) with cov + run: | + pytest --cov + - name: Upload coverage report to Codecov + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: | + bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml new file mode 100644 index 0000000000..116bddb93b --- /dev/null +++ b/.github/workflows/pr-review.yml @@ -0,0 +1,124 @@ +name: PR review + +on: + pull_request_review: + types: + - submitted + - edited + - dismissed + +env: + PR_APPROVED_LABEL_NAME: 'ci:approved' + +jobs: + check-approved: + name: Check if PR is approved + runs-on: ubuntu-latest + steps: + - name: Check reviews for approval + id: check-approved + uses: actions/github-script@master + with: + github-token: ${{ secrets.IDAES_BUILD_TOKEN }} + script: | + const ReviewState = { + dismissed: "DISMISSED", + approved: "APPROVED", + changesRequested: "CHANGES_REQUESTED", + commented: "COMMENTED", + }; + const minCountApproved = 2; + const specialReviewer = 'idaes-build'; + const reviewEvent = context.payload; + const pullRequest = reviewEvent.pull_request; + + function prettyPrint(obj, what) { + const indent = 2; + console.log(`${what}:`); + console.log(JSON.stringify(obj, null, indent)); + } + + // check count of approved reviews after selecting latest for each reviewer + async function selectLatestPerUser(reviews) { + const latestByUser = {}; + reviews.forEach((r) => { + // the reviews are in chronological order (earliest to latest) + // so to get the latest for each user we can use an Object as a map and loop over all reviews + // at each iteration, a more recent review for that user will replace an earlier one set before it + // "COMMENTED" reviews are skipped since they don't change the approval status + if (r.state != ReviewState.commented) { + latestByUser[r.user.login] = r; + } + }); + return Object.values(latestByUser); + } + + const fetchedReviews = await github.paginate( + github.pulls.listReviews, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequest.number, + }) + .then( + reviews => reviews + ); + + const countFetched = fetchedReviews.length; + const latestFetched = fetchedReviews[fetchedReviews.length - 1]; + console.log(`Fetched ${countFetched} reviews.`); + prettyPrint(latestFetched, 'Latest fetched review'); + + const latestReviews = await selectLatestPerUser(fetchedReviews); + console.log(`There are ${latestReviews.length} up-to-date reviews from unique reviewers.`); + + const countApproved = latestReviews.filter(r => r.state === ReviewState.approved).length; + console.log(`${countApproved} approving reviews (at least ${minCountApproved} required).`); + const isApproved = countApproved >= minCountApproved; + console.log(`Approved: ${isApproved}`); + + + async function ensureLabelPresence({labelName, shouldBePresent}) { + const commonArgs = { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + }; + const {data: labels} = await github.issues.listLabelsOnIssue({ + ...commonArgs + }); + prettyPrint(labels, 'Current labels'); + const isPresent = labels.filter(label => label.name === labelName).length == 1; + const msg = `Label ${labelName} is ${isPresent ? "" : "not"} present, when it should ${shouldBePresent ? "": "not"} be present.` + console.log(msg); + const alreadyPresent = shouldBePresent && isPresent; + const needsAdding = shouldBePresent && !isPresent; + const needsRemoving = !shouldBePresent && isPresent; + const alreadyAbsent = !shouldBePresent && !isPresent; + if (needsAdding) { + console.log(`Label ${labelName} will be added.`); + await github.issues.addLabels({ + ...commonArgs, + labels: [labelName] + }); + } else if (needsRemoving) { + console.log(`Label ${labelName} will be removed.`); + try { + await github.issues.removeLabel({ + ...commonArgs, + name: labelName, + }); + } catch (error) { + if (error.status === 404) { + console.log(`Could not remove label ${labelName} (label not found).`); + console.log('This is generally not an issue, and is probably due to the label being removed while this workflow was running.'); + } + } + } else if (alreadyPresent || alreadyAbsent) { + console.log(`Label ${labelName} is already in desired state, no further action needed.`); + } + } + const labelName = process.env.PR_APPROVED_LABEL_NAME; + await ensureLabelPresence({ + labelName: labelName, + shouldBePresent: isApproved, + }); diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000000..c8cfa934fd --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,93 @@ +name: PR + +on: + pull_request: + types: + # ready_for_review occurs when a PR is opened in non-draft mode, + # or when a draft PR is turned to non-draft + - ready_for_review + # synchronize occurs whenever commits are pushed to the PR branch + # - synchronize + +jobs: + pytest: + name: Tests (py${{ matrix.python-version }}/${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: + - '3.6' + - '3.7' + os: + - ubuntu-18.04 + - windows-latest + steps: + - name: Display debug info + run: | + echo '${{ toJSON(matrix) }}' + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: '.' + - name: Run pytest (not integration) + run: | + pytest -m 'not integration' + build-docs: + name: Build Sphinx docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: -r requirements-dev.txt + - name: Build Sphinx docs + uses: ./.github/actions/build-docs + - name: Publish built docs + uses: actions/upload-artifact@v2 + with: + name: idaes-pse-docs-html + path: docs/build/html/ + retention-days: 7 + pylint: + name: pylint (errors only) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Run pylint + uses: ./.github/actions/pylint + pytest-coverage: + name: Run pytest with coverage report + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Set up idaes + uses: ./.github/actions/setup-idaes + with: + install-target: -r requirements-dev.txt + - name: Run pytest (not integration) with cov + run: | + pytest -m 'not integration' --cov + - name: Upload coverage report to Codecov + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: | + bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/pull-request-main.yml b/.github/workflows/pull-request-main.yml deleted file mode 100644 index cf3f689fb0..0000000000 --- a/.github/workflows/pull-request-main.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Pull request (main) CI tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: - inputs: - git-ref: - description: Git hash (optional) - required: false - -defaults: - run: - shell: bash - -jobs: - run-tests: - name: Install and test (py=${{ matrix.python-version }}, os=${{ matrix.os }}) - runs-on: ${{ matrix.os }} - strategy: - # if fail-fast == true (the default), jobs for the remaining values in the matrix are cancelled - fail-fast: false - matrix: - python-version: - - '3.6' - - '3.7' - os: - - ubuntu-18.04 - - windows-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install IDAES with dependencies - run: | - python -m pip install --progress-bar off -r requirements.txt - - name: Display IDAES version - run: | - idaes --version - - name: Install IDAES extensions - run: | - idaes get-extensions --verbose - # TODO: these commands are Unix-shell specific - # and will need to be changed if we want to test on CMD.exe/Powershell - find $(idaes data-directory) -ls - # add bin directory to $PATH (only valid for subsequent steps) - echo "$(idaes bin-directory)" >> $GITHUB_PATH - - name: Test access to executables - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - ipopt.exe -v - else - ipopt -v - fi - - name: Run tests (excluding integration) - run: | - pytest -c pytest.ini -m 'not integration' idaes/ - - name: Run integration tests - # if this is a draft PR, skip integration tests - # the idea is that it should not be possible for a PR to be merged without integration tests: - # either it's a draft PR (and thus can't be merged), or it isn't (and integration tests will not be skipped) - if: github.event.pull_request.draft == false - run: | - pytest -c pytest.ini -m integration idaes/ - # TODO: pytest --cov - # TODO: coveralls (coverage report) - build-docs: - name: Build Sphinx docs (HTML) - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install Sphinx dependencies - run: | - # idaes and the modules in idaes.__init__ are imported when accessing idaes.ver - python -m pip --no-cache-dir install --progress-bar off -r requirements-dev.txt - - name: Install IDAES extensions - run: | - idaes get-extensions --verbose - find $(idaes data-directory) -ls - # add bin directory to $PATH (only valid for subsequent steps) - echo "$(idaes bin-directory)" >> $GITHUB_PATH - - name: Test access to executables - run: | - ipopt -v - - name: Build Sphinx docs - run: | - cd docs/ - python build.py - - name: Publish built docs - uses: actions/upload-artifact@v2 - with: - name: idaes-pse-docs-html - path: docs/build/html/ - retention-days: 7 - pylint: - name: Run pylint (errors only) - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install pylint dependencies - run: | - # TODO pylint and astroid versions will need to be pinned - python -m pip --no-cache-dir install --progress-bar off pylint - # in general the package being pylinted does not need to be installed to run pylint on it - # but in our case the pylint plugin needs to be able to import some code from idaes and pyomo - python -m pip --no-cache-dir install --progress-bar off -r requirements.txt - - name: Run pylint - run: | - pylint -E --ignore-patterns="test_.*" idaes || true -