From 8d10484ad3b73d94b6fa849f7bd34aa0b05ea906 Mon Sep 17 00:00:00 2001 From: LGTM Migrator Date: Wed, 16 Nov 2022 15:14:51 +0000 Subject: [PATCH 01/10] Add CodeQL workflow for GitHub code scanning --- .github/codeql.yml | 5 +++++ .github/workflows/codeql.yml | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/codeql.yml create mode 100644 .github/workflows/codeql.yml diff --git a/.github/codeql.yml b/.github/codeql.yml new file mode 100644 index 00000000..d2cb3da5 --- /dev/null +++ b/.github/codeql.yml @@ -0,0 +1,5 @@ +query-filters: + - exclude: + id: py/import-and-import-from + - exclude: + id: py/not-named-self diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..46c475a7 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,42 @@ +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "46 2 * * 5" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql.yml + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" From efddc3ba69a2803a1b04a2bb5497243791e60c93 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Nov 2022 20:32:26 +0100 Subject: [PATCH 02/10] CI: Add some CodeQL ignore rules with too many hits - py/empty-except - py/mixed-returns - py/unused-local-variable --- .github/codeql.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/codeql.yml b/.github/codeql.yml index d2cb3da5..6d342574 100644 --- a/.github/codeql.yml +++ b/.github/codeql.yml @@ -1,5 +1,23 @@ query-filters: + + # Suppress some LGTM warnings. + + - exclude: + id: py/empty-except + + # A module is imported with the "import" and "import from" statements. + # https://lgtm.com/rules/1818040193/ - exclude: id: py/import-and-import-from + + - exclude: + id: py/mixed-returns + + # Disable rule to compensate parameter naming in `CrateCompiler._get_crud_params`. + # Using an alternative name for the first parameter of an instance method makes code more difficult to read. + # https://lgtm.com/rules/910082/ - exclude: id: py/not-named-self + + - exclude: + id: py/unused-local-variable From 048d1887ee54da8256f9d725878f219c98aec95a Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Nov 2022 20:45:36 +0100 Subject: [PATCH 03/10] CI: Allow redundant build requests to cancel running CodeQL jobs --- .github/workflows/codeql.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 46c475a7..1401bdf3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,6 +8,14 @@ on: schedule: - cron: "46 2 * * 5" + # Allow job to be triggered manually. + workflow_dispatch: + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: analyze: name: Analyze From 76be2e4a93bbfcb2142695762f6e9418c39b88ca Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 21 Nov 2022 20:33:52 +0100 Subject: [PATCH 04/10] CI: Remove lgtm configuration, it is CodeQL now --- lgtm.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 lgtm.yml diff --git a/lgtm.yml b/lgtm.yml deleted file mode 100644 index 55db2cfa..00000000 --- a/lgtm.yml +++ /dev/null @@ -1,12 +0,0 @@ -queries: - - # Suppress some LGTM warnings. - - # A module is imported with the "import" and "import from" statements. - # https://lgtm.com/rules/1818040193/ - - exclude: py/import-and-import-from - - # Disable rule to compensate parameter naming in `CrateCompiler._get_crud_params`. - # Using an alternative name for the first parameter of an instance method makes code more difficult to read. - # https://lgtm.com/rules/910082/ - - exclude: py/not-named-self From 57dc0f71995f46e2974b51d813b941b4e636fb3d Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 22 Nov 2022 15:59:16 +0100 Subject: [PATCH 05/10] CI: Improve CodeQL exclude rule configuration --- .github/codeql.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/codeql.yml b/.github/codeql.yml index 6d342574..65efe675 100644 --- a/.github/codeql.yml +++ b/.github/codeql.yml @@ -1,23 +1,35 @@ +# Suppress some CodeQL warnings. query-filters: - # Suppress some LGTM warnings. - + # A few occurrences of try/except clauses without naming a specific exception type. + # TODO: May be improved. + # https://codeql.github.com/codeql-query-help/python/py-empty-except/ - exclude: id: py/empty-except # A module is imported with the "import" and "import from" statements. - # https://lgtm.com/rules/1818040193/ + # CodeQL says "Importing a module twice using the import xxx and from xxx import yyy is confusing.". + # While it has a point, it is still more than acceptable. + # https://codeql.github.com/codeql-query-help/python/py-import-and-import-from/ - exclude: id: py/import-and-import-from + # Quite a few functions don't explicitly return values, but + # instead implicitly return `None`, when falling through. + # TODO: May be improved. + # https://codeql.github.com/codeql-query-help/python/py-mixed-returns/ - exclude: id: py/mixed-returns - # Disable rule to compensate parameter naming in `CrateCompiler._get_crud_params`. - # Using an alternative name for the first parameter of an instance method makes code more difficult to read. - # https://lgtm.com/rules/910082/ + # The parameter naming in `CrateCompiler._get_crud_params` is so that it triggers this admonition. + # However, by using an alternative name for the first parameter of an instance method, it would + # make the code harder to read. + # https://codeql.github.com/codeql-query-help/python/py-not-named-self/ - exclude: id: py/not-named-self + # A few occurrences of unused local variables. + # TODO: May be improved. + # https://codeql.github.com/codeql-query-help/python/py-unused-local-variable/ - exclude: id: py/unused-local-variable From e6afb20b9b082b736f8d6092a8c18aaa9a3bd8ec Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 22 Nov 2022 16:17:08 +0100 Subject: [PATCH 06/10] CI: Don't use CodeQL Autobuild, install package explicitly --- .github/workflows/codeql.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1401bdf3..d725c52c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,8 +41,12 @@ jobs: config-file: ./.github/codeql.yml queries: +security-and-quality - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + #- name: Autobuild + # uses: github/codeql-action/autobuild@v2 + + - name: Install project + run: | + pip install --editable=.[sqlalchemy,test,doc] - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 From 4397cc2e7867fb13e621b4d96c749b2cc3f2e27f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 22 Nov 2022 16:19:55 +0100 Subject: [PATCH 07/10] Chore: Remove lgtm inline comments They don't work anymore, and it looks like there is no pendant for CodeQL. --- src/crate/client/sqlalchemy/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 22a20492..a747bbe5 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -22,9 +22,9 @@ import string from collections import defaultdict -import sqlalchemy as sa # lgtm[py/import-and-import-from] +import sqlalchemy as sa from sqlalchemy.dialects.postgresql.base import PGCompiler -from sqlalchemy.sql import compiler, crud, selectable # lgtm[py/import-and-import-from] +from sqlalchemy.sql import compiler, crud, selectable from .types import MutableDict from .sa_version import SA_VERSION, SA_1_4 From c4edd0d09e1d819dcae4c4d9ba18181534c3d5fc Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 22 Nov 2022 18:26:09 +0100 Subject: [PATCH 08/10] Tests: Improve `bin/test`, mitigating `py/procedure-return-value-used` `zope.testrunner.run` already invokes `sys.exit`. No need to do it here. --- bin/test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/test b/bin/test index bc720824..97594c8b 100755 --- a/bin/test +++ b/bin/test @@ -11,6 +11,6 @@ base = os.path.dirname(base) sys.argv[0] = os.path.abspath(sys.argv[0]) if __name__ == '__main__': - sys.exit(zope.testrunner.run((['--auto-color', '--verbose']) + [ + zope.testrunner.run((['--auto-color', '--verbose']) + [ '--test-path', join(base, 'src'), - ])) + ]) From c9815422fbe0cc254926cf8ff512e04fad8f67f9 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 23 Nov 2022 16:11:42 +0100 Subject: [PATCH 09/10] CI: Don't suppress CodeQL rule `py/unused-local-variable` --- .github/codeql.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/codeql.yml b/.github/codeql.yml index 65efe675..19f66800 100644 --- a/.github/codeql.yml +++ b/.github/codeql.yml @@ -27,9 +27,3 @@ query-filters: # https://codeql.github.com/codeql-query-help/python/py-not-named-self/ - exclude: id: py/not-named-self - - # A few occurrences of unused local variables. - # TODO: May be improved. - # https://codeql.github.com/codeql-query-help/python/py-unused-local-variable/ - - exclude: - id: py/unused-local-variable From 152cbbba13eeb5e26e5cd8713e326d7fa1a8cd97 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 23 Nov 2022 16:33:04 +0100 Subject: [PATCH 10/10] CI: Mitigate one occurrence of `py/unused-local-variable` --- src/crate/testing/layer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 3bd3fc99..3c5ed939 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -358,7 +358,6 @@ def _wait_for(self, validator): time.sleep(self.wait_interval) self.monitor.consumers.remove(line_buf) - line_buf = None def _wait_for_start(self): """Wait for instance to be started"""