From e5bd591c2540bd7fc8cc8205f205f8413d50e01b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 00:29:09 -0400 Subject: [PATCH 01/71] chore(deps): brotli 1.1.0 breaks under python 2 temporarily fix by pinning to brotli 1.0.9. https://github.com/google/brotli/issues/1074 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d75fccd4..cfa90646 100644 --- a/.travis.yml +++ b/.travis.yml @@ -138,7 +138,7 @@ install: - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune==0.8.4; fi - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* && $TRAVIS_PYTHON_VERSION != "2."* ]]; then pip install Markdown; fi - pip install 'markdown2<=2.4.8' - - pip install brotli + - pip install brotli==1.0.9 # zstd fails to build under python nightly aborting test. # allow testing to still happen if the optional package doesn't install. - pip install zstd || true From 3f4e918cc35e558bed501461fa8cbd5f539f4c22 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 00:42:01 -0400 Subject: [PATCH 02/71] build: disable documention on xapian for travis-ci. --disable-documentation passed to configure doesn't work and I have a crash in sphinx that prevents testing. Use -enable-documentation=no which appears to be how 1.4.14 expects it at least. no-github-ci to disable testing for github as this change affects travis only at this point. If it works, I'll disable doc build on github to to save cycles. --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index cfa90646..2506ff41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,14 +80,14 @@ before_install: - tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz - cd xapian-bindings-$XAPIAN_VER/ - echo $TRAVIS_PYTHON_VERSION - - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi + - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --enable-documentation=no; fi # edit the configure script. distutils.sysconfig.get_config_vars('SO') # doesn't work for 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO to EXT_SUFFIX # to get valid value. - - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi + - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi + - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi + - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi - case "$TRAVIS_PYTHON_VERSION" in disable) echo skipping xapian build;; *) make && make install; esac - PATH=$VIRTUAL_ENV/bin:$PATH From 42125078ac1417f25bfb7a5854b8131d43f42bbc Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 01:02:36 -0400 Subject: [PATCH 03/71] build: try disable doc again. It looks like disable-documention does work but only with --enable-maintainer-mode as well. So let's try this combo. --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2506ff41..69ea131a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,14 +80,14 @@ before_install: - tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz - cd xapian-bindings-$XAPIAN_VER/ - echo $TRAVIS_PYTHON_VERSION - - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --enable-documentation=no; fi + - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation --enable-maintainer-mode; fi # edit the configure script. distutils.sysconfig.get_config_vars('SO') # doesn't work for 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO to EXT_SUFFIX # to get valid value. - - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi - - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi - - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --enable-documentation=no; fi + - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi + - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi + - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi - case "$TRAVIS_PYTHON_VERSION" in disable) echo skipping xapian build;; *) make && make install; esac - PATH=$VIRTUAL_ENV/bin:$PATH From 3373b680f73a8a02a4c0ad727a05757ccbcfd574 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 01:17:57 -0400 Subject: [PATCH 04/71] build: revert all attempts to disable docs travis-ci will remain broken for nightly (3.12) testing. Trying to enable maintainer mode broke on a missing dot executable. Disable nightly since something is borked in the base config there and I don't want to waste credits on it. --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69ea131a..b38a9f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ python: # - 3.8 # - 3.6 # - 3.11-dev - - nightly +# - nightly # - pypy3 services: @@ -80,14 +80,14 @@ before_install: - tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz - cd xapian-bindings-$XAPIAN_VER/ - echo $TRAVIS_PYTHON_VERSION - - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation --enable-maintainer-mode; fi + - if [[ $TRAVIS_PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi # edit the configure script. distutils.sysconfig.get_config_vars('SO') # doesn't work for 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO to EXT_SUFFIX # to get valid value. - - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi - - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi - - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation --enable-maintainer-mode; fi + - if [[ $TRAVIS_PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi + - if [[ $TRAVIS_PYTHON_VERSION == "nightly" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi + - if [[ $TRAVIS_PYTHON_VERSION == "pypy3" ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - case "$TRAVIS_PYTHON_VERSION" in disable) echo skipping xapian build;; *) make && make install; esac - PATH=$VIRTUAL_ENV/bin:$PATH From dd36940f54811245f98fd8be030221b81b07e4fc Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 01:40:58 -0400 Subject: [PATCH 05/71] build: build xapian on 3.12 See if xapian will build on github with 3.12rc2 using the standard build method. Removed override to 1.14.22 for 3.12 and enabled build rather than skipping on 3.12. --- .github/workflows/ci-test.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 032244c9..56425d9f 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -189,11 +189,7 @@ jobs: # older python and newest on newer. if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi - if [[ $PYTHON_VERSION == '3.12'* ]] ; then \ - XAPIAN_VER=1.4.22; \ - else - XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; \ - fi + XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER cd /tmp curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz @@ -205,7 +201,7 @@ jobs: # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - case "$PYTHON_VERSION" in nightly|3.12*) echo skipping xapian build;; *) make && sudo make install; esac + case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests run: pip install flake8 mock pytest pytest-cov requests From ec63caa0e9008b620cee49261aa80e709903a005 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 02:00:00 -0400 Subject: [PATCH 06/71] build: xapian under 3.12 - missing version info in build components The build with 1.4.18 fails to get the shared library file names correct. Trying with 1.4.22 as the release against the 1.4.18 dev libraries to see if it works better. The sed script I have that munges the configure script to change the method used for getting the components should work but it obviously doesn't. Failure under 3.12: /usr/bin/install -c -m 644 xapian/__init__.py \ ./xapian/__pycache__/__init__..pyc \ ./xapian/__pycache__/__init__. \ '/opt/hostedtoolcache/Python/3.12.0-rc.2/x64/lib/python3.12/site-packages/xapian' /usr/bin/install: cannot stat './xapian/__pycache__/__init__..pyc': No such file or directory /usr/bin/install: cannot stat './xapian/__pycache__/__init__.': No such file or directory Success under 3.11 which also needs the sed script: /usr/bin/install -c -m 644 xapian/__init__.py \ xapian/__pycache__/__init__.cpython-311.pyc \ xapian/__pycache__/__init__.cpython-311.opt-1.pyc \ '/opt/hostedtoolcache/Python/3.11.5/x64/lib/python3.11/site-packages/xapian' --- .github/workflows/ci-test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 56425d9f..815a2f68 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -189,7 +189,11 @@ jobs: # older python and newest on newer. if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi - XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER + if [[ $PYTHON_VERSION == '3.12'* ]] ; then \ + XAPIAN_VER=1.4.22; \ + else + XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; \ + fi cd /tmp curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz @@ -200,7 +204,8 @@ jobs: # 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. - if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi + if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure + --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests From 91be109b9f85f9e1a786b8f413d6b7c372a9f114 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 02:03:45 -0400 Subject: [PATCH 07/71] build: fix syntax error. Bad line break. [skip travis] --- .github/workflows/ci-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 815a2f68..20b9f6d3 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -204,8 +204,7 @@ jobs: # 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. - if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure - --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi + if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests From 103d1969c62da064fc322f1b2d5bf121eb39f62b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 02:09:55 -0400 Subject: [PATCH 08/71] build: disable xapian under 3.12 python. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit as documented in issue2551277 1.4.22 doesn't build with: xapian_wrap.cc: In function ‘PyObject* PyInit__xapian()’: xapian_wrap.cc:65031:116: error: ‘FLAG_NO_POSITIONS’ is not a member of ‘Xapian::QueryParser’ 65031 | SWIG_Python_SetConstant(d,"QueryParser_FLAG_NO_POSITIONS", SWIG_From_int(static_cast< int >(Xapian::QueryParser::FLAG_NO_POSITIONS))); --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 20b9f6d3..032244c9 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -205,7 +205,7 @@ jobs: # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac + case "$PYTHON_VERSION" in nightly|3.12*) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests run: pip install flake8 mock pytest pytest-cov requests From 1a9c3853f78b190c9703a80c36d164eb7ae96eeb Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 03:11:46 -0400 Subject: [PATCH 09/71] build: try 3.12 xapian build again. Well configure needs to be taught not to use imp library under 3.12 as imp doesn't exist anymore. Update sed script for configure to make the required modifications. This should work going back to 3.6 as the replacement: imp.gettag() -> sys.implementation.cache_tag imp.util.cache_from_source -> importlib.util.cache_from_source works in 3.6.9 at least and doc claims they were introduced in 3.2/3.4. Geesh what a pain. Good thing there was a dockerfile with 3.12rc2 I could use to debug this. --- .github/workflows/ci-test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 032244c9..b11e7df1 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -204,8 +204,17 @@ jobs: # 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. - if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi - case "$PYTHON_VERSION" in nightly|3.12*) echo skipping xapian build;; *) make && sudo make install; esac + if [[ $PYTHON_VERSION == "3."* ]]; then \ + sed -i \ + -e '/PYTHON3_SO=/s/distutils\.//g' \ + -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ + -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ + -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ + -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/' \ + configure; \ + ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ + fi + case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests run: pip install flake8 mock pytest pytest-cov requests From 7c19dd491529aa1e9f8e8efc7e55f03cd6b6f0f3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 03:13:27 -0400 Subject: [PATCH 10/71] build: fix indent in yaml. --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b11e7df1..844031a4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -213,7 +213,7 @@ jobs: -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/' \ configure; \ ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ - fi + fi case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac - name: Install pytest and other packages needed for running tests From 4d2a770f912760c68dcc8e1aac052b25a8156cea Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 03:24:16 -0400 Subject: [PATCH 11/71] build: xapian under 3.12 add debugging as this broke xapian on all builds. For 3.12 use discovered xapian version not 1.4.22 override. --- .github/workflows/ci-test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 844031a4..fd50a0e3 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -184,16 +184,13 @@ jobs: - name: Install xapian run: | + set -xv sudo apt-get install libxapian-dev # Sphinx required to build the xapian python bindings. Use 1.8.5 on # older python and newest on newer. if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi - if [[ $PYTHON_VERSION == '3.12'* ]] ; then \ - XAPIAN_VER=1.4.22; \ - else - XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; \ - fi + XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; cd /tmp curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz @@ -205,6 +202,7 @@ jobs: # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. if [[ $PYTHON_VERSION == "3."* ]]; then \ + cp configure configure.FCS; \ sed -i \ -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ @@ -212,6 +210,7 @@ jobs: -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/' \ configure; \ + diff -u configure.FCS configure; \ ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ fi case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac From 1d92a3f3cf81923be740a0910ad7d5162ef30bc3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 03:35:03 -0400 Subject: [PATCH 12/71] build: make sed sub global on line. --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index fd50a0e3..1e156411 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -208,7 +208,7 @@ jobs: -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ - -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/' \ + -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ configure; \ diff -u configure.FCS configure; \ ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ From 3becacbf9b35ce2836158d7c9c12eeeca3e0a57d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 03:38:22 -0400 Subject: [PATCH 13/71] build: prevent diff from exiting shell script. [skip travis] --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 1e156411..72639853 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -210,7 +210,7 @@ jobs: -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ configure; \ - diff -u configure.FCS configure; \ + diff -u configure.FCS configure || true; \ ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ fi case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac From 768f49f528f98b63c0162b96a441b45f3d91b6c9 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 17 Sep 2023 15:58:09 -0400 Subject: [PATCH 14/71] fix: update licenses, add license to PKG-INFO Forgot to add Python Software Foundation License to the trove categories when I vendored cgi.py. Also added setting to set License in PKG-INFO displayed by 'pip show roundup'. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 39c92983..27b7b63b 100755 --- a/setup.py +++ b/setup.py @@ -196,6 +196,7 @@ def main(): description="A simple-to-use and -install issue-tracking system" " with command-line, web and e-mail interfaces. Highly" " customisable.", + license="OSI Approved: MIT License, Zope Public License, Python Software Foundation License", long_description=long_description, long_description_content_type='text/x-rst', url='https://www.roundup-tracker.org', @@ -213,6 +214,7 @@ def main(): 'License :: OSI Approved', 'License :: OSI Approved :: MIT License', 'License :: OSI Approved :: Zope Public License', + 'License :: OSI Approved :: Python Software Foundation License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', From 83648a21186635656d50a51613d3d6bf9d7b8fbe Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 18 Sep 2023 01:05:01 -0400 Subject: [PATCH 15/71] fix: hadolint fixups Fix warnings reported by hadolint: Quote strings to prevent embedded spaces in varibles from causing misparse Use find in place of ls to generate file list for sdist files as it handles more charsets sanely To run hadolint: docker run --rm -i ghcr.io/hadolint/hadolint < scripts/Docker/Dockerfile --- scripts/Docker/Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index b07f42f8..b30abf94 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -74,7 +74,7 @@ SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG VERBOSE RUN [ -z "${VERBOSE}" ] || set -xv; \ - CWD=$PWD && \ + CWD="$PWD" && \ upgrades=$(python3 -m pip --no-cache --disable-pip-version-check \ list --outdated | awk 'NR > 2 {print $1}'); \ if [ -n "$upgrades" ]; then \ @@ -82,7 +82,7 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ python -m pip --no-cache --disable-pip-version-check \ install -U $upgrades < /dev/null; \ else \ - echo Nothing to pip update; \ + echo "Nothing to pip update"; \ fi; \ ls -l /usr/local/lib/python3.11/site-packages; \ VER=$(apk list -I 'xapian-core-dev' | \ @@ -90,14 +90,14 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ cd /tmp && \ wget -q "https://oligarchy.co.uk/xapian/$VER/xapian-bindings-$VER.tar.xz" && \ tar -Jxvf "xapian-bindings-$VER.tar.xz" && \ - cd xapian-bindings-$VER/ && \ + cd "xapian-bindings-$VER/" && \ pip --no-cache-dir install sphinx && \ sed -i -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure && \ ./configure --prefix=/usr/local --with-python3 --disable-documentation && \ make && make install && \ pip uninstall --no-cache-dir -y sphinx && \ - pip uninstall --no-cache-dir -y -r $CWD/sphinxdeps.txt && \ + pip uninstall --no-cache-dir -y -r "$CWD/sphinxdeps.txt" && \ rm -rf /usr/local/share/doc/xapian-bindings # add requirements for pip here, e.g. Whoosh, gpg, zstd or other @@ -129,7 +129,7 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ "pypi"*) \ version_spec=$( printf "%s" "$source" | \ sed -ne 's/^pypi\([~=!<>].*\)/\1/p'); \ - pip install --no-cache-dir roundup${version_spec}; \ + pip install --no-cache-dir "roundup${version_spec}"; \ cp -ril /usr/local/lib/"python${pythonversion}"/site-packages/usr/local/share/* \ /usr/local/share;; \ "pip_local") \ @@ -137,14 +137,14 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ pip -V && \ pip install --no-cache-dir . ;; \ "pip_sdist") \ - dist=$(ls install/dist | sed -ne '/roundup-[0-9].*\.tar\.gz$/p' | tail -n 1); \ + dist=$(find install/dist | sed -ne '/roundup-[0-9].*\.tar\.gz$/p' | tail -n 1); \ if [ -z "$dist" ] ; then \ printf "Unable to find a source distribution file in dist\n"; \ printf "Exiting\n"; \ exit 1; \ fi; \ printf "Building with distribution: %s\n" "$dist"; \ - pip install --no-cache-dir install/dist/$dist;; \ + pip install --no-cache-dir "install/dist/$dist";; \ *) \ echo "invalid value for source: $source"; \ echo "must be local, pypi, pip_local or pip_sdist"; \ @@ -192,7 +192,7 @@ RUN apk --no-cache upgrade; \ python -m pip --no-cache --disable-pip-version-check \ install -U $upgrades < /dev/null; \ else \ - echo Nothing to pip update; \ + echo "Nothing to pip update"; \ fi ARG source From 73db6d48fda146b2810143e59bb3e07b779ce865 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 18 Sep 2023 01:06:24 -0400 Subject: [PATCH 16/71] fix: update comments force exit on pipefail --- scripts/Docker/roundup_start | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 4cbb2cf3..21fd8624 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -4,7 +4,11 @@ # If they are, start the server otherwise run roundup-admin # for installation and initialization. -# "$@" should be a set of tracker=directory pairs. +# "$@" should be one of the recognized keywords or arguments for +# roundup-server including one or more tracker tracker=directory pairs. + +# exit on errors +set -eo pipefail if ! [ -z "$SHELL_DEBUG" ]; then set -xv From 275a30ffc6a2649051d5cef65469a4d4eed6a7ae Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 18 Sep 2023 01:57:50 -0400 Subject: [PATCH 17/71] docs: clarify Api version method priority; payload @apiver Define how version is determined when multiple conflicting methods are used. Add doc on use of @apiver in data payload for PUT/POST (and maybe PATCH). Now that this is defined, sent email to devel list to see if the order should be changed. The url parameter is the lowest priorty setting and will be ignored if any of the 3 higher priority methods are used. However using the url setting is the easiest for testing. --- doc/rest.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/rest.txt b/doc/rest.txt index c14b7952..8d34607d 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -222,7 +222,8 @@ Versioning Currently there is only one version of the API. Versions are simple integers. The current version is ``1``. Version selection is -implemented in the server using one of three methods: +implemented in the server using one of four methods (in priority +order, highest first): 1. Explicit version param in accept header: ``application/json; version=1`` @@ -230,7 +231,12 @@ implemented in the server using one of three methods: 2. Version suffix in vendor accept header: ``application/vnd.json.test-v1+json`` - 3. Adding version specifier in query string: ``@apiver=1`` + 3. Adding ``@apiver: 1`` in the input data wrapper (for POST, PUT) + + 4. Adding version specifier in query string: ``@apiver=1`` (for GET). + +The highest priority version method will be used if multiple +methods are used. If an explicit version is not provided, the server default is used. The server default is reported by querying the ``/rest/`` endpoint as From abf50e60dacd6f2dbbac4e110c8cc397df9defb3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 00:29:45 -0400 Subject: [PATCH 18/71] build: update python:3-alpine version Update the sha256 sum used to download the current 3-alpine version. Also abstracted it into a variable (argument) that is used in both the build and deployment images. --- scripts/Docker/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index b30abf94..3170e2ed 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -8,6 +8,8 @@ # docker run --rm -v /.../issue.tracker:/usr/src/app/tracker \ # -p 9017:8080 roundup-app:latest +# parameterize the sha256 sum to pin version of python:3-alpine +ARG SHA256=5d769f990397afbb2aca24b0655e404c0f2806d268f454b052e81e39d87abf42 # Global vars for all build stages ARG VERBOSE= @@ -26,8 +28,8 @@ ARG source=local # COPY from install dir in second stage. ARG pythonversion=3.11 -#FROM python:3-alpine -FROM python@sha256:603975e62d85aa07578034d3d10ffa1983b7618a6abb6371cf51941be6b8842c as build +#FROM python:3-alpine via SHA256 sum +FROM python@sha256:$SHA256 as build # Inherit global values https://github.com/moby/moby/issues/37345 ARG appdir @@ -158,7 +160,7 @@ RUN if [ -n "$pip_mod" ]; then pip install --no-cache-dir ${pip_mod}; fi # build a new smaller docker image for execution. Build image above # is 1G in size. # FROM python:3-alpine -FROM python@sha256:603975e62d85aa07578034d3d10ffa1983b7618a6abb6371cf51941be6b8842c +FROM python@sha256:$SHA256 # import from global ARG appdir From 11c6582706eb0bddc09c73ce121d7de1dc0ead5b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 00:36:20 -0400 Subject: [PATCH 19/71] docs: add "roles" to list of required user object properties. Not sure why it wasn't there before. Without the "roles" property the authorization system kind of fails. --- doc/reference.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference.txt b/doc/reference.txt index daf73c45..bcb4c354 100644 --- a/doc/reference.txt +++ b/doc/reference.txt @@ -686,7 +686,7 @@ You must never: **Remove the user class** This class is the only *required* class in Roundup. -**Remove the "username", "address", "password" or "realname" user properties** +**Remove the "username", "address", "password", "roles" or "realname" user properties** Various parts of Roundup require these properties. Don't remove them. **Change the type of a property** From d4485f1a552a4d27a763285f40ba5a22073bfaa0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 02:02:17 -0400 Subject: [PATCH 20/71] build: make Dockerfile work for 3.12 release 3.12 build of xapian requires some additional edits. Copy them from the current 3.12 compatible workflow. This should mean that future updates to python:3-alpine to 3.12 will "just work". --- scripts/Docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 3170e2ed..9784dd09 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -95,7 +95,11 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ cd "xapian-bindings-$VER/" && \ pip --no-cache-dir install sphinx && \ sed -i -e '/PYTHON3_SO=/s/distutils\.//g' \ - -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure && \ + -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ + -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ + -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ + -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ + configure && \ ./configure --prefix=/usr/local --with-python3 --disable-documentation && \ make && make install && \ pip uninstall --no-cache-dir -y sphinx && \ From a1b8021e89caafb2651c22151c71ad23fc3fa2c2 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 12:25:40 -0400 Subject: [PATCH 21/71] fix,docs: Move args around into groups; add docs. Create basic and advanced user ARG groups; add some doc explanations. --- scripts/Docker/Dockerfile | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 9784dd09..82c8f497 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -8,24 +8,32 @@ # docker run --rm -v /.../issue.tracker:/usr/src/app/tracker \ # -p 9017:8080 roundup-app:latest +# arguments expected to be useful on command line. Lower case names. + +# Source of roundup code: +# 'local' directory using setup.py, +# 'pypi[...]' to install latest final or version specified release +# from PyPI, E.G. pypi=2.4; pypi<2.3 etc. All pip version specifiers +# are allowed. +# 'pip_local' local directory using pip to install, or +# 'pip_sdist' to install dist/newest_version-tarball +ARG source=local + +# Internal settings that are for advanced users to override. + # parameterize the sha256 sum to pin version of python:3-alpine +# Must use the same version in both build stages. ARG SHA256=5d769f990397afbb2aca24b0655e404c0f2806d268f454b052e81e39d87abf42 -# Global vars for all build stages +# Set to any non-empy value to enable shell debugging for troubleshooting ARG VERBOSE= # application directory ARG appdir=/usr/src/app -# support roundup install from 'local' directory, -# 'pypi' to install latest final or version specified release -# from PyPI, -# 'pip_local' local directory using pip to install, or -# 'pip_sdist' to install dist/newest_version-tarball -ARG source=local - -# Python version as a.b Used for installation directory and -# COPY from install dir in second stage. +# Python version as a.b Used as path component for +# installation directory and COPY from install dir +# in second build stage. ARG pythonversion=3.11 #FROM python:3-alpine via SHA256 sum @@ -119,7 +127,7 @@ COPY frontends install/frontends/ COPY locale install/locale/ COPY roundup install/roundup/ COPY share install/share/ -# dist is optional so include README.txt so we don't get an error +# dist/* might not exist, so include README.txt so we don't get an error COPY README.txt dist/* install/dist/ # verify source has one of two valid values then From 2571eae8a9a254ae0797f99dac45746e6fdbc1f5 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 21:33:52 -0400 Subject: [PATCH 22/71] build(deps): bump actions/checkout from 4.0.0 to 4.1.0 - https://github.com/roundup-tracker/roundup/pull/50 --- .github/workflows/anchore.yml | 2 +- .github/workflows/ci-test.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/ossf-scorecard.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index ab0e43f9..8a6ba14d 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the code - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Build the Docker image run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 72639853..68fa0208 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -93,7 +93,7 @@ jobs: # if: {{ false }} # continue running if step fails # continue-on-error: true - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1caf4cf7..70786651 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2.6.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 69a31cc3..a1b2fc88 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.1.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: persist-credentials: false From 1d8895e5e50c5526d6b9c6f05309fe51bef1c383 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 21:35:47 -0400 Subject: [PATCH 23/71] build(docker): add file package to clear error when building xapian It looks like the error was non-fatal. The image I believe worked fine. But, clean up the noise. --- scripts/Docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 82c8f497..2d915eab 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -51,6 +51,7 @@ RUN apk --no-cache upgrade # Can't use apk to add python packages as it installs 3.9 python version. # g++ installs cc1plus needed by pip install RUN apk --no-cache add \ + file \ g++ \ gcc \ gpgme-dev \ From e50e256f3b12a7375925fdf6faf8c5f8132d1d4b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 22:24:51 -0400 Subject: [PATCH 24/71] build(docker): disable pip_sdist mode in github ci: COPY README.txt dist/* install/dist/ crashes the docker build if the dist directory is missing. This prevents anchore/grype from scanning the image. This is a new issue as it doesn't happen on my system. The README.txt file is there explicitly to prevent COPY from failing if nothing matches the wildcard. This used to work, but .... So I am disabling building from local sdist. This only affects maintainers. Building from sdist is done to generate a docker that matches a source distribution that has yet to be uploaded to pypi. Other use cases can build using the default of local. I haven't found a way to make a dist subdir in the docker build context directory. I can't even figure out how to identify the context directory name inside the build container. Also updated installation docs. --- doc/installation.txt | 1 + scripts/Docker/Dockerfile | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/installation.txt b/doc/installation.txt index a9962d32..b0295f4e 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -1754,6 +1754,7 @@ You can build a docker container in one of 4 modes defined by the by some Python users. ``--build-arg="source=pip_sdist"`` + Disabled. This is meant for maintainer/developer use. It installs using pip from a source distribution (sdist) tarball built by following the RELEASE.txt. It is meant for testing diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 2d915eab..c960db4c 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -129,7 +129,7 @@ COPY locale install/locale/ COPY roundup install/roundup/ COPY share install/share/ # dist/* might not exist, so include README.txt so we don't get an error -COPY README.txt dist/* install/dist/ +#COPY README.txt dist/* install/dist/ # verify source has one of two valid values then # install in python3 standard directories from local copy @@ -152,17 +152,18 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ pip -V && \ pip install --no-cache-dir . ;; \ "pip_sdist") \ + printf "\n\npip_sdist disabled\n\n" && exit 1 ; \ dist=$(find install/dist | sed -ne '/roundup-[0-9].*\.tar\.gz$/p' | tail -n 1); \ if [ -z "$dist" ] ; then \ - printf "Unable to find a source distribution file in dist\n"; \ - printf "Exiting\n"; \ + printf "\n\nUnable to find a source distribution file in dist\n"; \ + printf "Exiting\n\n"; \ exit 1; \ fi; \ printf "Building with distribution: %s\n" "$dist"; \ pip install --no-cache-dir "install/dist/$dist";; \ *) \ - echo "invalid value for source: $source"; \ - echo "must be local, pypi, pip_local or pip_sdist"; \ + printf "\n\ninvalid value for source: $source\n"; \ + printf "must be local, pypi, pip_local or pip_sdist\n\n"; \ exit 1;; \ esac From 6148e502f98356d52fa34e19fd2782bce06c21db Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 22:54:34 -0400 Subject: [PATCH 25/71] build: disable CVE-2018-20225 pip package shadow ignore as long as status is not-fixed. description: ** DISPUTED ** An issue was discovered in pip (all versions) because it installs the version with the highest version number, even if the user had intended to obtain a private package from a private index. This only affects use of the --extra-index-url option, and exploitation requires that the package does not already exist in the public index (and thus the attacker can put the package there with an arbitrary version number). NOTE: it has been reported that this is intended functionality and the user is responsible for using --extra-index-url securely. [skip travis] --- .grype.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .grype.yaml diff --git a/.grype.yaml b/.grype.yaml new file mode 100644 index 00000000..c664008c --- /dev/null +++ b/.grype.yaml @@ -0,0 +1,4 @@ +ignore: + - vulnerability: CVE-2018-20225 + fix-state: not-fixed + From 1c38d0dc825860e182186b5f09e51e1842cf4429 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 23:12:59 -0400 Subject: [PATCH 26/71] build: try different ID for pip grype error. The bare CVE id CVE-2018-20225 didn't work. In the sarif output it report id: CVE-2018-20225-pip, so try that. --- .grype.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.grype.yaml b/.grype.yaml index c664008c..594c12a8 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -1,4 +1,5 @@ ignore: - vulnerability: CVE-2018-20225 fix-state: not-fixed - + - vulnerability: CVE-2018-20225-pip + fix-state: not-fixed From 697c4d1a01850ffb742f56a3bc61825d8f94bda4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 23:36:25 -0400 Subject: [PATCH 27/71] build: try copying .grype.yaml to $HOME .grype.yaml in root of checkout seems to not work. Try copying to $HOME. from: https://stackoverflow.com/questions/70923388/create-a-file-in-github-action [skip travis] --- .github/workflows/anchore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index 8a6ba14d..d50ffdf3 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -42,6 +42,8 @@ jobs: run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image run: docker image ls + - name: copy grype.yaml into $home + run: cp .grype.yaml $HOME/ - name: Run the Anchore scan action itself with GitHub Advanced Security code scanning integration enabled uses: anchore/scan-action@24fd7c9060f3c96848dd1929fac8d796fb5ae4b4 # v3.3.6 id: scan From 98d6503a8668f8275970b327b57f636214afe03f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 24 Sep 2023 23:54:18 -0400 Subject: [PATCH 28/71] build: break YAML to see if grype throws an error. A debug run of anchore is showing the config as: configpath: /home/runner/work/roundup/roundup/.grype.yaml when running: Executing: grype -vv -o sarif --fail-on medium localbuild/testimage:latest Try breaking the yaml to see if it is actually being loaded. [skip travis] --- .github/workflows/anchore.yml | 2 -- .grype.yaml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index d50ffdf3..8a6ba14d 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -42,8 +42,6 @@ jobs: run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image run: docker image ls - - name: copy grype.yaml into $home - run: cp .grype.yaml $HOME/ - name: Run the Anchore scan action itself with GitHub Advanced Security code scanning integration enabled uses: anchore/scan-action@24fd7c9060f3c96848dd1929fac8d796fb5ae4b4 # v3.3.6 id: scan diff --git a/.grype.yaml b/.grype.yaml index 594c12a8..04c349bf 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -1,5 +1,5 @@ ignore: - - vulnerability: CVE-2018-20225 + - vlnerability: CVE-2018-20225 fix-state: not-fixed - vulnerability: CVE-2018-20225-pip - fix-state: not-fixed + fix-state: not-fixed From 67b296617e1ce284b4192c3e4e1fa684dd077713 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 25 Sep 2023 00:03:46 -0400 Subject: [PATCH 29/71] build: fix format and remove fix-state. The run failed. So the .grype.yaml is being loaded. Only set the vulnerability id. Ignore fix-state. This will mean that I don't get a warning if it gets fixed, but this is unlikely to be fixed. --- .grype.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.grype.yaml b/.grype.yaml index 04c349bf..b852f927 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -1,5 +1,3 @@ ignore: - - vlnerability: CVE-2018-20225 - fix-state: not-fixed + - vulnerability: CVE-2018-20225 - vulnerability: CVE-2018-20225-pip - fix-state: not-fixed From 0c7cf54478c8ab2750f0117c0c786d388ebe32ec Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 28 Sep 2023 09:43:37 -0400 Subject: [PATCH 30/71] refactor: issue2551293 - remove schema_hook from Tracker instance. Not used, obsolete. --- CHANGES.txt | 3 +++ roundup/instance.py | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 89e1e945..2d2f9fe8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,9 @@ Fixed: source install. (John Rouillard) - Document use of pyreadline3 to allow roundup-admin to have CLI editing on windows. (John Rouillard) +- issue2551293 - remove schema_hook from Tracker instance. Looks like + it was an obsolete hook used for testing. Never documented and not + accessible from schema.py. Features: diff --git a/roundup/instance.py b/roundup/instance.py index dbb2dbec..eb03967b 100644 --- a/roundup/instance.py +++ b/roundup/instance.py @@ -61,10 +61,6 @@ def __init__(self, tracker_home, optimize=0): """ self.tracker_home = tracker_home self.optimize = optimize - # if set, call schema_hook after executing schema.py will get - # same variables (in particular db) as schema.py main purpose is - # for regression tests - self.schema_hook = None self.config = configuration.CoreConfig(tracker_home) self.actions = {} self.cgi_actions = {} @@ -118,15 +114,11 @@ def open(self, name=None): if self.optimize: # execute preloaded schema object self._exec(self.schema, env) - if isinstance(self.schema_hook, Callable): - self.schema_hook(**env) # use preloaded detectors detectors = self.detectors else: # execute the schema file self._execfile('schema.py', env) - if isinstance(self.schema_hook, Callable): - self.schema_hook(**env) # reload extensions and detectors for extension in self.get_extensions('extensions'): extension(self) From 8f9b5eec8493ffe8b32fcc104c9fc07051aa337b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 09:53:22 -0400 Subject: [PATCH 31/71] bug: Fix roundup-admin security command. Lowercase optionalarg. Roles are indexed by lower case role name. So 'security User' and 'security user' should generate the same output. Also add testing for this case. Thread: https://sourceforge.net/p/roundup/mailman/roundup-users/thread/CAH-41398iTPhze7D_pZB8tqTBHF%3Dq6HYonbcG%2B%2BYN-ioDssXBw%40mail.gmail.com/#msg41557225 starting from: https://sourceforge.net/p/roundup/mailman/message/41557225/ --- CHANGES.txt | 4 +++ roundup/admin.py | 3 +- test/test_admin.py | 72 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2d2f9fe8..d03c2fc1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -50,6 +50,10 @@ Fixed: - issue2551293 - remove schema_hook from Tracker instance. Looks like it was an obsolete hook used for testing. Never documented and not accessible from schema.py. +- Fix roundup-admin security command. Lowercase its optional + argument. Roles are indexed by lower case role name. So 'security + User' and 'security user' should generate the same output. (John + Rouillard from issue on mailing list by Chuck Cunningham) Features: diff --git a/roundup/admin.py b/roundup/admin.py index ea0d1e02..7902ed47 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -1715,7 +1715,8 @@ def do_security(self, args): if len(args) == 1: role = args[0] try: - roles = [(args[0], self.db.security.role[args[0]])] + roles = [(args[0].lower(), + self.db.security.role[args[0].lower()])] except KeyError: sys.stdout.write(_('No such Role "%(role)s"\n') % locals()) return 1 diff --git a/test/test_admin.py b/test/test_admin.py index 47044dc9..28501be1 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1164,7 +1164,77 @@ def disabletestHelpInitopts(self): self.assertTrue(expected[0] in out) self.assertTrue("Back ends:" in out) - def testSecurity(self): + def testSecurityListOne(self): + self.install_init() + self.admin=AdminTool() + + with captured_output() as (out, err): + # make sure UsEr returns result for user. Roles are + # lower cased interally + sys.argv=['main', '-i', self.dirname, 'security', "user" ] + ret = self.admin.main() + + result = """Role "user": + User may access the web interface (Web Access) + User may use the email interface (Email Access) + User may access the rest interface (Rest Access) + User may access the xmlrpc interface (Xmlrpc Access) + User is allowed to access issue (View for "issue" only) + User is allowed to edit issue (Edit for "issue" only) + User is allowed to create issue (Create for "issue" only) + User is allowed to access file (View for "file" only) + User is allowed to edit file (Edit for "file" only) + User is allowed to create file (Create for "file" only) + User is allowed to access msg (View for "msg" only) + User is allowed to edit msg (Edit for "msg" only) + User is allowed to create msg (Create for "msg" only) + User is allowed to access keyword (View for "keyword" only) + User is allowed to edit keyword (Edit for "keyword" only) + User is allowed to create keyword (Create for "keyword" only) + User is allowed to access priority (View for "priority" only) + User is allowed to access status (View for "status" only) + (View for "user": ('id', 'organisation', 'phone', 'realname', 'timezone', 'username') only) + User is allowed to view their own user details (View for "user" only) + User is allowed to edit their own user details (Edit for "user": ('username', 'password', 'address', 'realname', 'phone', 'organisation', 'alternate_addresses', 'queries', 'timezone') only) + User is allowed to view their own and public queries (View for "query" only) + (Search for "query" only) + User is allowed to edit their queries (Edit for "query" only) + User is allowed to retire their queries (Retire for "query" only) + User is allowed to restore their queries (Restore for "query" only) + User is allowed to create queries (Create for "query" only) +""" + print(out.getvalue()) + + self.assertEqual(result, out.getvalue()) + self.assertEqual(ret, 0) + + + # test 2 all role names are lower case, make sure + # any role name is correctly lower cased + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, 'security', "UsEr" ] + ret = self.admin.main() + + print(out.getvalue()) + + self.assertEqual(result, out.getvalue()) + self.assertEqual(ret, 0) + + # test 3 Check error if role does not exist + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, 'security', "NoSuch Role" ] + ret = self.admin.main() + + result='No such Role "NoSuch Role"\n' + print('>', out.getvalue()) + + self.assertEqual(result, out.getvalue()) + self.assertEqual(ret, 1) + + + def testSecurityListAll(self): ''' Note the tests will fail if you run this under pdb. the context managers capture the pdb prompts and this screws up the stdout strings with (pdb) prefixed to the line. From c59bfbd751cf34b7b4a2412190572ef352cc2893 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 13:37:17 -0400 Subject: [PATCH 32/71] build: add vendored package to setup.py. It wasn't being installed. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 27b7b63b..473224f3 100755 --- a/setup.py +++ b/setup.py @@ -105,6 +105,7 @@ def main(): packages = [ 'roundup', 'roundup.anypy', + 'roundup.anypy.vendored', 'roundup.cgi', 'roundup.cgi.PageTemplates', 'roundup.cgi.TAL', From 509c3a006529de1f64423979c799b5028f0b2f7f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 15:59:51 -0400 Subject: [PATCH 33/71] Add german translation for 'Continue adding keywords. Courtesy of google translate that gave me two different sentences one when I left off the period and another with the period. --- locale/de.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locale/de.po b/locale/de.po index 5df9cf00..9af6e071 100644 --- a/locale/de.po +++ b/locale/de.po @@ -4362,7 +4362,7 @@ msgstr "" #: ../share/roundup/templates/classic/html/keyword.item.html:51 msgid "Continue adding keywords." -msgstr "" +msgstr "Fügen Sie weitere Keywords hinzu." #: ../share/roundup/templates/classic/html/msg.index.html:3 #: ../share/roundup/templates/devel/html/msg.index.html:3 From dd9507a88c93b1dfa92cffa16c9b780b7902fabf Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 16:07:43 -0400 Subject: [PATCH 34/71] Add german translation for 'Continue adding keywords. Oops. Use the one with the period. --- locale/de.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locale/de.po b/locale/de.po index 9af6e071..a33b8569 100644 --- a/locale/de.po +++ b/locale/de.po @@ -4362,7 +4362,7 @@ msgstr "" #: ../share/roundup/templates/classic/html/keyword.item.html:51 msgid "Continue adding keywords." -msgstr "Fügen Sie weitere Keywords hinzu." +msgstr "Fügen Sie weitere Schlüsselwörter hinzu." #: ../share/roundup/templates/classic/html/msg.index.html:3 #: ../share/roundup/templates/devel/html/msg.index.html:3 From 9f27d740fa307df3fb415f757f9c20960ccde871 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 20:45:25 -0400 Subject: [PATCH 35/71] refactor: parameterize the root prefix number of characters To make i18n work on windows, we need to clean the three character root elements(drive letter, colon, backslash) from the front of the various paths. Parameterize the number of chars. Old way hard coded to 1, which leaves :/ in the path and generates bad LOCALE_DIRS. From issues getting Roundup running on windows discussed on mailing list by Simon Eigeldinger. Thread starts with: https://sourceforge.net/p/roundup/mailman/message/41557096/ subject: Installing Roundup on Windows 2023-10-05. --- roundup/i18n.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/roundup/i18n.py b/roundup/i18n.py index 4ada7310..fe4a176e 100644 --- a/roundup/i18n.py +++ b/roundup/i18n.py @@ -52,8 +52,11 @@ # locale root is prefix/share/locale. if os.name == "nt": _mo_path = [".."] * 4 + ["share", "locale"] + root_prefix_chars = 3 # remove c:\ or other drive letter else: _mo_path = [".."] * 5 + ["share", "locale"] + root_prefix_chars = 1 # remove / + _mo_path = os.path.normpath(os.path.join(msgfmt.__file__, *_mo_path)) if _mo_path not in LOCALE_DIRS: LOCALE_DIRS.append(_mo_path) @@ -67,18 +70,18 @@ for _N in 1, 2: path = os.path.dirname(path) # path is /usr/local/lib/python3.10/site-packages -_ldir = os.path.join(path, sys.prefix[1:], 'share', 'locale') +_ldir = os.path.join(path, sys.prefix[root_prefix_chars:], 'share', 'locale') if os.path.isdir(_ldir): LOCALE_DIRS.append(_ldir) # try other places locale files are hidden on install -_ldir = os.path.join(path, sys.prefix[1:], 'local', 'share', 'locale') +_ldir = os.path.join(path, sys.prefix[root_prefix_chars:], 'local', 'share', 'locale') if os.path.isdir(_ldir): LOCALE_DIRS.append(_ldir) try: - _ldir = os.path.join(path, sys.base_prefix[1:], 'local', 'share', 'locale') + _ldir = os.path.join(path, sys.base_prefix[root_prefix_chars:], 'local', 'share', 'locale') if os.path.isdir(_ldir): LOCALE_DIRS.append(_ldir) - _ldir = os.path.join(path, sys.base_prefix[1:], 'share', 'locale') + _ldir = os.path.join(path, sys.base_prefix[root_prefix_chars:], 'share', 'locale') if os.path.isdir(_ldir): LOCALE_DIRS.append(_ldir) except AttributeError: From e93968576b1c23f980a5b45222092c210dc7cc16 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 6 Oct 2023 21:43:56 -0400 Subject: [PATCH 36/71] fix: exit quickly on keyboard interrupt When exiting roundup_server (issue was seen in use of roundup_demo) using ^C, the keyboard interrupt message is generated but the application didn't exit until the 60 second socket timeout is complete. This seems to be more of an issue with Windows. With this change the socket is explicitly shut down telling the client what's happening. Then the file descriptor is closed. --- CHANGES.txt | 2 ++ roundup/scripts/roundup_server.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d03c2fc1..2b224e9a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -54,6 +54,8 @@ Fixed: argument. Roles are indexed by lower case role name. So 'security User' and 'security user' should generate the same output. (John Rouillard from issue on mailing list by Chuck Cunningham) +- make roundup-server exit more quickly on ^C. This seems to be + limited to windows. Features: diff --git a/roundup/scripts/roundup_server.py b/roundup/scripts/roundup_server.py index 7f034612..b9783f8f 100644 --- a/roundup/scripts/roundup_server.py +++ b/roundup/scripts/roundup_server.py @@ -1167,6 +1167,12 @@ def run(port=undefined, success_message=None): httpd.serve_forever() except KeyboardInterrupt: print('Keyboard Interrupt: exiting') + try: + httpd.socket.shutdown(socket.SHUT_RDWR) + except OSError: + # forced shutdown can throw an error. + # we don't care as we are going away. + pass httpd.socket.close() From 5a354201772c8a22c6463e48b250d3b187226338 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 8 Oct 2023 23:46:16 -0400 Subject: [PATCH 37/71] - https://github.com/roundup-tracker/roundup/pull/51 --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 68fa0208..f693282a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -97,7 +97,7 @@ jobs: # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: ${{ matrix.python-version }} allow-prereleases: true From d24abceaa19072b28e5c8ae0db4dd341597d14fc Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 8 Oct 2023 23:48:00 -0400 Subject: [PATCH 38/71] build(deps): bump ossf/scorecard-action from 2.2.0 to 2.3.0 - https://github.com/roundup-tracker/roundup/pull/52 [skip travis] --- .github/workflows/ossf-scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index a1b2fc88..eb6ac18c 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0 + uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 with: results_file: results.sarif results_format: sarif From 56cb0ad25d95d66123b05919b025fd7b92b0e34f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 10 Oct 2023 20:33:22 -0400 Subject: [PATCH 39/71] clarify doc on dispatcher_email config setting. An issue was brought up on the mailing list. https://sourceforge.net/p/roundup/mailman/message/43383465/ The description of dispatcher_email sounds like it should be sent email on issue creation. That's not it's role. Try to make it's role more obvious. Fix config.ini and reference.txt description. Add the newissuecopy.py detector to send email on the creation of an issue --- doc/reference.txt | 8 ++++---- roundup/configuration.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/reference.txt b/doc/reference.txt index bcb4c354..33ef59bd 100644 --- a/doc/reference.txt +++ b/doc/reference.txt @@ -144,10 +144,10 @@ Section **main** below is used. dispatcher_email -- ``roundup-admin`` - The 'dispatcher' is a role that can get notified of new items to the - database. It is used by the ERROR_MESSAGES_TO config setting. If the - email address doesn't contain an ``@`` part, the MAIL_DOMAIN defined - below is used. + The 'dispatcher' is a role that can get notified when errors occur + while sending email to a user. It is used by the ERROR_MESSAGES_TO config + setting. If the email address doesn't contain an ``@`` part, the + MAIL_DOMAIN defined below is used. email_from_tag -- default *blank* Additional text to include in the "name" part of the From: address used diff --git a/roundup/configuration.py b/roundup/configuration.py index 2ec27e6a..ece431c7 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -1026,7 +1026,7 @@ def str2value(self, value): "mail -> domain is added."), (MailAddressOption, "dispatcher_email", "roundup-admin", "The 'dispatcher' is a role that can get notified\n" - "of new items to the database.\n" + "when errors occur while sending email to a user.\n" "It is used by the ERROR_MESSAGES_TO config setting.\n" "If no domain is specified then the config item\n" "mail -> domain is added."), From 3a1102c97463325b2a357ffa22eb4b3b3fb59e2a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 15 Oct 2023 20:56:50 -0400 Subject: [PATCH 40/71] Update doc on --build-arg="source=pip_sdist" for docker. --- doc/installation.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/installation.txt b/doc/installation.txt index b0295f4e..239b2714 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -1754,11 +1754,13 @@ You can build a docker container in one of 4 modes defined by the by some Python users. ``--build-arg="source=pip_sdist"`` - Disabled. + Disabled - hopefully it will be available in the future. This is meant for maintainer/developer use. It installs using pip from a source distribution (sdist) tarball built by following the RELEASE.txt. It is meant for testing - releases. Normal users/admins should not use it. + releases or building a docker image that installs a new pending + source distribution release. Normal users/admins should not use it. + Use ``local`` or ``pip_local`` instead. Build a docker container using the code in the current directory, with this build command from the top of the source tree:: From 74c224404d46f61685c497e9cfb94eafb2956c8d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 15 Oct 2023 21:04:32 -0400 Subject: [PATCH 41/71] doc: remove charater breaking formatting. --- share/man/man1/roundup-mailgw.1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/man/man1/roundup-mailgw.1 b/share/man/man1/roundup-mailgw.1 index 240122f5..95d3e086 100644 --- a/share/man/man1/roundup-mailgw.1 +++ b/share/man/man1/roundup-mailgw.1 @@ -1,4 +1,4 @@ ->.TH ROUNDUP-MAILGW 1 "27 October 2022" +.TH ROUNDUP-MAILGW 1 "27 October 2022" .SH NAME roundup-mailgw \- mail gateway for roundup .SH SYNOPSIS From a0bb26b5e51d568f7052415e0172a81cec7ade2b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 16 Oct 2023 22:19:45 -0400 Subject: [PATCH 42/71] fix: roundup_start script exited for 2.3.0 roundup. I added 'set -eo pipefail' and must have missed a test case that caused demo mode to exit. Fix this. --- scripts/Docker/roundup_start | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 21fd8624..c6cd7271 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -15,6 +15,7 @@ if ! [ -z "$SHELL_DEBUG" ]; then fi trap exit INT +trap "echo 'Exiting on pipefail'" ERR do_exit=0 @@ -38,7 +39,10 @@ for tracker_spec in "$@"; do # if it's a request to start in demo/shell mode: case "$tracker" in demo) - version=$(roundup-admin -v | grep 2.2.0) + # if grep does not find 2.2.0 it exits non-zero and + # because of "-eo pipefail" the script exits. So || true + # so pipeline exits true. + version=$(roundup-admin -v | grep '2\.2\.0' || true) if [ -n "$version" ]; then printf "\nRoundup version: %s does not support docker demo mode\n" "$version" printf "Try building with a version newer than 2.2.0.\n" From 5d24bae235c0913c313a861a42ac7070c86929af Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 16 Oct 2023 22:50:14 -0400 Subject: [PATCH 43/71] doc: add attribution for change. --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2b224e9a..596485a7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -55,7 +55,7 @@ Fixed: User' and 'security user' should generate the same output. (John Rouillard from issue on mailing list by Chuck Cunningham) - make roundup-server exit more quickly on ^C. This seems to be - limited to windows. + limited to windows. (John Rouillard) Features: From 562748847c00c1b4e54fae8ad88bcb0b8f434a95 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 18 Oct 2023 12:28:13 -0400 Subject: [PATCH 44/71] doc: fix description tags on web page. --- website/www/index.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/www/index.txt b/website/www/index.txt index 014c60d8..3ee31e09 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -6,14 +6,14 @@ Roundup Issue Tracker :title: Roundup Issue Tracker :description: A simple-to-use and -install issue-tracking system with command-line, web, REST, XML-RPC and e-mail interfaces. - Adaptable to many uses cases. Allows you to customise the look + Adaptable to many use cases. Allows you to customize the look and feel and implement different workflows. :og\:type: website :og\:url: https://www.roundup-tracker.org/ :og\:title: Roundup Issue Tracker :og\:description: A simple-to-use and -install issue-tracking system with command-line, web, REST, XML-RPC and e-mail interfaces. - Adaptable to many uses cases. Allows you to customise the look + Adaptable to many use cases. Allows you to customize the look and feel and implement different workflows. :og\:image: https://www.roundup-tracker.org/_images/index_logged_out.png From 32ff10513634b601791ffa1a5a2c3570181446d4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 18 Oct 2023 12:34:23 -0400 Subject: [PATCH 45/71] chore: upgrade to newest python:3-alpine. --- scripts/Docker/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index c960db4c..129b12cc 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -23,7 +23,7 @@ ARG source=local # parameterize the sha256 sum to pin version of python:3-alpine # Must use the same version in both build stages. -ARG SHA256=5d769f990397afbb2aca24b0655e404c0f2806d268f454b052e81e39d87abf42 +ARG SHA256=ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac # Set to any non-empy value to enable shell debugging for troubleshooting ARG VERBOSE= @@ -34,7 +34,7 @@ ARG appdir=/usr/src/app # Python version as a.b Used as path component for # installation directory and COPY from install dir # in second build stage. -ARG pythonversion=3.11 +ARG pythonversion=3.12 #FROM python:3-alpine via SHA256 sum FROM python@sha256:$SHA256 as build @@ -83,6 +83,7 @@ ENV PIP_ROOT_USER_ACTION=ignore SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG VERBOSE +ARG pythonversion RUN [ -z "${VERBOSE}" ] || set -xv; \ CWD="$PWD" && \ @@ -95,7 +96,7 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ else \ echo "Nothing to pip update"; \ fi; \ - ls -l /usr/local/lib/python3.11/site-packages; \ + ls -l /usr/local/lib/python${pythonversion}/site-packages; \ VER=$(apk list -I 'xapian-core-dev' | \ sed 's/^xapian-core-dev-\([0-9.]*\)-.*/\1/') && \ cd /tmp && \ From 3da9b3d01b8a55167ff3f095b4cb69d2b251bc30 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 18 Oct 2023 15:49:53 -0400 Subject: [PATCH 46/71] fix: database error handling during import of a non-user item The code to handle the case of retired and active users causing errors during import can get called on other db errors as well (e.g. out of memory in postgresql). The code that trys to detect/repair the out of order user case hard coded the db.user class. This crashed when a db error occurred when loading another data object (e.g. msg). Fix the crash by using the proper db object class for the object class being loaded. Credit to Norbert Schlemmer for finding this. --- CHANGES.txt | 3 +++ roundup/backends/rdbms_common.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 596485a7..720efca1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -56,6 +56,9 @@ Fixed: Rouillard from issue on mailing list by Chuck Cunningham) - make roundup-server exit more quickly on ^C. This seems to be limited to windows. (John Rouillard) +- Fix error handling so failure during import of a non-user item + doesn't cause a second traceback. (Found by Norbert Schlemmer, fix + John Rouillard) Features: diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 6839c43f..bee9f586 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -3298,11 +3298,12 @@ def import_list(self, propnames, proplist): logger.info('Attempting to handle import exception ' 'for id %s: %s' % (newid, e)) - keyname = self.db.user.getkey() + classdb = self.db.getclass(self.classname) + keyname = classdb.getkey() if has_node or not keyname: # Not an integrity error raise self.db.restore_connection_on_error() - activeid = self.db.user.lookup(d[keyname]) + activeid = classdb.lookup(d[keyname]) self.db.sql(retired_sql, (-1, activeid)) # clear the active node # this can only happen on an addnode, so retry try: From c56d29ec57d8b32ca3c0d1b35ce1f1ebbd0fb032 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 18 Oct 2023 18:56:19 -0400 Subject: [PATCH 47/71] test: add explicit check for imported data. Make sure the duplicate user is properly retreived using lookup by username. --- test/db_test_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/db_test_base.py b/test/db_test_base.py index 4b04584d..df913abb 100644 --- a/test/db_test_base.py +++ b/test/db_test_base.py @@ -3059,6 +3059,9 @@ def testImportExport(self): # This is needed, otherwise journals won't be there for anydbm self.db.commit() + + self.assertEqual(self.db.user.lookup("duplicate"), active_dupe_id) + finally: shutil.rmtree('_test_export') From 69c68b5a6f4a24069a1654e1fe1ce3d09210974d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 19 Oct 2023 14:07:56 -0400 Subject: [PATCH 48/71] doc: document the messages generated from import/export The numbers reported when exporting/importing don't make sense unless you know how they are generated and used. Document that. Norbert Schlemmer asked about it while debugging an import issue with postgresql. So document it since I did the work. --- doc/admin_guide.txt | 67 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index f6ee06e2..523b731a 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -1475,6 +1475,73 @@ Also the tautological:: Remember the roundup commands that accept multiple designators accept them ',' separated so using '-dc' is almost always required. +A Note on Import and Export +--------------------------- + +This is a little in the weeds, but I have noticed this and was asked +about it so I am documenting it for the future. + +Running ``roundup-admin`` with ``-V`` to get additional info when +importing/exporting the tracker generates three types of messages. + +For example:: + + $ roundup-admin -i tracker -V export ./myExport + Exporting priority - 5 + Exporting Journal for priority + Exporting status - 1 + Exporting Journal for status + [...] + + + $ roundup-admin -i tracker -V import ./myExport + Importing priority - 7 + setting priority 8 + Importing status - 8 + setting status 9 + [...] + +Note the numbers for status. Exported ends up at 1, Imported ends up +at 8 and setting chooses 9. These numbers are derived differently and +used differently. You can't directly compare them. + +``Exporting issue - XXX``: + + ``XXX`` is the id number of the node being exported/processed from + the database. The order is determined by sorting by the key of the + class (as set by sortkey). If the class key is 'id', then it's a + string sort so '9' comes before '1009'. You might notice if the + export is slow the numbers jumping around. + + It does not usually end up as the total number of nodes + exported. However if it crashes, you know what node it was + processing at the time. + + In the example above, the status node with id 1 was the last one + when sorted alphabetically by name. + +``Importing - XXX``: + + ``XXX`` is the number of the node (not the node id) being + imported/currently processed at line XXX+1 in the file. It is an + incrementing number starting at 0 and never jumps around. Value 0 + is consumed when reading the header and not displayed. The final + value is the same as the number of objects and one less then the + number of lines in the file. If it crashes, you were processing + the line at XXX+1. + +``setting XXX``: + + ``XXX`` in the setting line should always be one more than the + number of imported objects. The setting value is the id for the + next created object of that type. So in theory the Importing + number should be one less than the setting number. + + However under certain circumstances, Roundup can skip an id + number. This can lead to a difference of more than 1 between the + Importing and setting numbers. It's not a problem. However setting + can (and must) always be higher than the Importing number. + .. _`customisation documentation`: customizing.html .. _`reference documentation`: reference.html From 63439b3900a424c5991395065e1cc78824cea939 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 19 Oct 2023 16:11:25 -0400 Subject: [PATCH 49/71] fix: out of memory error when importing under postgresql If you try importing more than 20k items under postgresql you can run out of memory: psycopg2.errors.OutOfMemory: out of shared memory HINT: You might need to increase max_locks_per_transaction. Tuning memory may help, it's unknown at this point. This checkin forces a commit to the postgres database after 10,000 rows have been added. This clears out the savepoints for each row and starts a new transaction. back_postgresql.py: Implement commit mechanism in checkpoint_data(). Add two class level attributes for tracking the number of savepoints and the limit when the commit should happen. roundup_admin.py: implement pragma and dynamically create the config item RDBMS_SAVEPOINT_LIMIT used by checkpoint_data. Also fixed formatting of descriptions when using pragma list in verbose mode. admin_guide.txt, upgrading.txt: Document change and use of pragma savepoint_limit in roundup-admin for changing the default of 10,000. test/db_test_base.py: add some more asserts. In existing testAdminImportExport, set the savepoint limit to 5 to test setting method and so that the commit code will be run by existing tests. This provides coverage, but does not actually test that the commit is done every 5 savepoints 8-(. The verification of every 5 savepoints was done manually using a pdb breakpoint just before the commit. acknowledgements.txt: Added 2.4.0 section mentioning Norbert as he has done a ton of testing with much larger datasets than I can test with. --- CHANGES.txt | 3 ++ doc/acknowledgements.txt | 18 +++++++++ doc/admin_guide.txt | 10 +++++ doc/upgrading.txt | 14 +++++++ roundup/admin.py | 44 +++++++++++++------- roundup/backends/back_postgresql.py | 62 ++++++++++++++++++++++++----- test/db_test_base.py | 14 +++++++ 7 files changed, 140 insertions(+), 25 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 720efca1..a1f4e050 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -59,6 +59,9 @@ Fixed: - Fix error handling so failure during import of a non-user item doesn't cause a second traceback. (Found by Norbert Schlemmer, fix John Rouillard) +- Handle out of memory error when importing large trackers in + PostgreSQL. (Found by Norbert Schlemmer, extensive testing by + Norbert, fix John Rouillard) Features: diff --git a/doc/acknowledgements.txt b/doc/acknowledgements.txt index e5e307d4..5277e90a 100644 --- a/doc/acknowledgements.txt +++ b/doc/acknowledgements.txt @@ -16,6 +16,24 @@ ideas and everything else that helped! .. _`Announcement with changelog for current release.`: announcement.html +2.4 +--- + +2.4.0 +~~~~~ + +Maintainer: John Rouillard + +Release Manager: John Rouillard + +Developer activity by changesets:: + + TBD + +Other contributers + +Norbert Schlemmer + 2.3 --- diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index 523b731a..9b06239a 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -962,6 +962,16 @@ Migrating Backends move the new tracker home into its place. 9. Restart web and email frontends. +If you are importing into PostgreSQL, it autocommits the data every +10000 objects/rows by default. This can slow down importing, but it +prevents an out of memory error caused by using a savepoint for each +object. You can control the commit frequency by using:: + + pragma savepoint_limit=20000 + +to set a higher or lower number in roundup-admin. In this example a +commit will be done every 20,000 objects/rows. The pragma can also be +set on the roundup-admin command line as described below. Moving a Tracker ---------------- diff --git a/doc/upgrading.txt b/doc/upgrading.txt index 31e04632..cf225212 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -141,6 +141,20 @@ It is unlikey that you will care unless you have done some expert level Roundup customization. If you have, use one of the imports above if you plan on running on Python 3.13 (expected in 2024) or newer. +Fixing PostgreSQL Out of Memory Errors when Importing Tracker (info) +-------------------------------------------------------------------- + +Importing a tracker into PostgreSQL can run out of memory with the +error:: + + psycopg2.errors.OutOfMemory: out of shared memory + HINT: You might need to increase max_locks_per_transaction. + +before changing your PostgreSQL configuration, try changing the pragma +``savepoint_limit`` to a lower value. By default it is set to +``10000``. In some cases this may be too high. See the `administration +guide`_ for further details. + .. index:: Upgrading; 2.2.0 to 2.3.0 Migrating from 2.2.0 to 2.3.0 diff --git a/roundup/admin.py b/roundup/admin.py index 7902ed47..7d38db32 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -35,8 +35,9 @@ from roundup import date, hyperdb, init, password, token_r from roundup import __version__ as roundup_version import roundup.instance -from roundup.configuration import (CoreConfig, NoConfigError, OptionUnsetError, - OptionValueError, ParsingOptionError, UserConfig) +from roundup.configuration import (CoreConfig, NoConfigError, Option, + OptionUnsetError, OptionValueError, + ParsingOptionError, UserConfig) from roundup.i18n import _, get_translation from roundup.exceptions import UsageError from roundup.anypy.my_input import my_input @@ -108,6 +109,7 @@ def __init__(self): 'display_protected': False, 'indexer_backend': "as set in config.ini", '_reopen_tracker': False, + 'savepoint_limit': 10000, 'show_retired': "no", '_retired_val': False, 'verbose': False, @@ -116,25 +118,29 @@ def __init__(self): } self.settings_help = { 'display_header': - _("Have 'display designator[,designator*]' show header inside " - " []'s before items. Includes retired/active status."), + _("Have 'display designator[,designator*]' show header inside\n" + " []'s before items. Includes retired/active status.\n"), 'display_protected': - _("Have 'display designator' and 'specification class' show " - "protected fields: creator, id etc."), + _("Have 'display designator' and 'specification class' show\n" + " protected fields: creator, id etc.\n"), 'indexer_backend': - _("Set indexer to use when running 'reindex' NYI"), + _("Set indexer to use when running 'reindex' NYI\n"), '_reopen_tracker': - _("Force reopening of tracker when running each command."), - - 'show_retired': _("Show retired items in table, list etc. One of 'no', 'only', 'both'"), - '_retired_val': _("internal mapping for show_retired."), - 'verbose': _("Enable verbose output: tracing, descriptions..."), - - '_inttest': "Integer valued setting. For testing only.", - '_floattest': "Float valued setting. For testing only.", + _("Force reopening of tracker when running each command.\n"), + + 'savepoint_limit': + _("set the number of rows imported before a database commit is\n" + " done. Used only for imports on PostgreSQL.\n"), + 'show_retired': _("Show retired items in table, list etc. " + "One of 'no', 'only', 'both'\n"), + '_retired_val': _("internal mapping for show_retired.\n"), + 'verbose': _("Enable verbose output: tracing, descriptions...\n"), + + '_inttest': "Integer valued setting. For testing only.\n", + '_floattest': "Float valued setting. For testing only.\n", } def get_class(self, classname): @@ -1049,6 +1055,14 @@ def do_import(self, args, import_files=True): if hasattr(csv, 'field_size_limit'): csv.field_size_limit(self.db.config.CSV_FIELD_SIZE) + # default value is 10000, only go through this if default + # is different. + if self.settings['savepoint_limit'] != 10000: + self.db.config.add_option(Option(self.db.config, + "rdbms", "savepoint_limit")) + self.db.config.options["RDBMS_SAVEPOINT_LIMIT"].set( + self.settings['savepoint_limit']) + # directory to import from dir = args[0] diff --git a/roundup/backends/back_postgresql.py b/roundup/backends/back_postgresql.py index 4da8c12f..09848f0c 100644 --- a/roundup/backends/back_postgresql.py +++ b/roundup/backends/back_postgresql.py @@ -152,12 +152,25 @@ class Database(rdbms_common.Database): holds the value for the type of db. It is used by indexer to identify the database type so it can import the correct indexer module when using native text search mode. + + import_savepoint_count: + count the number of savepoints that have been created during + import. Once the limit of savepoints is reached, a commit is + done and this is reset to 0. + """ arg = '%s' dbtype = "postgres" + import_savepoint_count = 0 + + # Value is set from roundup-admin using db.config["RDBMS_SAVEPOINT_LIMIT"] + # or to the default of 10_000 at runtime. Use 0 here to trigger + # initialization. + savepoint_limit = 0 + # used by some code to switch styles of query implements_intersect = 1 @@ -218,20 +231,49 @@ def checkpoint_data(self, savepoint="importing"): of operation in exception handler because postgres requires a rollback in case of error generating exception. Used with - restore_connecion_on_error to handle uniqueness + restore_connection_on_error to handle uniqueness conflict in import_table(). + + Savepoints take memory resources. Postgres keeps all + savepoints (rather than overwriting) until a + commit(). Commit every ~10,000 savepoints to prevent + running out of memory on import. + + NOTE: a commit outside of this method will not reset the + import_savepoint_count. This can result in an unneeded + commit on a new cursor (that has no savepoints) as there is + no way to find out if there is a savepoint or how many + savepoints are opened on a db connection/cursor. + + Because an import is a one shot deal and not part of a long + running daemon (e.g. the roundup-server), I am not too + worried about it. It will just slow the import down a tad. """ - # Savepoints take resources. Postgres keeps all - # savepoints (rather than overwriting) until a - # commit(). If an import fails because of a resource - # issue with savepoints, uncomment this line. I - # expect it will slow down the import but it should - # eliminate any issue with stored savepoints and - # resource use. - # - # self.sql('RELEASE SAVEPOINT %s' % savepoint) + self.sql('SAVEPOINT %s' % savepoint) + self.import_savepoint_count += 1 + + if not self.savepoint_limit: + if "RDBMS_SAVEPOINT_LIMIT" in self.config.keys(): + # note this config option is created on the fly + # by admin.py::do_import. It is never listed in + # config.ini. + self.savepoint_limit = self.config["RDBMS_SAVEPOINT_LIMIT"] + else: + self.savepoint_limit = 10000 + + if self.import_savepoint_count > self.savepoint_limit: + # track savepoints and commit every 10000 (or user value) + # so we don't run postgres out of memory. An import of a + # customer's tracker ran out of memory after importing + # ~23000 items with: psycopg2.errors.OutOfMemory: out of + # shared memory HINT: You might need to increase + # max_locks_per_transaction. + + self.commit() + self.import_savepoint_count = 0 + def restore_connection_on_error(self, savepoint="importing"): """Postgres leaves a connection/cursor in an unusable state after an error. Rollback the transaction to a diff --git a/test/db_test_base.py b/test/db_test_base.py index df913abb..29534deb 100644 --- a/test/db_test_base.py +++ b/test/db_test_base.py @@ -3061,6 +3061,7 @@ def testImportExport(self): self.db.commit() self.assertEqual(self.db.user.lookup("duplicate"), active_dupe_id) + self.assertEqual(self.db.user.is_retired(retired_dupe_id), True) finally: shutil.rmtree('_test_export') @@ -3151,12 +3152,25 @@ def stderrwrite(s): self.assertRaises(csv.Error, tool.do_import, ['_test_export']) self.nukeAndCreate() + + # make sure we have an empty db + with self.assertRaises(IndexError) as e: + # users 1 and 2 always are created on schema load. + # so don't use them. + self.db.user.getnode("5").values() + self.db.config.CSV_FIELD_SIZE = 3200 tool = roundup.admin.AdminTool() tool.tracker_home = home tool.db = self.db + # Force import code to commit when more than 5 + # savepoints have been created. + tool.settings['savepoint_limit'] = 5 tool.verbose = False tool.do_import(['_test_export']) + + # verify the data is loaded. + self.db.user.getnode("5").values() finally: roundup.admin.sys = sys shutil.rmtree('_test_export') From 435985ab7c5a8a15fb16c065df68297ad8a0602a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 19 Oct 2023 17:30:22 -0400 Subject: [PATCH 50/71] chore: fix after_success for travis. Did not run last commit. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b38a9f5e..f3116772 100644 --- a/.travis.yml +++ b/.travis.yml @@ -165,7 +165,7 @@ script: - PATH=$VIRTUAL_ENV/bin:$PATH - export LD_LIBRARY_PATH=$VIRTUAL_ENV/lib:$LD_LIBRARY_PATH - python -c "import sys; print('python version ', sys.version)" - - set -xv; if [[ "$TRAVIS_PYTHON_VERSION" != "2."* ]]; then + - if [[ "$TRAVIS_PYTHON_VERSION" != "2."* ]]; then python -m pytest -r a \ --durations=20 \ -W default \ @@ -183,7 +183,7 @@ script: after_success: # from https://docs.codecov.com/docs/codecov-uploader#integrity-checking-the-uploader - - curl https://keybase.io/codecovsecurity/pgp_keys.asc | + - curl https://keybase.io/codecovsecurity/pgp_keys.asc | \ gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM From 1ea5579953dd7b5d9a6e1c5a8ae9adbbfaf2e70a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 09:23:16 -0400 Subject: [PATCH 51/71] issue2551296 - from mock import Mock? Fix mock import for newer python3 to use unittest.mock rather than mock. --- CHANGES.txt | 3 +++ test/test_hyperdbvals.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index a1f4e050..c7874ae5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -62,6 +62,9 @@ Fixed: - Handle out of memory error when importing large trackers in PostgreSQL. (Found by Norbert Schlemmer, extensive testing by Norbert, fix John Rouillard) +- use unittest.mock rather than mock for + test/test_hyperdbvals.py. (found by Ralf Schlatterbeck. Fix John + Rouillard) Features: diff --git a/test/test_hyperdbvals.py b/test/test_hyperdbvals.py index 6c8af65c..30fdba6e 100644 --- a/test/test_hyperdbvals.py +++ b/test/test_hyperdbvals.py @@ -10,7 +10,11 @@ import unittest, os, shutil, errno, sys, difflib, re from hashlib import sha1 -from mock import Mock +try: + from unittest.mock import Mock +except ImportError: + # python 2.7 + from mock import Mock from roundup import init, instance, password, hyperdb, date From 87a40752713ed54491a8d4ef7dca50cbab45971f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 09:32:06 -0400 Subject: [PATCH 52/71] doc: add -it to docker command for running trivy It makes the progress bar display correctly. --- RELEASE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.txt b/RELEASE.txt index 3e216814..d97e56cb 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -236,7 +236,7 @@ Roundup release checklist: Also can scan (optionally) using trivy: - docker run --rm --volume \ + docker run -it --rm --volume \ /var/run/docker.sock:/var/run/docker.sock \ --name trivy aquasec/trivy:latest image rounduptracker/roundup:2.2.0 From a0617d200eafd7d313d4165ac82552b6872f8e48 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:40:48 -0400 Subject: [PATCH 53/71] fix: use variable template sub for printf. Don't use $1 inside printf string. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index c6cd7271..128f7705 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -78,11 +78,11 @@ for tracker_spec in "$@"; do anydbm|sqlite) backend="$arg";; postgres|mysql) - printf "demo mode only supports sqlite or anydbm backends, not $1. Exiting." + printf "demo mode only supports sqlite or anydbm backends, not %s. Exiting." "$1" exit 1;; nuke) nuke="$arg";; - *) printf "Unknown argument $1.\n" + *) printf "Unknown argument %s.\n" "$1" printf "Usage: demo [template] [db]\n" printf " template: one of " printf "classic devel jinja2 minimal responsive\n" From 03f68179e2d3c16a766caa51b7eba96d48ef2dc6 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:42:56 -0400 Subject: [PATCH 54/71] fix: use read -r to remove \ special meaning Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 128f7705..f1486902 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -29,7 +29,7 @@ for tracker_spec in "$@"; do # IFS== set a=b doesn't just assign $1 and $2 in busybox ash # it also clobbers '$@'. 'printf mumble | read' starts read in a # subshell so vars are not available in parent. - IFS="=" read tracker directory <<- EOE + IFS="=" read -r tracker directory <<- EOE $tracker_spec EOE # ^ is a tab for use with <<- From 917310bb5af9892e7673893e2752647cc7ff081b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:45:05 -0400 Subject: [PATCH 55/71] fix: quote variable substitution To prevent issues with spaces in the variable. The var should be a single word but it pays to be careful. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index f1486902..77fc0c9a 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -120,11 +120,11 @@ for tracker_spec in "$@"; do exec roundup-demo \ -B 0.0.0.0 \ -p 8080 \ - -b $backend \ - --urlport $PORT_8080 \ - -t $template \ + -b "$backend" \ + --urlport "$PORT_8080" \ + -t "$template" \ tracker/demo \ - $nuke + "$nuke" fi ;; shell) From e737d1e3168fd0a360556adca9ef6efb6c027752 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:50:39 -0400 Subject: [PATCH 56/71] fix: move POSIX unsupported items outside of set -e If the non-poisx fail under a POSIX shell, allow the script to still run but with the errors. So all non-POSIX supported stuff goes before set -e. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 77fc0c9a..88912074 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -7,15 +7,20 @@ # "$@" should be one of the recognized keywords or arguments for # roundup-server including one or more tracker tracker=directory pairs. -# exit on errors -set -eo pipefail +# exit on errors; POSIX (e.g. dash as on ubuntu) doesn't support +# pipefail. So setting it will fail. Allow that to happen then +# set exit on error so the script will still run. +set -o pipefail +# not supported by POSIX shell, but then neither is pipefail. +trap "echo 'Exiting on pipefail'" ERR + +set -e if ! [ -z "$SHELL_DEBUG" ]; then set -xv fi trap exit INT -trap "echo 'Exiting on pipefail'" ERR do_exit=0 From 7147ac72cd45b6394c6427d163e6fec36f193ff1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:53:22 -0400 Subject: [PATCH 57/71] fix: use POSIX supported test a = b rather than a == b. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 88912074..2e5ca447 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -179,7 +179,7 @@ parameters.\n" "$directory" # we have a valid config.ini so init database if not done # if we get errors, the db directory should be missing # and we print an error. - if [ $do_exit == 0 -a ! -e "$directory/db" ]; then + if [ $do_exit = 0 -a ! -e "$directory/db" ]; then printf "Initializing tracker %s\n" "$tracker" if ! roundup-admin -i "$directory" init; then # something went wrong. @@ -193,7 +193,7 @@ parameters.\n" "$directory" done # for "$@" # if any config.ini needs editing don't start up. -if [ $do_exit == 0 ]; then +if [ $do_exit = 0 ]; then # make roundup-server process pid 1 with exec exec roundup-server -n 0.0.0.0 "$@" fi From 2deff96e141a3441396fbfb19257814dafb33ec2 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 21:55:24 -0400 Subject: [PATCH 58/71] fix: use [ -n ... ] rather than ! [ -z ...] Style fix. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index 2e5ca447..eac75020 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -16,7 +16,7 @@ trap "echo 'Exiting on pipefail'" ERR set -e -if ! [ -z "$SHELL_DEBUG" ]; then +if [ -n "$SHELL_DEBUG" ]; then set -xv fi From ea0a4a4bcda7fbf47a5a303bb4d353c5109fed42 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 22:00:16 -0400 Subject: [PATCH 59/71] fix: do not use -a as argument to test Apparently now well defined. Suggest using if [ ... ] && [ ... ]; then rather than if [ ... -a ... ]; then with complex ... expressions. Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/Docker/roundup_start b/scripts/Docker/roundup_start index eac75020..6a4b91d5 100755 --- a/scripts/Docker/roundup_start +++ b/scripts/Docker/roundup_start @@ -102,7 +102,7 @@ for tracker_spec in "$@"; do # run demo make sure to set bind address -B to 0.0.0.0 # otherwise we can never make it out of the docker container. # use -p to force port to match the exported docker port. - if [ -f tracker/demo/config.ini -a -z "$nuke" ]; then + if [ -f tracker/demo/config.ini ] && [ -z "$nuke" ]; then if [ "$demoArgs" -ne 0 ]; then printf "Error: backend and template arguments to demo " printf "are invalid if a tracker\nis configured and " @@ -179,7 +179,7 @@ parameters.\n" "$directory" # we have a valid config.ini so init database if not done # if we get errors, the db directory should be missing # and we print an error. - if [ $do_exit = 0 -a ! -e "$directory/db" ]; then + if [ $do_exit = 0 ] && ! [ -e "$directory/db" ]; then printf "Initializing tracker %s\n" "$tracker" if ! roundup-admin -i "$directory" init; then # something went wrong. From 9ffa1d076aab33aba764c5994077532ebc71e94b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 24 Oct 2023 22:03:55 -0400 Subject: [PATCH 60/71] fix: quote variable substitution to prevent whitespace issue Again the tracker variable should not have embeded whitespace but.... Courtesy of https://www.shellcheck.net/ --- scripts/Docker/roundup_healthcheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Docker/roundup_healthcheck b/scripts/Docker/roundup_healthcheck index 512183d9..b488a237 100755 --- a/scripts/Docker/roundup_healthcheck +++ b/scripts/Docker/roundup_healthcheck @@ -4,4 +4,4 @@ # returns last one for testing that server is up. Does not test # each tracker. tracker=$(ps -ef | sed -ne '/roundup-server/s/^.*\s\(\w*\)=.*$/\1/p') -wget -q -O /dev/null --no-verbose http://localhost:8080/${tracker:-demo}/ +wget -q -O /dev/null --no-verbose http://localhost:8080/"${tracker:-demo}"/ From baf46327d1e0331a3cee030884ba9b25e54f6a49 Mon Sep 17 00:00:00 2001 From: Ralf Schlatterbeck Date: Wed, 25 Oct 2023 10:56:04 +0200 Subject: [PATCH 61/71] Add two small tests for link expressions This tests 'or' of querying a Link property and the negation of the 'or'. I added these when I wasn't sure it did the correct thing (it does). --- test/db_test_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/db_test_base.py b/test/db_test_base.py index 29534deb..1fec2229 100644 --- a/test/db_test_base.py +++ b/test/db_test_base.py @@ -2015,6 +2015,8 @@ def testFilteringLinkExpression(self): ae(filt(None, {a: ['1','-1']}), ['1','3','4']) ae(filt(None, {a: ['1','-1']}, ('+',a)), ['3','4','1']) ae(filt(None, {a: ['2','-1']}, ('+',a)), ['3','4','2']) + ae(filt(None, {a: ['2','-1','-4']}, ('+',a)), ['3','4','2']) + ae(filt(None, {a: ['2','-1','-4','-2']}, ('+',a)), ['1']) ae(filt(None, {a: ['1','-2']}), ['2','3','4']) ae(filt(None, {a: ['1','-2']}, ('+',a)), ['3','4','2']) ae(filt(None, {a: ['-1','-2']}, ('+',a)), ['1','2']) From ce2b540ac18d0854d8dc1ad5738c7dc54a1ba33b Mon Sep 17 00:00:00 2001 From: "Norbert Schlemmer Noschvie on github.com" Date: Wed, 25 Oct 2023 09:42:11 -0400 Subject: [PATCH 62/71] fix(docker): make healthcheck work when proxy is set In docker setups that use a proxy for internet access, exclude the wget healthcheck from using the proxy when connecting to localhost. Fix from Norbert Schlemmer. --- CHANGES.txt | 2 ++ scripts/Docker/roundup_healthcheck | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index c7874ae5..a32a98b2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -65,6 +65,8 @@ Fixed: - use unittest.mock rather than mock for test/test_hyperdbvals.py. (found by Ralf Schlatterbeck. Fix John Rouillard) +- disable proxy with wget in roundup_healthcheck. (Norbert SCHLEMMER + Noschvie on github.com) Features: diff --git a/scripts/Docker/roundup_healthcheck b/scripts/Docker/roundup_healthcheck index b488a237..dfa9dc25 100755 --- a/scripts/Docker/roundup_healthcheck +++ b/scripts/Docker/roundup_healthcheck @@ -4,4 +4,4 @@ # returns last one for testing that server is up. Does not test # each tracker. tracker=$(ps -ef | sed -ne '/roundup-server/s/^.*\s\(\w*\)=.*$/\1/p') -wget -q -O /dev/null --no-verbose http://localhost:8080/"${tracker:-demo}"/ +wget -q -O /dev/null --proxy off --no-verbose http://localhost:8080/"${tracker:-demo}"/ From 93a4d616fced1d0fb3d610b4e0e528ddfbb29ffc Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 25 Oct 2023 13:12:18 -0400 Subject: [PATCH 63/71] fix: rest - set self.start from client.start Make elasped time include time since client was initialized. So elapsed is as close as we can get to an overall request service time. May need to add rest_elapsed or some other subsystem based timers as we try to track a possible performance regression in 2.3.0. --- roundup/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roundup/rest.py b/roundup/rest.py index 06a5d963..34d9b5c6 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -433,7 +433,7 @@ def __init__(self, client, db): self.db = db self.translator = client.translator # record start time for statistics reporting - self.start = time.time() + self.start = client.start # disable stat reporting by default enable with @stats=True # query param self.report_stats = False From 81d61174b8dd29e9716eb55d9d2f82fc8d7c3e87 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 26 Oct 2023 11:24:04 -0400 Subject: [PATCH 64/71] fix: add support for dicttoxml2.py The older dicttoxml.py uses a type alias collections.Iterator that is removed post Python 3.10. Add support for dictoxml.py updated replacement. Norbert SCHLEMMER found it when testing the arm docker under 3.12. --- CHANGES.txt | 4 ++++ doc/rest.txt | 16 ++++++++++------ roundup/rest.py | 18 +++++++++++++----- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a32a98b2..38dda141 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -67,6 +67,10 @@ Fixed: Rouillard) - disable proxy with wget in roundup_healthcheck. (Norbert SCHLEMMER Noschvie on github.com) +- support dicttoxml2.py for Roundup running on 3.7 and + newer. dicttoxml uses a type alias: collection.Iterator that is + dropped in Python 3.10. (found by Norbert SCHLEMMER, fix John + Rouillard) Features: diff --git a/doc/rest.txt b/doc/rest.txt index 8d34607d..b11a93d6 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -311,14 +311,14 @@ Response Formats The default response format is json. -If you add the ``dicttoxml.py`` module you can request XML formatted +If you add the ``dicttoxml2.py`` module you can request XML formatted data using the header ``Accept: application/xml`` in your request. Both output formats are similar in structure. -``dicttoxml.py`` should be installed in the Python install directory, -or the file can be added to the Roundup installation directory long -ide ``rest.py``. It can also be enabled on a per tracker basis by -adding ``dicttoxml.py`` to the lib directory in your tracker home (you +``dicttoxml2.py`` should be installed in the Python install directory, +or the file can be added to the Roundup installation directory along +side ``rest.py``. It can also be enabled on a per tracker basis by +adding ``dicttoxml2.py`` to the lib directory in your tracker home (you may need to create the directory). Then this can be added to `interfaces.py`_ to enable xml:: @@ -334,7 +334,7 @@ The rest interface accepts the http accept header and can include way to specify alternate acceptable response formats. To make testing from the browser easier, you can also append the -extension `.json` or `.xml` to the path component of the url. This +extension ``.json`` or ``.xml`` to the path component of the url. This will force json or xml (if supported) output. If you use an extension it takes priority over any accept headers. @@ -342,6 +342,10 @@ The rest interface returns status 406 if you use an unrecognized extension. You will also get a 406 status if none of the entries in the accept header are available or if the accept header is invalid. +Note: ``dicttoxml2.py`` is an updated version of ``dicttoxml.py``. If +you are still using Python 2.7 or 3.6, you should use +``dicttoxml.py``. + General Guidelines ------------------ diff --git a/roundup/rest.py b/roundup/rest.py index 34d9b5c6..b7782013 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -35,15 +35,23 @@ logger = logging.getLogger('roundup.rest') try: - # if dicttoxml installed in roundup directory, use it - from roundup.dicttoxml import dicttoxml + # if dicttoxml2 (or dicttoxml for Python <= 3.6) + # is installed in roundup directory, use it + from roundup.dicttoxml2 import dicttoxml except ImportError: try: # else look in sys.path - from dicttoxml import dicttoxml + from dicttoxml2 import dicttoxml except ImportError: - # else not supported - dicttoxml = None + try: + from roundup.dicttoxml import dicttoxml + except ImportError: + try: + # else look in sys.path + from dicttoxml import dicttoxml + except ImportError: + # else not supported + dicttoxml = None # Py3 compatible basestring try: From 74a3df2fad962fdd78293de82adbc4d74efb2ac9 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 26 Oct 2023 11:28:54 -0400 Subject: [PATCH 65/71] doc: clarify use of .xml or .json extensions in REST interface Document that /rest and /rest/data do not support extensions. --- doc/rest.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/rest.txt b/doc/rest.txt index b11a93d6..fe20e2bf 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -336,15 +336,17 @@ way to specify alternate acceptable response formats. To make testing from the browser easier, you can also append the extension ``.json`` or ``.xml`` to the path component of the url. This will force json or xml (if supported) output. If you use an extension -it takes priority over any accept headers. +it takes priority over any accept headers. Note the extension does not +work for the ``/rest`` or ``/rest/data`` paths. In these cases it +returs a 404 error. Adding the header ``Accept: application/xml`` +allows these paths to return xml data. The rest interface returns status 406 if you use an unrecognized extension. You will also get a 406 status if none of the entries in the accept header are available or if the accept header is invalid. Note: ``dicttoxml2.py`` is an updated version of ``dicttoxml.py``. If -you are still using Python 2.7 or 3.6, you should use -``dicttoxml.py``. +you are still using Python 2.7 or 3.6, you can use ``dicttoxml.py``. General Guidelines From 1bc1be9de20383f69a100c5e0d7a2a04083d140a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 26 Oct 2023 13:17:52 -0400 Subject: [PATCH 66/71] fix: duplicate password id generated for user.item.html Fix the user_confirm_input macro at the end of html/page.html to modify the id so it doesn't duplicate the one used for the regular password. --- CHANGES.txt | 2 ++ doc/upgrading.txt | 23 +++++++++++++++++++ .../roundup/templates/classic/html/page.html | 2 +- share/roundup/templates/devel/html/page.html | 2 +- .../roundup/templates/minimal/html/page.html | 2 +- .../templates/responsive/html/page.html | 2 +- 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 38dda141..5728a4e2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -71,6 +71,8 @@ Fixed: newer. dicttoxml uses a type alias: collection.Iterator that is dropped in Python 3.10. (found by Norbert SCHLEMMER, fix John Rouillard) +- fix repeated password id with user.item.html in all templates except + jinja2. (John Rouillard) Features: diff --git a/doc/upgrading.txt b/doc/upgrading.txt index cf225212..d4e0b128 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -112,6 +112,29 @@ values or if a value must be changed manually. This will insert the bad API login rate limiting settings. +Fix duplicate id for confirm password in user.item.html (optional) +------------------------------------------------------------------ + +The TAL macro ``user_confirm_input`` at the end of ``html/page.html`` +for all templates except ``jinja2`` sets the ``id`` of the ``Confirm +password`` input the same as the ``Login Password`` input. This +creates an HTML error. Two items must not have the same id. + +However browsers ignore the error and things still work. If you were +to use css or javascript to target the ``password`` id, it would not +work as expected. + +To fix this, change the line near the end of your tracker's +``html/page.html`` from:: + + tal:attributes="id name; name string:@confirm@$name; readonly not:edit_ok" value=""> + +to:: + + tal:attributes="id string:confirm_$name; name string:@confirm@$name; readonly not:edit_ok" value=""> + +This will change the id to ``confirm_password``. + Bad Login Rate Limiting and Locking (info) ------------------------------------------ diff --git a/share/roundup/templates/classic/html/page.html b/share/roundup/templates/classic/html/page.html index f49defa7..3b624a49 100644 --- a/share/roundup/templates/classic/html/page.html +++ b/share/roundup/templates/classic/html/page.html @@ -369,5 +369,5 @@

body title

+ tal:attributes="id string:confirm_$name; name string:@confirm@$name; readonly not:edit_ok" value=""> diff --git a/share/roundup/templates/devel/html/page.html b/share/roundup/templates/devel/html/page.html index 631c3b38..00ac1940 100644 --- a/share/roundup/templates/devel/html/page.html +++ b/share/roundup/templates/devel/html/page.html @@ -434,6 +434,6 @@

body title

+ tal:attributes="id string:confirm_$name; name string:@confirm@$name; readonly not:edit_ok" value=""/> diff --git a/share/roundup/templates/minimal/html/page.html b/share/roundup/templates/minimal/html/page.html index b3e51fbb..848c7597 100644 --- a/share/roundup/templates/minimal/html/page.html +++ b/share/roundup/templates/minimal/html/page.html @@ -341,5 +341,5 @@

body title

+ tal:attributes="id string:confirm_$name; name string:@confirm@$name; readonly not:edit_ok" value=""> diff --git a/share/roundup/templates/responsive/html/page.html b/share/roundup/templates/responsive/html/page.html index adeeb2dd..182a2dcf 100644 --- a/share/roundup/templates/responsive/html/page.html +++ b/share/roundup/templates/responsive/html/page.html @@ -449,4 +449,4 @@

body title

+ tal:attributes="id string:confirm_$name; name string:@confirm@$name; readonly not:edit_ok" value=""/> From 9e640721cb2ac08fa2301f8a5f1debfb35e90e98 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 29 Oct 2023 13:18:36 -0400 Subject: [PATCH 67/71] doc: add comment on method to set savepoint_limit dynamically savepoint_limit isn't a real config option as in config.ini. I generate one on the fly for use in the backend during import. While this is arguably a valid config option, it is also something that is niche (import a tracker to postgres) and adds clutter to config.ini. --- roundup/admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roundup/admin.py b/roundup/admin.py index 7d38db32..6d351407 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -1058,6 +1058,9 @@ def do_import(self, args, import_files=True): # default value is 10000, only go through this if default # is different. if self.settings['savepoint_limit'] != 10000: + # create a new option on the fly in the config under the + # rdbms section. It is used by the postgresql backend's + # checkpoint_data method. self.db.config.add_option(Option(self.db.config, "rdbms", "savepoint_limit")) self.db.config.options["RDBMS_SAVEPOINT_LIMIT"].set( From abdacee23fa1168fcae235329a10cbce6ab6c0be Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 29 Oct 2023 14:54:32 -0400 Subject: [PATCH 68/71] doc: add statement on less stable nature of using interfaces.py --- doc/reference.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/reference.txt b/doc/reference.txt index 33ef59bd..6c4b4b21 100644 --- a/doc/reference.txt +++ b/doc/reference.txt @@ -1561,8 +1561,17 @@ directories are not on the Python system path when interfaces.py is evaluated. You need to add library directories explictly by modifying sys.path. +Interfaces.py allows you to interact with any part of Roundup's +internals. These internals are not as stable as defined interfaces +(e.g. extensions. detectors, schema). So the code in interfaces.py is +more likely to need modification when upgrading from version to +version. While the developers attempt to keep the examples working, +it may make more sense to change the internals to make the code +clearer, add more features etc. + See `Changing How the Core Code Works -`_ for examples. +`_ for examples +of using interfaces.py. Database Content ================ From 20d498c751c3cb6d9a21c3df2d2bc72f7d105ae1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 30 Oct 2023 21:57:21 -0400 Subject: [PATCH 69/71] chore: update the sha256 sum for the newest python:3-alpine image --- scripts/Docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 129b12cc..8b4e6540 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -23,7 +23,7 @@ ARG source=local # parameterize the sha256 sum to pin version of python:3-alpine # Must use the same version in both build stages. -ARG SHA256=ae35274f417fc81ba6ee1fc84206e8517f28117566ee6a04a64f004c1409bdac +ARG SHA256=a5d1738d6abbdff3e81c10b7f86923ebcb340ca536e21e8c5ee7d938d263dba1 # Set to any non-empy value to enable shell debugging for troubleshooting ARG VERBOSE= From 4ca4e590636bea6a831c9d303a8b67801b713468 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 2 Nov 2023 18:55:47 -0400 Subject: [PATCH 70/71] fix: close file properly in indexer_dbm.py:save_index() Fix this error found in debug logs of gentoo packaging of round 2.2.0. /roundup/backends/indexer_dbm.py:253: ResourceWarning: unclosed file <_io.BufferedWriter name='test-index/indexes/index.db-'> open(self.indexdb+'-', 'wb').write(zlib.compress(marshal.dumps(dbfil))) Also added test that calls save_index(), reloads the index and tests that the original item. I am not sure how Gentoo hit this But they were missing a number of backends. So it's possible that indexer_dbm.py is not getting fully tested depending on what is installed on the system. Codecov from CI didnt show indexer_dbm.py:save_index() being covered. --- CHANGES.txt | 1 + roundup/backends/indexer_dbm.py | 4 +++- test/test_indexer.py | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5728a4e2..a2e3c3fc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -73,6 +73,7 @@ Fixed: Rouillard) - fix repeated password id with user.item.html in all templates except jinja2. (John Rouillard) +- fix unclosed file when saving index in indexer_dbm.py. (John Rouillard) Features: diff --git a/roundup/backends/indexer_dbm.py b/roundup/backends/indexer_dbm.py index 6923ab28..d3f9560b 100644 --- a/roundup/backends/indexer_dbm.py +++ b/roundup/backends/indexer_dbm.py @@ -250,7 +250,9 @@ def save_index(self): # First write the much simpler filename/fileid dictionaries dbfil = {'WORDS': None, 'FILES': self.files, 'FILEIDS': self.fileids} - open(self.indexdb+'-', 'wb').write(zlib.compress(marshal.dumps(dbfil))) + marshal_fh = open(self.indexdb+'-', 'wb') + marshal_fh.write(zlib.compress(marshal.dumps(dbfil))) + marshal_fh.close() # The hard part is splitting the word dictionary up, of course letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_" diff --git a/test/test_indexer.py b/test/test_indexer.py index b50bf250..0494673b 100644 --- a/test/test_indexer.py +++ b/test/test_indexer.py @@ -104,6 +104,30 @@ def test_basics(self): self.assertSeqEqual(self.dex.find(['blah', 'hello']), []) self.assertSeqEqual(self.dex.find([]), []) + def test_save_load(self): + + # only run for anydbm test + if ( not type(self) is IndexerTest ): + pytest.skip("test_save_load tested only for anydbm backend") + + self.dex.add_text(('test', '1', 'foo'), 'b the hello world') + self.assertSeqEqual(self.dex.find(['hello']), [('test', '1', 'foo')]) + self.dex.save_index() + + # reopen saved db. + from roundup.backends.indexer_dbm import Indexer + self.dex = Indexer(db) + + # verify index is unloaded + self.assertEqual(self.dex.index_loaded(), False) + + # add also calls load_index(), so it should load the first item. + self.dex.add_text(('test', '2', 'foo'), 'b the olleh world') + + # note find also does a load_index() if not loaded. + self.assertSeqEqual(self.dex.find(['hello']), [('test', '1', 'foo')]) + self.assertSeqEqual(self.dex.find(['olleh']), [('test', '2', 'foo')]) + def test_change(self): self.dex.add_text(('test', '1', 'foo'), 'a the hello world') self.dex.add_text(('test', '2', 'foo'), 'blah blah the world') From 0af4484cbfaef12f3c2349f221f9f05b55201b2d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sat, 4 Nov 2023 21:09:21 -0400 Subject: [PATCH 71/71] Added tag 2.3.1a0 for changeset d17e57220a62 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 9147c881..82d0e059 100644 --- a/.hgtags +++ b/.hgtags @@ -143,3 +143,4 @@ c90104abe508e3886917243e4acd069c8ef7a1a4 2.2.0 239d9542b02062c56f88fd1de8b87c4d88d700ad 2.2.0 51fc06fabcee043db116e2fbdcdcf5e86b67ed3d 2.3.0b2 913a73b9fab58e9c7e43e1fad379b68cae6ee3ae 2.3.0 +d17e57220a62416fcd192199cf29ca48db3af1a4 2.3.1a0