diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..6fd5ad1a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,80 @@ +name: Tests +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + container: + image: python:2.7.18-buster + steps: + - uses: actions/checkout@v3 + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 --statistics --show-source ckanext + + test: + needs: lint + strategy: + matrix: + ckan-version: [2.8] + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + services: + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + CKAN_SITE_URL: http://test.ckan.net + WORKDIR: /__w/ckanext-geocat/ckanext-geocat + SOLR_CONFIG_CKAN_DIR: /opt/solr/server/solr/ckan/conf + + steps: + - uses: actions/checkout@v3 + - name: Create solr container + run: | + git clone https://github.com/opendata-swiss/ckanext-switzerland-ng.git + docker create --name test_solr --network ${{ job.container.network }} --network-alias solr \ + --workdir $WORKDIR --publish 8983:8983 \ + -e SOLR_HEAP=1024m -e GITHUB_ACTIONS=true -e CI=true \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/schema.xml":"$SOLR_CONFIG_CKAN_DIR/schema.xml" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/english_stop.txt":"$SOLR_CONFIG_CKAN_DIR/english_stop.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/fr_elision.txt":"$SOLR_CONFIG_CKAN_DIR/fr_elision.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/french_stop.txt":"$SOLR_CONFIG_CKAN_DIR/french_stop.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/german_dictionary.txt":"$SOLR_CONFIG_CKAN_DIR/german_dictionary.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/german_stop.txt":"$SOLR_CONFIG_CKAN_DIR/german_stop.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/italian_stop.txt":"$SOLR_CONFIG_CKAN_DIR/italian_stop.txt" \ + -v "${{ github.workspace }}/ckanext-switzerland-ng/solr/solrconfig.xml":"$SOLR_CONFIG_CKAN_DIR/solrconfig.xml" \ + ckan/ckan-solr:${{ matrix.ckan-version }} + docker start test_solr + - name: Create ckan container + run: | + docker create --name test_ckan --network ${{ job.container.network }} --network-alias ckan \ + -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" \ + -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" \ + -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" \ + -v "/home/runner/work/_temp/_github_home":"/github/home" \ + -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" \ + --entrypoint "tail" openknowledge/ckan-dev:${{ matrix.ckan-version }} "-f" "/dev/null" + docker start test_ckan + - name: Install requirements and set up database + run: | + docker exec test_ckan $WORKDIR/bin/install_test_requirements.sh ${{ matrix.ckan-version }} + - name: Run tests + run: | + docker exec test_ckan nosetests --ckan --with-pylons=$WORKDIR/test.ini \ + --with-coverage --cover-package=ckanext.geocat --cover-inclusive --cover-erase --cover-tests $WORKDIR/ckanext/geocat diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c959811e..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -sudo: required - -# the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897 -dist: trusty -group: deprecated-2017Q4 - -language: python - -python: -- '2.7' - -env: - - CKANVERSION=2.8 - -services: - - redis-server - - postgresql - -install: - - bash bin/travis-build.bash - -script: bin/travis-run.sh diff --git a/bin/install_test_requirements.sh b/bin/install_test_requirements.sh new file mode 100755 index 00000000..84bc29f8 --- /dev/null +++ b/bin/install_test_requirements.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +WORKDIR=/__w/ckanext-geocat/ckanext-geocat + +pip install --upgrade pip + +# Install ckanext-geocat +pip install -r "$WORKDIR"/requirements.txt +pip install -r "$WORKDIR"/dev-requirements.txt +pip install -e "$WORKDIR"/ + +# Install ckanext dependencies +pip install -e git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent +pip install -e git+https://github.com/ckan/ckanext-hierarchy.git#egg=ckanext-hierarchy +pip install -e git+https://github.com/ckan/ckanext-xloader.git#egg=ckanext-xloader +pip install -r https://raw.githubusercontent.com/ckan/ckanext-xloader/master/requirements.txt + +# Last commit before support for Python 2 was dropped +pip install -e git+https://github.com/ckan/ckanext-harvest.git@v1.4.2#egg=ckanext-harvest +pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/v1.4.2/requirements.txt + +# Last commit before support for Python 2 was dropped +pip install -e git+https://github.com/ckan/ckanext-dcat.git@0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6#egg=ckanext-dcat +pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6/requirements-py2.txt + +# Last commit before support for Python 2 was dropped +pip install -e git+https://github.com/ckan/ckanext-showcase.git@v1.5.2#egg=ckanext-showcase + +# Our ckanexts +pip install -e git+https://github.com/opendata-swiss/ckanext-dcatapchharvest.git#egg=ckanext-dcatapchharvest +pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-dcatapchharvest/master/requirements.txt +pip install -e git+https://github.com/opendata-swiss/ckanext-harvester_dashboard.git#egg=ckanext-harvester_dashboard +pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-harvester_dashboard/master/requirements.txt +pip install -e git+https://github.com/opendata-swiss/ckanext-subscribe.git#egg=ckanext-subscribe +pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-subscribe/master/requirements.txt +pip install -e git+https://github.com/opendata-swiss/ckanext-switzerland-ng.git#egg=ckanext-switzerland-ng +pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-switzerland-ng/master/requirements.txt +pip install -e git+https://github.com/opendata-swiss/ckanext-scheming.git#egg=ckanext-scheming + +# Replace default path to CKAN core config file with the one on the container +sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' "$WORKDIR"/test.ini + +# Init db and enable required plugins +paster --plugin=ckan config-tool "$WORKDIR"/test.ini "ckan.plugins = " +paster --plugin=ckan db init -c "$WORKDIR"/test.ini +paster --plugin=ckanext-harvest harvester initdb -c "$WORKDIR"/test.ini +paster --plugin=ckan config-tool "$WORKDIR"/test.ini "ckan.plugins = harvest ckan_harvester geocat_harvester ogdch ogdch_pkg scheming_datasets fluent" diff --git a/bin/travis-build.bash b/bin/travis-build.bash deleted file mode 100644 index 7adca274..00000000 --- a/bin/travis-build.bash +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash -set -e - -echo "This is travis-build.bash..." - -echo "Updating GPG keys..." -wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - -curl -L https://packagecloud.io/github/git-lfs/gpgkey | sudo apt-key add - -wget -qO - https://www.mongodb.org/static/pgp/server-3.2.asc | sudo apt-key add - - -echo "Adding archive repository for postgres..." -sudo rm /etc/apt/sources.list.d/pgdg* -echo "deb https://apt-archive.postgresql.org/pub/repos/apt trusty-pgdg-archive main" | sudo tee -a /etc/apt/sources.list -echo "deb-src https://apt-archive.postgresql.org/pub/repos/apt trusty-pgdg-archive main" | sudo tee -a /etc/apt/sources.list - -echo "Removing old repository for cassandra..." -sudo rm /etc/apt/sources.list.d/cassandra* - -echo "Installing the packages that CKAN requires..." -sudo apt-get update -qq -sudo apt-get install solr-jetty libcommons-fileupload-java - -echo "Installing CKAN and its Python dependencies..." -git clone https://github.com/ckan/ckan -cd ckan -if [ $CKANVERSION == 'master' ] -then - echo "CKAN version: master" -else - CKAN_TAG=$(git tag | grep ^ckan-$CKANVERSION | sort --version-sort | tail -n 1) - git checkout $CKAN_TAG - echo "CKAN version: ${CKAN_TAG#ckan-}" -fi - -# install the recommended version of setuptools -if [ -f requirement-setuptools.txt ] -then - echo "Updating setuptools..." - pip install -r requirement-setuptools.txt -fi - -python setup.py develop - -pip install -r requirements.txt -pip install -r dev-requirements.txt -cd - - -echo "Setting up Solr..." -printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -sudo cp schema.xml /etc/solr/conf/schema.xml -sudo service jetty restart - -echo "Creating the PostgreSQL user and database..." -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' - -echo "Initialising the database..." -cd ckan -paster db init -c test-core.ini -cd - - -echo "Installing ckanext-harvest and its requirements..." -git clone --depth 1 --branch v1.4.2 https://github.com/ckan/ckanext-harvest -cd ckanext-harvest -python setup.py develop -pip install -r pip-requirements.txt -paster harvester initdb -c ../ckan/test-core.ini -cd - - -# Begin installation of ckanext-switzerland-ng and its requirements. -# This is necessary so that the geocat harvester can save datasets using the -# correct schema, defined in ckanext.switzerland:dcat-ap-switzerland_scheming.json. -# We have to install the whole ckanext, as the schema relies on custom -# validators. Hopefully we can untangle this later to avoid all this installing. - -# The version of setuptools in CKAN (36.1) falls over when installing -# dependencies for ckanext-scheming on Travis, specifically ckantoolkit. -# A newer setuptools version works, possibly because in versions >42, -# pip is used to install dependencies listed in install_requires in -# setup.py. -echo "Upgrading setuptools..." -pip install --upgrade setuptools - -echo "Installing ckanext-scheming and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-scheming.git -cd ckanext-scheming -git checkout repair-pyyaml-dependency -python setup.py develop -cd - - -echo "Installing ckanext-fluent and its requirements..." -git clone https://github.com/ckan/ckanext-fluent -cd ckanext-fluent -python setup.py develop -cd - - -echo "Installing ckanext-hierarchy and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-hierarchy -cd ckanext-hierarchy -python setup.py develop -cd - - -echo "Installing ckanext-dcat and its requirements..." -git clone https://github.com/ckan/ckanext-dcat -cd ckanext-dcat -# Last commit before support for Python 2 was dropped -git checkout 0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6 -python setup.py develop -pip install -r requirements-py2-py36.txt -pip install -r dev-requirements-py2.txt -cd - - -echo "Installing ckanext-dcatapchharvest and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-dcatapchharvest -cd ckanext-dcatapchharvest -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt -cd - - -echo "Installing ckanext-harvester_dashboard and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-harvester_dashboard -cd ckanext-harvester_dashboard -python setup.py develop -pip install -r requirements.txt -cd - - -echo "Installing ckanext-xloader and its requirements..." -git clone https://github.com/ckan/ckanext-xloader -cd ckanext-xloader -python setup.py develop -pip install -r requirements.txt -cd - - -echo "Installing ckanext-showcase..." -git clone --depth 1 --branch v1.5.2 https://github.com/ckan/ckanext-showcase -cd ckanext-showcase -python setup.py develop -cd - - -echo "Installing ckanext-subscribe and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-subscribe -cd ckanext-subscribe -python setup.py develop -pip install -r requirements.txt -cd - - -echo "Installing ckanext-switzerland and its requirements..." -git clone https://github.com/opendata-swiss/ckanext-switzerland-ng -cd ckanext-switzerland-ng -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt -cd - - -# End installation of ckanext-switzlerland et al. - -echo "Installing ckanext-geocat and its requirements..." -pip install -r requirements.txt -pip install -r dev-requirements.txt -python setup.py develop - -echo "Moving test.ini into a subdir..." -mkdir subdir -mv test.ini subdir - -echo "travis-build.bash is done." diff --git a/bin/travis-run.sh b/bin/travis-run.sh deleted file mode 100755 index b03a7e02..00000000 --- a/bin/travis-run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -e - -function cleanup { - exit $? -} - -trap "cleanup" EXIT - -# Check PEP-8 code style and McCabe complexity -flake8 --statistics --show-source ckanext - -nosetests -v --ckan --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.geocat --cover-inclusive --cover-erase --cover-tests ckanext/geocat diff --git a/ckanext/geocat/utils/ogdch_map_utils.py b/ckanext/geocat/utils/ogdch_map_utils.py index ec5a33a5..969b6454 100644 --- a/ckanext/geocat/utils/ogdch_map_utils.py +++ b/ckanext/geocat/utils/ogdch_map_utils.py @@ -206,6 +206,7 @@ def map_resource(geocat_resource, issued, modified, rights): resource_dict['issued'] = issued resource_dict['modified'] = modified resource_dict['rights'] = rights + resource_dict['license'] = rights if geocat_resource.get('format'): resource_dict['format'] = geocat_resource['format'] resource_dict['description'] = geocat_resource.get('description') diff --git a/ckanext/geocat/utils/xpath_utils.py b/ckanext/geocat/utils/xpath_utils.py index 2f2fb6ef..33aa07a5 100644 --- a/ckanext/geocat/utils/xpath_utils.py +++ b/ckanext/geocat/utils/xpath_utils.py @@ -173,7 +173,7 @@ def xpath_get_rights_dict_form_rights_node(node): if value_locale: rights_dict[locale.lower()] = _clean_string(value_locale[0]) return rights_dict - except: + except Exception: return '' @@ -376,7 +376,7 @@ def _get_normed_protocol(protocol): def _clean_string(value): try: - return re.sub('\s+', ' ', value).strip() + return re.sub(r'\s+', ' ', value).strip() except TypeError: return value diff --git a/requirements.txt b/requirements.txt index 383e19b5..6682209f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ lxml>=4.6.2 OWSLib==0.18.0 +pyproj==1.9.6 python-dateutil==2.8.1 -pyyaml==5.4 +# TODO: Revert to 5.4 once https://github.com/yaml/pyyaml/issues/724 is fixed +pyyaml==5.3.1 rdflib==5.0.0 diff --git a/schema.xml b/schema.xml deleted file mode 100644 index 0a0a9385..00000000 --- a/schema.xml +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -index_id -text - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test.ini b/test.ini index 25b8af1b..b19793cc 100644 --- a/test.ini +++ b/test.ini @@ -9,13 +9,17 @@ host = 0.0.0.0 port = 5000 [app:main] -# Config for Travis testing +# Config for GitHub testing use = config:../ckan/test-core.ini -solr_url = http://127.0.0.1:8983/solr +solr_url = http://solr:8983/solr/ckan +sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test +ckan.redis.url = redis://redis:6379/1 # Config for local testing (uncomment to use, but don't commit) # use = config:/usr/lib/ckan/venv/src/ckan/test-core.ini # solr_url = http://solr:8983/solr +# sqlalchemy.url = postgresql://ckan:ckan@db/ckan_test +# ckan.redis.url = redis://redis:6379/1 ckan.plugins = harvest ckan_harvester geocat_harvester ogdch ogdch_pkg scheming_datasets fluent