diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..a2fced6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: Build PyDataverse +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + name: Build pyDataverse + steps: + - name: "Checkout" + uses: "actions/checkout@v4" + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python Dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install poetry + + poetry install diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..b268138 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,8 @@ +name: Ruff +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..7e583da --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,16 @@ +name: Build and publish + +on: + release: + types: [released] + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: "Build and publish to PyPi" + uses: JRubics/poetry-publish@v1.17 + with: + pypi_token: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/test_build.yml b/.github/workflows/tests.yml similarity index 78% rename from .github/workflows/test_build.yml rename to .github/workflows/tests.yml index 9411b14..574d777 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/tests.yml @@ -5,13 +5,8 @@ jobs: custom_test: runs-on: ubuntu-latest strategy: - matrix: - python-version: [ - "3.8", - "3.9", - "3.10", - "3.11" - ] + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] name: Test pyDataverse env: PORT: 8080 @@ -28,8 +23,10 @@ jobs: - name: Install Python Dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install -r ./requirements/tests.txt - pip install -e . + python3 -m pip install poetry + + poetry install --with tests + - name: Run tests env: API_TOKEN_SUPERUSER: ${{ steps.dataverse.outputs.api_token }} @@ -37,4 +34,4 @@ jobs: BASE_URL: ${{ steps.dataverse.outputs.base_url }} DV_VERSION: ${{ steps.dataverse.outputs.dv_version }} run: | - python3 -m pytest \ No newline at end of file + python3 -m poetry run pytest diff --git a/.gitignore b/.gitignore index 5610c23..bbd89c2 100644 --- a/.gitignore +++ b/.gitignore @@ -157,3 +157,9 @@ src/pyDataverse/docs/source/_build src/pyDataverse/docs/Makefile env-config/ wiki/ + +# Poetry lock +poetry.lock + +# Ruff +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b87e3e3..cd1a9e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,35 +1,37 @@ default_language_version: - python: python3 + python: python3 exclude: ^migrations/ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - exclude: _pytest/(debugging|hookspec).py + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 hooks: - - id: check-added-large-files - - id: check-case-conflict - - id: check-docstring-first - - id: check-json - - id: check-symlinks - - id: check-xml - - id: check-yaml - - id: detect-private-key - - id: end-of-file-fixer - - id: pretty-format-json - args: [--autofix,--no-sort-keys] -- repo: https://github.com/psf/black - rev: 19.10b0 + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-json + - id: check-symlinks + - id: check-xml + - id: check-yaml + - id: detect-private-key + - id: end-of-file-fixer + - id: pretty-format-json + args: [--autofix, --no-sort-keys] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 hooks: - - id: black - language_version: python3.6 -- repo: https://github.com/asottile/blacken-docs + - id: ruff + args: [--fix] + - id: ruff-format + + - repo: https://github.com/asottile/blacken-docs rev: v1.7.0 hooks: - - id: blacken-docs + - id: blacken-docs additional_dependencies: [black==19.10b0] -- repo: https://github.com/codespell-project/codespell - # Configuration for codespell is in setup.cfg - rev: v2.2.6 - hooks: - - id: codespell + - repo: https://github.com/codespell-project/codespell + # Configuration for codespell is in setup.cfg + rev: v2.2.6 + hooks: + - id: codespell diff --git a/.readthedocs.yml b/.readthedocs.yml index c53ae54..37b2417 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,20 +3,20 @@ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required -version: 2 +version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: src/pyDataverse/docs/source/conf.py + configuration: pyDataverse/docs/source/conf.py # Optionally build your docs in additional formats such as PDF and ePub -formats: all +formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.6 - install: - - requirements: requirements/docs.txt - - method: pip - path: . - system_packages: true + version: 3.6 + install: + - requirements: requirements/docs.txt + - method: pip + path: . + system_packages: true diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 51d4143..697eac4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -229,7 +229,7 @@ you created the branch, check the section on :ref:`updating a PR `. -From here, you now can move forward to +From here, you now can move forward to - contribute to the documentation (see below) - contribute to the :ref:`code base ` @@ -267,7 +267,7 @@ Some other important things to know about the docs: - The pyDataverse documentation consists of two parts: - the docstrings in the code itself and - - the docs in the folder ``src/pyDataverse/doc/`` + - the docs in the folder ``pyDataverse/doc/`` - The docstrings provide a clear explanation of the usage of the individual functions, while the documentation consists of tutorial-like overviews per topic together with some other information (what’s new, installation, this page you're viewing right now, etc). - The docstrings follow the `Numpy Docstring Standard `_. @@ -298,7 +298,7 @@ to ``upstream/develop``, so you have to branch-off from it too. There is one exception: If you want to suggest a change to the docs in the folder -``src/pyDataverse/docs/`` (e. g. fix a typo in +``pyDataverse/docs/`` (e. g. fix a typo in :ref:`User Guide - Basic Usage `), you can also pull to ``upstream/master``. This means, you have also to branch-off from the ``master`` branch. @@ -411,7 +411,7 @@ To run black alone, use .. code-block:: shell - black src/pyDataverse/file_changed.py + black pyDataverse/file_changed.py .. _contributing_code_type-hints: @@ -430,7 +430,7 @@ type hints. After making any change you can ensure your type hints are correct b .. code-block:: shell - mypy src/pyDataverse/file_changed.py + mypy pyDataverse/file_changed.py .. _contributing_code_testing-with-ci: diff --git a/MANIFEST.in b/MANIFEST.in index 9cab79a..f37f118 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include src/pyDataverse/schemas * +recursive-include pyDataverse/schemas * diff --git a/src/pyDataverse/__init__.py b/pyDataverse/__init__.py similarity index 86% rename from src/pyDataverse/__init__.py rename to pyDataverse/__init__.py index 6c5fe28..bea7692 100644 --- a/src/pyDataverse/__init__.py +++ b/pyDataverse/__init__.py @@ -7,15 +7,12 @@ from __future__ import absolute_import -import urllib3 - -urllib3.disable_warnings() # noqa __author__ = "Stefan Kasberger" __email__ = "stefan.kasberger@univie.ac.at" __copyright__ = "Copyright (c) 2019 Stefan Kasberger" __license__ = "MIT License" -__version__ = "0.3.1" +__version__ = "0.3.2" __url__ = "https://github.com/GDCC/pyDataverse" __download_url__ = "https://pypi.python.org/pypi/pyDataverse" __description__ = "A Python module for Dataverse." diff --git a/src/pyDataverse/api.py b/pyDataverse/api.py similarity index 100% rename from src/pyDataverse/api.py rename to pyDataverse/api.py diff --git a/src/pyDataverse/docs/source/_images/collection_dataset.png b/pyDataverse/docs/source/_images/collection_dataset.png similarity index 100% rename from src/pyDataverse/docs/source/_images/collection_dataset.png rename to pyDataverse/docs/source/_images/collection_dataset.png diff --git a/src/pyDataverse/docs/source/_static/.gitkeep b/pyDataverse/docs/source/_static/.gitkeep similarity index 100% rename from src/pyDataverse/docs/source/_static/.gitkeep rename to pyDataverse/docs/source/_static/.gitkeep diff --git a/src/pyDataverse/docs/source/_templates/layout.html b/pyDataverse/docs/source/_templates/layout.html similarity index 100% rename from src/pyDataverse/docs/source/_templates/layout.html rename to pyDataverse/docs/source/_templates/layout.html diff --git a/src/pyDataverse/docs/source/_templates/sidebar_intro.html b/pyDataverse/docs/source/_templates/sidebar_intro.html similarity index 100% rename from src/pyDataverse/docs/source/_templates/sidebar_intro.html rename to pyDataverse/docs/source/_templates/sidebar_intro.html diff --git a/src/pyDataverse/docs/source/_templates/sidebar_related-links.html b/pyDataverse/docs/source/_templates/sidebar_related-links.html similarity index 100% rename from src/pyDataverse/docs/source/_templates/sidebar_related-links.html rename to pyDataverse/docs/source/_templates/sidebar_related-links.html diff --git a/src/pyDataverse/docs/source/community/contact.rst b/pyDataverse/docs/source/community/contact.rst similarity index 100% rename from src/pyDataverse/docs/source/community/contact.rst rename to pyDataverse/docs/source/community/contact.rst diff --git a/src/pyDataverse/docs/source/community/releases.rst b/pyDataverse/docs/source/community/releases.rst similarity index 100% rename from src/pyDataverse/docs/source/community/releases.rst rename to pyDataverse/docs/source/community/releases.rst diff --git a/src/pyDataverse/docs/source/conf.py b/pyDataverse/docs/source/conf.py similarity index 99% rename from src/pyDataverse/docs/source/conf.py rename to pyDataverse/docs/source/conf.py index b62a143..a207fdc 100644 --- a/src/pyDataverse/docs/source/conf.py +++ b/pyDataverse/docs/source/conf.py @@ -19,7 +19,7 @@ import os import sys -sys.path.insert(0, os.path.abspath("../../..")) +sys.path.insert(0, os.path.abspath("../..")) project = "pyDataverse" author = "Stefan Kasberger" diff --git a/src/pyDataverse/docs/source/contributing/contributing.rst b/pyDataverse/docs/source/contributing/contributing.rst similarity index 100% rename from src/pyDataverse/docs/source/contributing/contributing.rst rename to pyDataverse/docs/source/contributing/contributing.rst diff --git a/src/pyDataverse/docs/source/index.rst b/pyDataverse/docs/source/index.rst similarity index 100% rename from src/pyDataverse/docs/source/index.rst rename to pyDataverse/docs/source/index.rst diff --git a/src/pyDataverse/docs/source/reference.rst b/pyDataverse/docs/source/reference.rst similarity index 100% rename from src/pyDataverse/docs/source/reference.rst rename to pyDataverse/docs/source/reference.rst diff --git a/src/pyDataverse/docs/source/snippets/pip-install.rst b/pyDataverse/docs/source/snippets/pip-install.rst similarity index 100% rename from src/pyDataverse/docs/source/snippets/pip-install.rst rename to pyDataverse/docs/source/snippets/pip-install.rst diff --git a/src/pyDataverse/docs/source/snippets/requirements.rst b/pyDataverse/docs/source/snippets/requirements.rst similarity index 100% rename from src/pyDataverse/docs/source/snippets/requirements.rst rename to pyDataverse/docs/source/snippets/requirements.rst diff --git a/src/pyDataverse/docs/source/snippets/warning_production.rst b/pyDataverse/docs/source/snippets/warning_production.rst similarity index 100% rename from src/pyDataverse/docs/source/snippets/warning_production.rst rename to pyDataverse/docs/source/snippets/warning_production.rst diff --git a/src/pyDataverse/docs/source/user/advanced-usage.rst b/pyDataverse/docs/source/user/advanced-usage.rst similarity index 97% rename from src/pyDataverse/docs/source/user/advanced-usage.rst rename to pyDataverse/docs/source/user/advanced-usage.rst index 6c98545..769ce1a 100644 --- a/src/pyDataverse/docs/source/user/advanced-usage.rst +++ b/pyDataverse/docs/source/user/advanced-usage.rst @@ -42,7 +42,7 @@ Prepare **Additional Resources** -- CSV templates from ``src/pyDataverse/templates/`` are used (see :ref:`CSV templates `) +- CSV templates from ``pyDataverse/templates/`` are used (see :ref:`CSV templates `) - Data from ``tests/data/user-guide/`` is used (`GitHub repo `_) @@ -101,10 +101,10 @@ converts boolean values, and loads JSON cells properly. Once we have the data in Python, we can easily import the data into pyDataverse. -For this, loop over each Dataset :class:`dict`, to: +For this, loop over each Dataset :class:`dict`, to: #. Instantiate an empty :class:`Dataset ` -#. add the data with :meth:`set() ` and +#. add the data with :meth:`set() ` and #. append the instance to a :class:`list`. :: diff --git a/src/pyDataverse/docs/source/user/basic-usage.rst b/pyDataverse/docs/source/user/basic-usage.rst similarity index 100% rename from src/pyDataverse/docs/source/user/basic-usage.rst rename to pyDataverse/docs/source/user/basic-usage.rst diff --git a/src/pyDataverse/docs/source/user/csv-templates.rst b/pyDataverse/docs/source/user/csv-templates.rst similarity index 95% rename from src/pyDataverse/docs/source/user/csv-templates.rst rename to pyDataverse/docs/source/user/csv-templates.rst index d7e471d..d36294c 100644 --- a/src/pyDataverse/docs/source/user/csv-templates.rst +++ b/pyDataverse/docs/source/user/csv-templates.rst @@ -71,12 +71,12 @@ There is also a more detailed tutorial on how to use the CSV templates for mass imports in the :ref:`User Guide - Advanced `. -The CSV templates can be found in ``src/pyDataverse/templates/`` -(`GitHub repo `_): +The CSV templates can be found in ``pyDataverse/templates/`` +(`GitHub repo `_): -- `dataverses.csv `_ -- `datasets.csv `_ -- `datafiles.csv `_ +- `dataverses.csv `_ +- `datasets.csv `_ +- `datafiles.csv `_ .. _user_csv-templates_usage_create-csv: diff --git a/src/pyDataverse/docs/source/user/faq.rst b/pyDataverse/docs/source/user/faq.rst similarity index 100% rename from src/pyDataverse/docs/source/user/faq.rst rename to pyDataverse/docs/source/user/faq.rst diff --git a/src/pyDataverse/docs/source/user/installation.rst b/pyDataverse/docs/source/user/installation.rst similarity index 100% rename from src/pyDataverse/docs/source/user/installation.rst rename to pyDataverse/docs/source/user/installation.rst diff --git a/src/pyDataverse/docs/source/user/resources.rst b/pyDataverse/docs/source/user/resources.rst similarity index 100% rename from src/pyDataverse/docs/source/user/resources.rst rename to pyDataverse/docs/source/user/resources.rst diff --git a/src/pyDataverse/docs/source/user/use-cases.rst b/pyDataverse/docs/source/user/use-cases.rst similarity index 100% rename from src/pyDataverse/docs/source/user/use-cases.rst rename to pyDataverse/docs/source/user/use-cases.rst diff --git a/src/pyDataverse/exceptions.py b/pyDataverse/exceptions.py similarity index 100% rename from src/pyDataverse/exceptions.py rename to pyDataverse/exceptions.py diff --git a/src/pyDataverse/models.py b/pyDataverse/models.py similarity index 95% rename from src/pyDataverse/models.py rename to pyDataverse/models.py index a938869..2a835e2 100644 --- a/src/pyDataverse/models.py +++ b/pyDataverse/models.py @@ -1,4 +1,5 @@ """Dataverse data-types data model.""" + from __future__ import absolute_import import json @@ -103,7 +104,9 @@ def validate_json(self, filename_schema=None): assert isinstance(filename_schema, str) return validate_data( - json.loads(self.json(validate=False)), filename_schema, file_format="json", + json.loads(self.json(validate=False)), + filename_schema, + file_format="json", ) def from_json( @@ -867,7 +870,6 @@ def from_json( ) if "metadataBlocks" in json_dict["datasetVersion"]: - # citation if "citation" in json_dict["datasetVersion"]["metadataBlocks"]: citation = json_dict["datasetVersion"]["metadataBlocks"]["citation"] @@ -954,7 +956,8 @@ def from_json( if "displayName" in socialscience: self.__setattr__( - "socialscience_displayName", socialscience["displayName"], + "socialscience_displayName", + socialscience["displayName"], ) for field in socialscience["fields"]: @@ -966,27 +969,27 @@ def from_json( elif field["typeName"] == "targetSampleSize": data["targetSampleSize"] = {} if "targetSampleActualSize" in field["value"]: - data["targetSampleSize"][ - "targetSampleActualSize" - ] = field["value"]["targetSampleActualSize"]["value"] + data["targetSampleSize"]["targetSampleActualSize"] = ( + field["value"]["targetSampleActualSize"]["value"] + ) if "targetSampleSizeFormula" in field["value"]: - data["targetSampleSize"][ - "targetSampleSizeFormula" - ] = field["value"]["targetSampleSizeFormula"]["value"] + data["targetSampleSize"]["targetSampleSizeFormula"] = ( + field["value"]["targetSampleSizeFormula"]["value"] + ) elif field["typeName"] == "socialScienceNotes": data["socialScienceNotes"] = {} if "socialScienceNotesType" in field["value"]: - data["socialScienceNotes"][ - "socialScienceNotesType" - ] = field["value"]["socialScienceNotesType"]["value"] + data["socialScienceNotes"]["socialScienceNotesType"] = ( + field["value"]["socialScienceNotesType"]["value"] + ) if "socialScienceNotesSubject" in field["value"]: data["socialScienceNotes"][ "socialScienceNotesSubject" ] = field["value"]["socialScienceNotesSubject"]["value"] if "socialScienceNotesText" in field["value"]: - data["socialScienceNotes"][ - "socialScienceNotesText" - ] = field["value"]["socialScienceNotesText"]["value"] + data["socialScienceNotes"]["socialScienceNotesText"] = ( + field["value"]["socialScienceNotesText"]["value"] + ) else: print( "Attribute {0} not valid for import (dv_up).".format( @@ -1200,7 +1203,10 @@ def json(self, data_format=None, validate=True, filename_schema=None): ) # Generate fields attributes - for (key, val,) in self.__attr_import_dv_up_citation_fields_arrays.items(): + for ( + key, + val, + ) in self.__attr_import_dv_up_citation_fields_arrays.items(): if key in data_dict: v = data_dict[key] citation["fields"].append( @@ -1341,25 +1347,25 @@ def json(self, data_format=None, validate=True, filename_schema=None): if "targetSampleActualSize" in target_sample_size: if target_sample_size["targetSampleActualSize"] is not None: tmp_dict["targetSampleActualSize"] = {} - tmp_dict["targetSampleActualSize"][ - "typeName" - ] = "targetSampleActualSize" + tmp_dict["targetSampleActualSize"]["typeName"] = ( + "targetSampleActualSize" + ) tmp_dict["targetSampleActualSize"]["multiple"] = False tmp_dict["targetSampleActualSize"]["typeClass"] = "primitive" - tmp_dict["targetSampleActualSize"][ - "value" - ] = target_sample_size["targetSampleActualSize"] + tmp_dict["targetSampleActualSize"]["value"] = ( + target_sample_size["targetSampleActualSize"] + ) if "targetSampleSizeFormula" in target_sample_size: if target_sample_size["targetSampleSizeFormula"] is not None: tmp_dict["targetSampleSizeFormula"] = {} - tmp_dict["targetSampleSizeFormula"][ - "typeName" - ] = "targetSampleSizeFormula" + tmp_dict["targetSampleSizeFormula"]["typeName"] = ( + "targetSampleSizeFormula" + ) tmp_dict["targetSampleSizeFormula"]["multiple"] = False tmp_dict["targetSampleSizeFormula"]["typeClass"] = "primitive" - tmp_dict["targetSampleSizeFormula"][ - "value" - ] = target_sample_size["targetSampleSizeFormula"] + tmp_dict["targetSampleSizeFormula"]["value"] = ( + target_sample_size["targetSampleSizeFormula"] + ) socialscience["fields"].append( { "typeName": "targetSampleSize", @@ -1376,36 +1382,36 @@ def json(self, data_format=None, validate=True, filename_schema=None): if "socialScienceNotesType" in social_science_notes: if social_science_notes["socialScienceNotesType"] is not None: tmp_dict["socialScienceNotesType"] = {} - tmp_dict["socialScienceNotesType"][ - "typeName" - ] = "socialScienceNotesType" + tmp_dict["socialScienceNotesType"]["typeName"] = ( + "socialScienceNotesType" + ) tmp_dict["socialScienceNotesType"]["multiple"] = False tmp_dict["socialScienceNotesType"]["typeClass"] = "primitive" - tmp_dict["socialScienceNotesType"][ - "value" - ] = social_science_notes["socialScienceNotesType"] + tmp_dict["socialScienceNotesType"]["value"] = ( + social_science_notes["socialScienceNotesType"] + ) if "socialScienceNotesSubject" in social_science_notes: if social_science_notes["socialScienceNotesSubject"] is not None: tmp_dict["socialScienceNotesSubject"] = {} - tmp_dict["socialScienceNotesSubject"][ - "typeName" - ] = "socialScienceNotesSubject" + tmp_dict["socialScienceNotesSubject"]["typeName"] = ( + "socialScienceNotesSubject" + ) tmp_dict["socialScienceNotesSubject"]["multiple"] = False tmp_dict["socialScienceNotesSubject"]["typeClass"] = "primitive" - tmp_dict["socialScienceNotesSubject"][ - "value" - ] = social_science_notes["socialScienceNotesSubject"] + tmp_dict["socialScienceNotesSubject"]["value"] = ( + social_science_notes["socialScienceNotesSubject"] + ) if "socialScienceNotesText" in social_science_notes: if social_science_notes["socialScienceNotesText"] is not None: tmp_dict["socialScienceNotesText"] = {} - tmp_dict["socialScienceNotesText"][ - "typeName" - ] = "socialScienceNotesText" + tmp_dict["socialScienceNotesText"]["typeName"] = ( + "socialScienceNotesText" + ) tmp_dict["socialScienceNotesText"]["multiple"] = False tmp_dict["socialScienceNotesText"]["typeClass"] = "primitive" - tmp_dict["socialScienceNotesText"][ - "value" - ] = social_science_notes["socialScienceNotesText"] + tmp_dict["socialScienceNotesText"]["value"] = ( + social_science_notes["socialScienceNotesText"] + ) socialscience["fields"].append( { "typeName": "socialScienceNotes", @@ -1456,7 +1462,10 @@ def json(self, data_format=None, validate=True, filename_schema=None): ) # Generate fields attributes - for (key, val,) in self.__attr_import_dv_up_journal_fields_arrays.items(): + for ( + key, + val, + ) in self.__attr_import_dv_up_journal_fields_arrays.items(): if key in data_dict: journal["fields"].append( { @@ -1469,9 +1478,9 @@ def json(self, data_format=None, validate=True, filename_schema=None): data["datasetVersion"]["metadataBlocks"]["citation"] = citation if "socialscience" in locals(): - data["datasetVersion"]["metadataBlocks"][ - "socialscience" - ] = socialscience + data["datasetVersion"]["metadataBlocks"]["socialscience"] = ( + socialscience + ) if "geospatial" in locals(): data["datasetVersion"]["metadataBlocks"]["geospatial"] = geospatial if "journal" in locals(): diff --git a/src/pyDataverse/schemas/json/datafile_upload_schema.json b/pyDataverse/schemas/json/datafile_upload_schema.json similarity index 100% rename from src/pyDataverse/schemas/json/datafile_upload_schema.json rename to pyDataverse/schemas/json/datafile_upload_schema.json diff --git a/src/pyDataverse/schemas/json/dataset_upload_default_schema.json b/pyDataverse/schemas/json/dataset_upload_default_schema.json similarity index 100% rename from src/pyDataverse/schemas/json/dataset_upload_default_schema.json rename to pyDataverse/schemas/json/dataset_upload_default_schema.json diff --git a/src/pyDataverse/schemas/json/dataverse_upload_schema.json b/pyDataverse/schemas/json/dataverse_upload_schema.json similarity index 100% rename from src/pyDataverse/schemas/json/dataverse_upload_schema.json rename to pyDataverse/schemas/json/dataverse_upload_schema.json diff --git a/src/pyDataverse/schemas/json/dspace_schema.json b/pyDataverse/schemas/json/dspace_schema.json similarity index 100% rename from src/pyDataverse/schemas/json/dspace_schema.json rename to pyDataverse/schemas/json/dspace_schema.json diff --git a/src/pyDataverse/templates/datafiles.csv b/pyDataverse/templates/datafiles.csv similarity index 100% rename from src/pyDataverse/templates/datafiles.csv rename to pyDataverse/templates/datafiles.csv diff --git a/src/pyDataverse/templates/datasets.csv b/pyDataverse/templates/datasets.csv similarity index 100% rename from src/pyDataverse/templates/datasets.csv rename to pyDataverse/templates/datasets.csv diff --git a/src/pyDataverse/templates/dataverses.csv b/pyDataverse/templates/dataverses.csv similarity index 100% rename from src/pyDataverse/templates/dataverses.csv rename to pyDataverse/templates/dataverses.csv diff --git a/src/pyDataverse/utils.py b/pyDataverse/utils.py similarity index 99% rename from src/pyDataverse/utils.py rename to pyDataverse/utils.py index b931d7d..f22acf7 100644 --- a/src/pyDataverse/utils.py +++ b/pyDataverse/utils.py @@ -1,4 +1,5 @@ """Helper functions.""" + import csv import json import os @@ -585,13 +586,13 @@ def dataverse_tree_walker( datasets = [] datafiles = [] - if type(data) == list: + if isinstance(data, list): for elem in data: dv, ds, df = dataverse_tree_walker(elem) dataverses += dv datasets += ds datafiles += df - elif type(data) == dict: + elif isinstance(data, dict): if data["type"] == "dataverse": dv_tmp = {} for key in dv_keys: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3ccce23 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[tool.poetry] +name = "pyDataverse" +version = "0.3.2" +description = "A Python module for Dataverse." +authors = [ + "Stephan Kasberger ", + "Jan Range ", +] +license = "MIT" +readme = "README.md" +repository = "https://github.com/gdcc/pyDataverse" +packages = [{ include = "pyDataverse" }] + +[tool.poetry.dependencies] +python = "^3.8.1" +httpx = "^0.27.0" +jsonschema = "^4.21.1" + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +black = "^24.3.0" +radon = "^6.0.1" +mypy = "^1.9.0" +autopep8 = "^2.1.0" +pydocstyle = "^6.3.0" +pygments = "^2.17.2" +pytest = "^8.1.1" +pytest-cov = "^5.0.0" +tox = "^4.14.2" +selenium = "^4.19.0" +wheel = "^0.43.0" +pre-commit = "3.5.0" +sphinx = "7.1.2" +restructuredtext-lint = "^1.4.0" +rstcheck = "^6.2.1" +ruff = "^0.4.4" + + +[tool.poetry.group.tests] +optional = true + +[tool.poetry.group.tests.dependencies] +pytest = "^8.1.1" +pytest-cov = "^5.0.0" +tox = "^4.14.2" +selenium = "^4.19.0" + +[tool.poetry.group.docs] +optional = true + +[tool.poetry.group.docs.dependencies] +sphinx = "7.1.2" +pydocstyle = "^6.3.0" +restructuredtext-lint = "^1.4.0" +pygments = "^2.17.2" +rstcheck = "^6.2.1" + +[tool.poetry.group.lint] +optional = true + +[tool.poetry.group.lint.dependencies] +black = "^24.3.0" +radon = "^6.0.1" +mypy = "^1.9.0" +autopep8 = "^2.1.0" +ruff = "^0.4.4" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +pythonpath = ["src"] +addopts = ["-v", "--cov=dvuploader"] + +[tool.coverage.run] +source = "tests" + +[tool.coverage.report] +show_missing = true + +[tool.radon] +cc_min = "B" + +[tool.flake8] +max-line-length = 88 +ignore = ["E129", "E203", "E501", "W503", "S101"] +exclude = [".tox", ".egg", "conf.py", "conftest.py"] + +[tool.pylint] +max-line-length = 88 + +[tool.pylama.pycodestyle] +max_line_length = 88 + +[tool.pylama.pylint] +max_line_length = 88 +disable = "R" diff --git a/requirements.txt b/requirements.txt index 11d7a96..8ba8abd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ # Requirements --r requirements/common.txt +httpx==0.27.0 +jsonschema==4.21.1 diff --git a/requirements/common.txt b/requirements/common.txt deleted file mode 100644 index d17ab22..0000000 --- a/requirements/common.txt +++ /dev/null @@ -1,2 +0,0 @@ -httpx>=0.26.0 -jsonschema>=3.2.0 diff --git a/requirements/development.txt b/requirements/development.txt deleted file mode 100644 index e633bf3..0000000 --- a/requirements/development.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Requirements for development. --r common.txt --r lint.txt --r docs.txt --r tests.txt -wheel -pre-commit -twine diff --git a/requirements/docker.txt b/requirements/docker.txt deleted file mode 100644 index 6624a30..0000000 --- a/requirements/docker.txt +++ /dev/null @@ -1 +0,0 @@ --r common.txt diff --git a/requirements/docs.txt b/requirements/docs.txt deleted file mode 100644 index 7b376cd..0000000 --- a/requirements/docs.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Requirements for documentation test (tox) --r common.txt -sphinx==1.7.* -git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme -git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme -pydocstyle -restructuredtext_lint -pygments -rstcheck diff --git a/requirements/lint.txt b/requirements/lint.txt deleted file mode 100644 index 3968095..0000000 --- a/requirements/lint.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Requirements for linting tests (tox) -black==19.10b0 -flake8==3.7.4 -flake8-docstrings==1.5.0 -flake8-rst-docstrings==v0.0.13 -flake8-blind-except==v0.1.1 -flake8-builtins==1.5.3 -flake8-bandit==v2.1.0 -flake8-breakpoint==v1.1.0 -flake8-bugbear==20.1.4 -flake8-comprehensions==3.2.3 -flake8-requirements==v1.3.2 -flake8-return==0.3.0 -flake8-pytest-style==v1.2.1 -pylint -radon -mypy==0.790 -autopep8 diff --git a/requirements/packaging.txt b/requirements/packaging.txt deleted file mode 100644 index 9906383..0000000 --- a/requirements/packaging.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Requirements for packaging test (tox) -twine diff --git a/requirements/production.txt b/requirements/production.txt deleted file mode 100644 index 6624a30..0000000 --- a/requirements/production.txt +++ /dev/null @@ -1 +0,0 @@ --r common.txt diff --git a/requirements/tests.txt b/requirements/tests.txt deleted file mode 100644 index 45b26c5..0000000 --- a/requirements/tests.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Requirements for tests (pytest) --r common.txt -pytest -pytest-cov -tox -selenium==3.141.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1739684..0000000 --- a/setup.cfg +++ /dev/null @@ -1,41 +0,0 @@ -[metadata] -license_file = LICENSE.txt - -[tool:pytest] -testpaths = tests - -[coverage:run] -source = wheel -omit = .tox/* - -[coverage:report] -show_missing = true - -[radon] -cc_min = B - -[flake8] -max-line-length = 88 -ignore = E129,E203,E501,W503,S101 -exclude = - .tox - .egg - conf.py - conftest.py - -[pylint] -max-line-length = 88 - -[pylama:pycodestyle] -max_line_length = 88 - -[pylama:pylint] -max_line_length = 88 -disable = R - -[codespell] -# Ref: https://github.com/codespell-project/codespell#using-a-config-file -skip = .git*,*.svg,data -check-hidden = true -# ignore-regex = -# ignore-words-list = diff --git a/setup.py b/setup.py deleted file mode 100644 index cc0d04a..0000000 --- a/setup.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Find out more at https://github.com/GDCC/pyDataverse.""" - -import codecs -import os -import re -from setuptools.command.test import test as TestCommand -from setuptools import find_packages -from setuptools import setup -import sys - -ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) -os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) - - -def read_file(*file_paths): - """Read text file.""" - with codecs.open(os.path.join(ROOT_DIR, *file_paths), "r") as fp: - return fp.read() - - -def find_version(*file_paths): - """Find package version from file.""" - version_file = read_file(*file_paths) - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", - version_file, - re.M, - ) - if version_match: - return version_match.group(1) - - raise RuntimeError("Unable to find version string.") - - -class Tox(TestCommand): - """Tox class.""" - - def finalize_options(self): - """Finalize options.""" - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - """Run tests.""" - # import here, cause outside the eggs aren't loaded - import tox - - errcode = tox.cmdline(self.test_args) - sys.exit(errcode) - - -INSTALL_REQUIREMENTS = [ - # A string or list of strings specifying what other distributions need to - # be installed when this one is. - "httpx>=0.26.0", - "jsonschema>=3.2.0", -] - -SETUP_REQUIREMENTS = [] - -TESTS_REQUIREMENTS = [] - -CLASSIFIERS = [ - # How mature is this project? Common values are - # 2 - Pre-Alpha - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Natural Language :: English", -] - -setup( - author="Stefan Kasberger", - author_email="stefan.kasberger@univie.ac.at", - name="pyDataverse", - version=find_version("src", "pyDataverse", "__init__.py"), - description="A Python module for the Dataverse API's and its data-types", - long_description=read_file("README.md"), - long_description_content_type="text/markdown", - license="MIT", - url="https://github.com/gdcc/pyDataverse", - python_requires=">=3.6", - platforms=["OS Independent"], - classifiers=CLASSIFIERS, - install_requires=INSTALL_REQUIREMENTS, - packages=find_packages("src"), - package_dir={"": "src"}, - setup_requires=SETUP_REQUIREMENTS, - tests_require=TESTS_REQUIREMENTS, - cmdclass={"test": Tox}, - include_package_data=True, - keywords=["pyDataverse", "dataverse", "api"], - zip_safe=False, - project_urls={ - "Documentation": "https://pydataverse.readthedocs.io/", - "Issue Tracker": "https://github.com/gdcc/pyDataverse/issues", - "Changelog": "https://pydataverse.readthedocs.io/en/latest/community/releases.html", - }, -) diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 3786179..f882127 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -1,16 +1,13 @@ -from datetime import datetime -import json import os import pytest from httpx import Response from time import sleep from pyDataverse.api import NativeApi from pyDataverse.exceptions import ApiAuthorizationError -from pyDataverse.exceptions import ApiResponseError from pyDataverse.exceptions import ApiUrlError from pyDataverse.models import Dataset from pyDataverse.utils import read_file -from ..conftest import test_config, import_dataverse_min_dict, import_dataset_min_dict +from ..conftest import test_config BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) @@ -129,7 +126,7 @@ def test_token_empty_string(self): def test_token_right_create_dataset_rights(self): BASE_URL = os.getenv("BASE_URL").rstrip("/") api_su = NativeApi(BASE_URL, os.getenv("API_TOKEN_SUPERUSER")) - api_nru = NativeApi(BASE_URL, os.getenv("API_TOKEN_TEST_NO_RIGHTS")) + # api_nru = NativeApi(BASE_URL, os.getenv("API_TOKEN_TEST_NO_RIGHTS")) resp = api_su.get_info_version() assert resp.json()["data"]["version"] == os.getenv("DV_VERSION") diff --git a/tests/conftest.py b/tests/conftest.py index 8b32cd0..76047a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,15 @@ """Find out more at https://github.com/GDCC/pyDataverse.""" + import os import pytest from pyDataverse.api import NativeApi -from pyDataverse.utils import read_json def test_config(): test_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.dirname(test_dir) test_data_dir = os.path.join(test_dir, "data") - json_schemas_dir = os.path.join(root_dir, "src/pyDataverse/schemas/json") + json_schemas_dir = os.path.join(root_dir, "pyDataverse/schemas/json") test_data_output_dir = os.path.join(test_data_dir, "output") invalid_filename_strings = ["wrong", ""] invalid_filename_types = [(), [], 12, 12.12, set(), True, False] diff --git a/tests/data/dataset_upload_full_default.json b/tests/data/dataset_upload_full_default.json index 3e04e9a..848dd1b 100644 --- a/tests/data/dataset_upload_full_default.json +++ b/tests/data/dataset_upload_full_default.json @@ -437,18 +437,13 @@ "typeName": "kindOfData", "multiple": true, "typeClass": "primitive", - "value": [ - "KindOfData1", - "KindOfData2" - ] + "value": ["KindOfData1", "KindOfData2"] }, { "typeName": "language", "multiple": true, "typeClass": "controlledVocabulary", - "value": [ - "German" - ] + "value": ["German"] }, { "typeName": "series", @@ -494,37 +489,25 @@ "typeName": "relatedMaterial", "multiple": true, "typeClass": "primitive", - "value": [ - "RelatedMaterial1", - "RelatedMaterial2" - ] + "value": ["RelatedMaterial1", "RelatedMaterial2"] }, { "typeName": "relatedDatasets", "multiple": true, "typeClass": "primitive", - "value": [ - "RelatedDatasets1", - "RelatedDatasets2" - ] + "value": ["RelatedDatasets1", "RelatedDatasets2"] }, { "typeName": "otherReferences", "multiple": true, "typeClass": "primitive", - "value": [ - "OtherReferences1", - "OtherReferences2" - ] + "value": ["OtherReferences1", "OtherReferences2"] }, { "typeName": "dataSources", "multiple": true, "typeClass": "primitive", - "value": [ - "DataSources1", - "DataSources2" - ] + "value": ["DataSources1", "DataSources2"] }, { "typeName": "originOfSources", @@ -586,10 +569,7 @@ "typeName": "geographicUnit", "multiple": true, "typeClass": "primitive", - "value": [ - "GeographicUnit1", - "GeographicUnit2" - ] + "value": ["GeographicUnit1", "GeographicUnit2"] }, { "typeName": "geographicBoundingBox", @@ -633,19 +613,13 @@ "typeName": "unitOfAnalysis", "multiple": true, "typeClass": "primitive", - "value": [ - "UnitOfAnalysis1", - "UnitOfAnalysis2" - ] + "value": ["UnitOfAnalysis1", "UnitOfAnalysis2"] }, { "typeName": "universe", "multiple": true, "typeClass": "primitive", - "value": [ - "Universe1", - "Universe2" - ] + "value": ["Universe1", "Universe2"] }, { "typeName": "timeMethod", @@ -802,39 +776,25 @@ "typeName": "astroType", "multiple": true, "typeClass": "controlledVocabulary", - "value": [ - "Image", - "Mosaic", - "EventList", - "Cube" - ] + "value": ["Image", "Mosaic", "EventList", "Cube"] }, { "typeName": "astroFacility", "multiple": true, "typeClass": "primitive", - "value": [ - "Facility1", - "Facility2" - ] + "value": ["Facility1", "Facility2"] }, { "typeName": "astroInstrument", "multiple": true, "typeClass": "primitive", - "value": [ - "Instrument1", - "Instrument2" - ] + "value": ["Instrument1", "Instrument2"] }, { "typeName": "astroObject", "multiple": true, "typeClass": "primitive", - "value": [ - "Object1", - "Object2" - ] + "value": ["Object1", "Object2"] }, { "typeName": "resolution.Spatial", @@ -858,19 +818,13 @@ "typeName": "coverage.Spectral.Bandpass", "multiple": true, "typeClass": "primitive", - "value": [ - "Bandpass1", - "Bandpass2" - ] + "value": ["Bandpass1", "Bandpass2"] }, { "typeName": "coverage.Spectral.CentralWavelength", "multiple": true, "typeClass": "primitive", - "value": [ - "3001", - "3002" - ] + "value": ["3001", "3002"] }, { "typeName": "coverage.Spectral.Wavelength", @@ -946,10 +900,7 @@ "typeName": "coverage.Spatial", "multiple": true, "typeClass": "primitive", - "value": [ - "SkyCoverage1", - "SkyCoverage2" - ] + "value": ["SkyCoverage1", "SkyCoverage2"] }, { "typeName": "coverage.Depth", @@ -1056,10 +1007,7 @@ "typeName": "studyAssayOtherOrganism", "multiple": true, "typeClass": "primitive", - "value": [ - "OtherOrganism1", - "OtherOrganism2" - ] + "value": ["OtherOrganism1", "OtherOrganism2"] }, { "typeName": "studyAssayMeasurementType", @@ -1076,10 +1024,7 @@ "typeName": "studyAssayOtherMeasurmentType", "multiple": true, "typeClass": "primitive", - "value": [ - "OtherMeasurementType1", - "OtherMeasurementType2" - ] + "value": ["OtherMeasurementType1", "OtherMeasurementType2"] }, { "typeName": "studyAssayTechnologyType", @@ -1088,7 +1033,7 @@ "value": [ "culture based drug susceptibility testing, single concentration", "culture based drug susceptibility testing, two concentrations", - "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", + "culture based drug susceptibility testing, three or more concentrations (minimum inhibitory concentration measurement)", "flow cytometry" ] }, @@ -1097,20 +1042,17 @@ "multiple": true, "typeClass": "controlledVocabulary", "value": [ - "210-MS GC Ion Trap (Varian)", - "220-MS GC Ion Trap (Varian)", - "225-MS GC Ion Trap (Varian)", - "300-MS quadrupole GC/MS (Varian)" + "210-MS GC Ion Trap (Variant)", + "220-MS GC Ion Trap (Variant)", + "225-MS GC Ion Trap (Variant)", + "300-MS quadrupole GC/MS (Variant)" ] }, { "typeName": "studyAssayCellType", "multiple": true, "typeClass": "primitive", - "value": [ - "CellType1", - "CellType2" - ] + "value": ["CellType1", "CellType2"] } ] }, diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index baf5d9d..9a1088e 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -1,4 +1,4 @@ -from pyDataverse.utils import read_json, save_tree_data, dataverse_tree_walker +from pyDataverse.utils import read_json, dataverse_tree_walker from ..conftest import test_config diff --git a/tox.ini b/tox.ini index ed5c4a3..3b3c173 100644 --- a/tox.ini +++ b/tox.ini @@ -49,20 +49,20 @@ description = pylint for linting deps = -r{toxinidir}/requirements/lint.txt commands = - pylint src/pyDataverse/ + pylint pyDataverse/ pylint tests/ [testenv:mypy] deps = -r{toxinidir}/requirements/lint.txt commands = - mypy src/pyDataverse/ setup.py tests/ + mypy pyDataverse/ setup.py tests/ [flake8] max-line-length = 80 ignore = E129 exclude = - src/pyDataverse/docs/source/conf.py + pyDataverse/docs/source/conf.py .tox .egg @@ -73,7 +73,7 @@ deps = commands = pip uninstall -y flake8-pytest-style pip uninstall -y flake8-rst-docstrings - flake8 src/pyDataverse/ + flake8 pyDataverse/ [testenv:flake8_docs] description = flake8 for style guide and docstring testing @@ -88,7 +88,7 @@ commands = pip uninstall -y flake8-comprehensions pip uninstall -y flake8-requirements pip uninstall -y flake8-return - flake8 --docstring-convention numpy src/pyDataverse/ + flake8 --docstring-convention numpy pyDataverse/ flake8 --docstring-convention numpy tests/ [testenv:flake8_tests] @@ -103,7 +103,7 @@ description = black for auto-formatting deps = -r{toxinidir}/requirements/lint.txt commands = - black src/pyDataverse/ + black pyDataverse/ black tests/ [testenv:pre-commit] @@ -116,14 +116,14 @@ description = invoke sphinx-build to build the HTML docs deps = -r{toxinidir}/requirements/docs.txt commands = - sphinx-build -d src/pyDataverse/docs/build/docs_doctree src/pyDataverse/docs/source docs/build/html --color -b html {posargs} + sphinx-build -d pyDataverse/docs/build/docs_doctree pyDataverse/docs/source docs/build/html --color -b html {posargs} [testenv:pydocstyle] description = pydocstyle for auto-formatting deps = -r{toxinidir}/requirements/docs.txt commands = - pydocstyle src/pyDataverse/ + pydocstyle pyDataverse/ pydocstyle tests/ [testenv:packaging] @@ -133,7 +133,7 @@ recreate = True deps = -r{toxinidir}/requirements/packaging.txt commands = - # rm -R {toxinidir}/src/pyDataverse.egg-info + # rm -R {toxinidir}/pyDataverse.egg-info # rm -R {toxinidir}/dist # rm -R {toxinidir}/build python setup.py sdist bdist_wheel @@ -169,14 +169,14 @@ description = Radon McCabe number deps = -r{toxinidir}/requirements/lint.txt commands = - radon cc src/pyDataverse/ -a + radon cc pyDataverse/ -a [testenv:radon-mi] description = Radon Maintainability Index deps = -r{toxinidir}/requirements/lint.txt commands = - radon mi src/pyDataverse/ + radon mi pyDataverse/ radon mi tests/ [testenv:radon-raw] @@ -184,7 +184,7 @@ description = Radon raw metrics deps = -r{toxinidir}/requirements/lint.txt commands = - radon raw src/pyDataverse/ + radon raw pyDataverse/ radon raw tests/ [testenv:radon-hal] @@ -192,5 +192,5 @@ description = Radon Halstead metrics deps = -r{toxinidir}/requirements/lint.txt commands = - radon hal src/pyDataverse/ + radon hal pyDataverse/ radon hal tests/