Merge pull request #290 from capitalone/develop

v0.6.0

.github/workflows/edgetest.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     name: running edgetest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           ref: develop
       - name: Copy files for locopy

.github/workflows/publish-docs.yml

@@ -12,14 +12,14 @@ jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         # fetch all tags so `versioneer` can properly determine current version
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
     - name: Install dependencies
       run: python -m pip install .[dev]
     - name: Build

.github/workflows/publish-package.yml

@@ -11,12 +11,12 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         # fetch all tags so `versioneer` can properly determine current version
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Install dependencies

.github/workflows/python-package.yml

@@ -10,17 +10,32 @@ on:
     branches: [develop, main]
 
 jobs:
+  lint-and-format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+            python-version: "3.9"
+      - name: Install dependencies
+        run: python -m pip install .[qa]
+      - name: Linting by ruff
+        run: ruff check
+      - name: Formatting by ruff
+        run: ruff format --check
+
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: [3.9, '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.pre-commit-config.yaml

@@ -1,11 +1,25 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.6.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.7
     hooks:
-      - id: black
-        types: [file, python]
-        language_version: python3.9
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+        types_or: [ python, jupyter ]
+  # # Mypy: Optional static type checking
+  # # https://github.com/pre-commit/mirrors-mypy
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.11.1
+  #   hooks:
+  #     - id: mypy
+  #       exclude: ^(docs|tests)\/
+  #       language_version: python3.9
+  #       args: [--namespace-packages, --explicit-package-bases, --ignore-missing-imports, --non-interactive, --install-types]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
     hooks:
-      - id: isort
+      - id: trailing-whitespace
+      - id: debug-statements
+      - id: end-of-file-fixer

CODEOWNERS

@@ -1 +1 @@
-*       @fdosani @ak-gupta @jdawang @gladysteh99 @NikhilJArora
+*       @fdosani @ak-gupta @jdawang @gladysteh99

README.rst

@@ -13,7 +13,7 @@ A Python library to assist with ETL processing for:
 
 In addition:
 
-- The library supports Python 3.8 to 3.10
+- The library supports Python 3.9 to 3.11
 - DB Driver (Adapter) agnostic. Use your favourite driver that complies with
   `DB-API 2.0 <https://www.python.org/dev/peps/pep-0249/>`_
 - It provides functionality to download and upload data to S3 buckets, and internal stages (Snowflake)

docs/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # SPDX-Copyright: Copyright (c) Capital One Services, LLC
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2018 Capital One Services, LLC
@@ -27,8 +26,9 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-import sys
 import os
+import sys
+
 import sphinx_rtd_theme
 from locopy._version import __version__
 
@@ -151,7 +151,13 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "locopy.tex", "locopy Documentation", "Faisal Dosani, Ian Robertson", "manual")
+    (
+        master_doc,
+        "locopy.tex",
+        "locopy Documentation",
+        "Faisal Dosani, Ian Robertson",
+        "manual",
+    )
 ]
 
 
@@ -185,4 +191,9 @@
 intersphinx_mapping = {"https://docs.python.org/": None}
 
 # autodoc
-autodoc_default_flags = ["members", "undoc-members", "show-inheritance", "inherited-members"]
+autodoc_default_flags = [
+    "members",
+    "undoc-members",
+    "show-inheritance",
+    "inherited-members",
+]

locopy/__init__.py

@@ -13,8 +13,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""A Python library to assist with ETL processing."""
 
-from .database import Database
-from .redshift import Redshift
-from .s3 import S3
-from .snowflake import Snowflake
+from locopy.database import Database
+from locopy.redshift import Redshift
+from locopy.s3 import S3
+from locopy.snowflake import Snowflake
+
+__all__ = ["S3", "Database", "Redshift", "Snowflake"]

locopy/_version.py

@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.5.9"
+__version__ = "0.6.0"

locopy/database.py

@@ -14,20 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Database Module
-"""
+"""Database Module."""
+
 import time
 
-from .errors import CredentialsError, DBError
-from .logger import INFO, get_logger
-from .utility import read_config_yaml
+from locopy.errors import CredentialsError, DBError
+from locopy.logger import INFO, get_logger
+from locopy.utility import read_config_yaml
 
 logger = get_logger(__name__, INFO)
 
 
-class Database(object):
-    """This is the base class for all DBAPI 2 database connectors which will inherit this
-    functionality. The ``Database`` class will manage connections and handle executing queries.
+class Database:
+    """Base class for all DBAPI 2 database connectors which will inherit this functionality.
+
+    The ``Database`` class will manage connections and handle executing queries.
     Most of the functionality should work out of the box for classes which inherit minus the
     abstract method for ``connect`` which may vary across databases.
 
@@ -72,13 +73,16 @@ def __init__(self, dbapi, config_yaml=None, **kwargs):
         self.cursor = None
 
         if config_yaml and self.connection:
-            raise CredentialsError("Please provide kwargs or a YAML configuraton, not both.")
+            raise CredentialsError(
+                "Please provide kwargs or a YAML configuraton, not both."
+            )
         if config_yaml:
             self.connection = read_config_yaml(config_yaml)
 
     def connect(self):
-        """Creates a connection to a database by setting the values of the ``conn`` and ``cursor``
-        attributes.
+        """Create a connection to a database.
+
+        Sets the values of the ``conn`` and ``cursor`` attributes.
 
         Raises
         ------
@@ -90,11 +94,12 @@ def connect(self):
             self.cursor = self.conn.cursor()
         except Exception as e:
             logger.error("Error connecting to the database. err: %s", e)
-            raise DBError("Error connecting to the database.")
+            raise DBError("Error connecting to the database.") from e
 
     def disconnect(self):
-        """Terminates the connection by closing the values of the ``conn`` and ``cursor``
-        attributes.
+        """Terminate the connection.
+
+        Closes the values of the ``conn`` and ``cursor`` attributes.
 
         Raises
         ------
@@ -108,7 +113,9 @@ def disconnect(self):
                 self.conn.close()
             except Exception as e:
                 logger.error("Error disconnecting from the database. err: %s", e)
-                raise DBError("There is a problem disconnecting from the database.")
+                raise DBError(
+                    "There is a problem disconnecting from the database."
+                ) from e
         else:
             logger.info("No connection to close")
 
@@ -153,7 +160,7 @@ def execute(self, sql, commit=True, params=(), many=False, verbose=True):
                     self.cursor.execute(sql, params)
             except Exception as e:
                 logger.error("Error running SQL query. err: %s", e)
-                raise DBError("Error running SQL query.")
+                raise DBError("Error running SQL query.") from e
             if commit:
                 self.conn.commit()
             elapsed = time.time() - start_time
@@ -167,8 +174,9 @@ def execute(self, sql, commit=True, params=(), many=False, verbose=True):
             raise DBError("Cannot execute SQL on a closed connection.")
 
     def column_names(self):
-        """Pull column names out of the cursor description. Depending on the
-        DBAPI, it could return column names as bytes: ``b'column_name'``
+        """Pull column names out of the cursor description.
+
+        Depending on the DBAPI, it could return column names as bytes: ``b'column_name'``.
 
         Returns
         -------
@@ -177,12 +185,13 @@ def column_names(self):
         """
         try:
             return [column[0].decode().lower() for column in self.cursor.description]
-        except:
+        except Exception:
             return [column[0].lower() for column in self.cursor.description]
 
     def to_dataframe(self, size=None):
-        """Return a dataframe of the last query results.  This imports Pandas
-        in here, so that it's not needed for other use cases.  This is just a
+        """Return a dataframe of the last query results.
+
+        This imports Pandas in here, so that it's not needed for other use cases.  This is just a
         convenience method.
 
         Parameters
@@ -214,7 +223,7 @@ def to_dataframe(self, size=None):
         return pandas.DataFrame(fetched, columns=columns)
 
     def to_dict(self):
-        """Generate dictionaries of rows
+        """Generate dictionaries of rows.
 
         Yields
         ------
@@ -226,7 +235,7 @@ def to_dict(self):
             yield dict(zip(columns, row))
 
     def _is_connected(self):
-        """Checks the connection and cursor class arrtribues are initalized.
+        """Check the connection and cursor class arrtributes are initalized.
 
         Returns
         -------
@@ -235,16 +244,18 @@ def _is_connected(self):
         """
         try:
             return self.conn is not None and self.cursor is not None
-        except:
+        except Exception:
             return False
 
     def __enter__(self):
+        """Open the connection."""
         logger.info("Connecting...")
         self.connect()
         logger.info("Connection established.")
         return self
 
     def __exit__(self, exc_type, exc, exc_tb):
+        """Close the connection."""
         logger.info("Closing connection...")
         self.disconnect()
         logger.info("Connection closed.")