DO NOT MERGE - TPX PoC code

app_layer/adapters/csv_file_processor.py

@@ -0,0 +1,57 @@
+import os
+from typing import Optional, Any, List
+from app_layer.ports.file_processor import FileProcessor
+from hr.models import HRModel
+
+from payroll import models as payroll_models
+from payroll.models import PayrollEntry
+
+
+class CsvFileProcessor(FileProcessor):
+    """
+    Class CsvFileProcessor
+
+    This class is a subclass of FileProcessor and is responsible for processing CSV files and sending the processed content to an output adapter.
+
+    Methods:
+    - process_file(bucket_name: str, file_path: str, results: List[Optional[Any]] = None) -> bool
+    - send_to_output(log: LogService, output_adapter, file_path: str, content: str)
+    """
+
+    def process_file(self, bucket_name: str, file_path: str):
+        # log.deb(f'processing csv file: {file_path}...')
+        is_valid = os.path.basename(file_path.lower()).startswith(('hrauto',
+                                                                   'payrollauto'))
+
+        if not is_valid:
+            return False
+
+        results = []
+        file_type_mapping = {
+            'hrauto': 'hr',
+            'payrollauto': 'payroll'
+        }
+
+        # Determine file type based on prefix
+        for prefix, filetype in file_type_mapping.items():
+            if file_path.lower().startswith(prefix):
+                results.append(dict(file_type=filetype))
+                break
+
+        # Get file type (without popping from list) from results list (value type dict)
+        filetype = results[-1].get('file_type')
+
+        # Extract file_name from file_path
+        if filetype == 'hr':
+            hr_model = HRModel()
+            hr_model.parse_csv(bucket_name, file_path)
+        elif filetype == 'payroll':
+            payroll_model = PayrollEntry()
+            payroll_model.parse_csv(bucket_name, file_path)
+        else:
+            return False
+
+        return True
+
+    def send_to_output(self, output_adapter, file_path: str, content: str):
+        output_adapter.send(file_path, content)

app_layer/adapters/excel_file_processor.py

@@ -0,0 +1,17 @@
+# adapters/excel_file_processor.py
+
+import os
+from typing import Optional, Any
+
+from app_layer.ports.file_processor import FileProcessor
+
+
+class ExcelFileProcessor(FileProcessor):
+    def process_file(self, bucket_name, file_path: str, results: Optional[Any] = None):
+        # log.deb(f'processing excel file: {file_path}...')
+        is_valid = os.path.basename(file_path).startswith('ActualsDBT')
+        # log.deb(f'processing excel file: {file_path} - valid[{is_valid}]')
+        return is_valid
+
+    def send_to_output(self, output_adapter, file_path: str, content: str):
+        output_adapter.send(file_path, content)

app_layer/adapters/s3_output_bucket.py

@@ -0,0 +1,7 @@
+from app_layer.ports.output_service import OutputService
+
+
+class S3OutputBucket(OutputService):
+    def send(self, output_adapter, file_path: str, content: str):
+        # Email sending logic goes here
+        print(f'sending email with content: {content} for file: {file_path}')

app_layer/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

app_layer/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AppLayerConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'app_layer'

app_layer/exception.py

@@ -0,0 +1,130 @@
+#  Pixel Code
+#  Email: [email protected]
+#
+#  Copyright (c) 2024.
+#
+#  All rights reserved.
+#
+#  No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+#  BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+#  SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+#  OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#  For permission requests, write to the author, at the email address above.
+
+"""
+exception.py contains custom exception classes for the service.
+
+The ServiceException class is a custom exception class for the service.
+
+Attributes
+----------
+message (str)
+    The message of the exception
+
+Methods
+----------
+append_message(message) -> ServiceException
+    Append a message to the exception
+to_dict() -> dict
+    Convert the exception to a dictionary
+
+Custom exceptions are defined for:
+    - Empty or None Value
+    - Unknown Value
+    - Value Exists
+    - Key Not Found
+"""
+__version__: str = '0.0.1'
+
+
+class ServiceException(Exception):
+    """
+    A custom exception class for service
+
+    Attributes
+    ----------
+    message (str)
+        The message of the exception
+
+    Methods
+    ----------
+    append_message(message) -> ServiceException
+        Append a message to the exception
+    to_dict() -> dict
+        Convert the exception to a dictionary
+
+    """
+
+    message = None
+
+    def __init__(self, message: str):
+        """
+        Constructs all the necessary attributes for the ServiceException object.
+
+        Parameters
+        ----------
+        message : str
+            The message of the exception
+        """
+        self.message = message
+        super().__init__(self.message)
+
+    @classmethod
+    def append_message(cls, message) -> 'ServiceException':
+        """
+        Append a message to the exception
+
+        Parameters
+        ----------
+        message : str
+            The message to append to the exception
+
+        Returns
+        ----------
+        ServiceException
+            The exception with the appended message
+        """
+        cls.message = message if cls.message is None else cls.message + f" - {message}"
+        return cls(cls.message)
+
+    def to_dict(self) -> dict:
+        """
+        Convert the exception to a dictionary
+
+        Returns
+        ----------
+        dict
+            The exception as a dictionary
+        """
+        return {
+            "message": self.message
+        }
+
+
+# Custom exception for empty or none value not permitted
+class EmptyOrNoneValueException(ServiceException):
+    def __init__(self, message="empty or none value not permitted"):
+        super().__init__(message)
+
+
+# Custom exception for unknown and/or not recognised value
+class UnknownValueException(ServiceException):
+    def __init__(self, message="unknown and/or not recognised value"):
+        super().__init__(message)
+
+
+# Custom exception for value already exists
+class ValueExistsException(ServiceException):
+    def __init__(self, message="value already exists"):
+        super().__init__(message)
+
+
+# Custom exception for key not found
+class KeyNotFoundException(ServiceException):
+    def __init__(self, message="key not found"):
+        super().__init__(message)

app_layer/layer/service.py

@@ -0,0 +1,45 @@
+from typing import Any
+from app_layer.adapters.csv_file_processor import CsvFileProcessor
+from app_layer.adapters.excel_file_processor import ExcelFileProcessor
+from hr.models import HRModel
+
+
+def process_data_input_source_files(file_paths: [str]) -> dict[str, Any]:
+    if not file_paths:
+        return {'status_code': 204, 'message': 'no files to process'}
+
+    buckets = {}
+    for path in file_paths:
+        bucket_name, file_name = path.split('/', 1)
+        file_type = file_name.split('.')[-1].lower()
+
+        if bucket_name not in buckets:
+            buckets[bucket_name] = {}
+        buckets[bucket_name][file_name] = file_type
+
+    processors = {
+        'csv': CsvFileProcessor(),
+        'xlsx': ExcelFileProcessor(),
+        'xls': ExcelFileProcessor()
+    }
+
+    # Dictionary to store valid data file processors
+    valid_file_processors = {}
+
+    # Process each file and store the valid data file processors into database
+    for bucket_name, files in buckets.items():
+        for file_name, file_type in files.items():
+            file_path = f'{bucket_name}/{file_name}'
+            processor = processors.get(file_type)
+            if processor:
+                valid_file_processors[file_path] = processor \
+                    if processor.process_file(bucket_name, file_path) \
+                    else None
+            else:
+                return {'status_code': 400, 'message': 'invalid file type'}
+
+    # HR service
+    hr_model = HRModel()
+    hr_model.update_records_with_basic_pay_superannuation_ernic_values()
+    return {'status_code': 200, 'message': 'all input processor files processed'}
+

app_layer/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

app_layer/ports/file_processor.py

@@ -0,0 +1,61 @@
+from abc import ABCMeta, abstractmethod
+from typing import List, Optional, Any
+
+
+class FileProcessor(metaclass=ABCMeta):
+    """
+    This code defines an abstract base class called FileProcessor.
+
+    It is designed to be subclassed by other classes that will implement the functionality of processing a file and
+    sending the processed content to an output adapter.
+
+    The class has two abstract methods:
+
+    1. process_file(self, file_path: str):
+       - This method is responsible for processing the content of a file located at the given file_path.
+       - Subclasses must override this method and provide their own implementation.
+
+    2. send_to_output(self, output_adapter, file_path: str, content: str):
+       - This method is responsible for sending the processed content to an output adapter.
+       - It takes an output_adapter object, the file_path of the processed file, and the content that needs to be sent.
+       - Subclasses must override this method and provide their own implementation.
+
+    This class should not be instantiated directly. Instead, it should be used as a base class for implementing specific
+    file processing functionality.
+    """
+    @abstractmethod
+    def process_file(self, bucket_name: str, file_path: str, results: List[Optional[Any]] = None):
+        """
+        This method is an abstract method that should be overridden by a subclass.
+
+        Parameters:
+        - log: An instance of the LogService class used for logging.
+        - file_path: The path of the file to be processed.
+        - file_path: str
+
+        Returns:
+        None
+
+        This method is an abstract method and must be implemented by subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def send_to_output(self, output_adapter, file_path: str, content: str):
+        """
+        send_to_output(self, log: LogService, output_adapter, file_path: str, content: str)
+
+        Sends the specified content to the output using the provided output adapter.
+
+        Parameters:
+        - log (LogService): The log service to use for logging events and errors.
+        - output_adapter: The output adapter that handles the specific output mechanism.
+        - file_path (str): The path to the file where the content will be sent.
+        - content (str): The content to send.
+
+        Returns:
+        None
+
+        This method is an abstract method and must be implemented by subclasses.
+        """
+        pass

app_layer/ports/output_service.py

@@ -0,0 +1,43 @@
+# ports/output_service.py
+
+from abc import ABCMeta, abstractmethod
+
+
+class OutputService(metaclass=ABCMeta):
+    """
+    This is the documentation for the `OutputService` class.
+
+    Class: OutputService
+    --------------------
+    This is an abstract base class that defines the interface for sending output. It is meant to be subclassed and
+    implemented by concrete output service classes.
+
+    Methods
+    -------
+    send(file_path: str, content: str)
+        This is an abstract method that subclasses must implement. It takes two parameters:
+        - file_path: A string representing the file path where the output should be sent.
+        - content: A string representing the content of the output.
+
+        Note:
+        - This method should be overridden in subclasses with the specific implementation logic for sending the output.
+
+    Exceptions
+    ----------
+    None
+
+    Example usage
+    -------------
+    ```
+    class MyOutputService(OutputService):
+        def send(self, log: LogService, ssm_client: BaseClient, file_path: str, content: str):
+            # Implement the logic to send the output to the specified file path
+            pass
+
+    output_service = MyOutputService()
+    output_service.send('/path/to/file.txt', 'Hello, World!')
+    ```
+    """
+    @abstractmethod
+    def send(self, ssm_client, file_path: str, content: str):
+        pass

app_layer/serializers.py

@@ -0,0 +1,7 @@
+# app_layer/serializers.py
+from rest_framework import serializers
+
+
+class S3EventSerializer(serializers.Serializer):
+    bucket = serializers.CharField()
+    key = serializers.CharField()

app_layer/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.