From a7f2f1097ff37fee4cc414854d4cc36fca2535fa Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Wed, 28 Aug 2024 21:55:53 +0100 Subject: [PATCH 01/23] feat:started to add models and migrations for HR, payroll, and transactions This commit introduces new Django models for the my_hr, payroll, and transactions apps. Corresponding migrations have been created to support these models. The settings have also been updated to include these new apps. --- config/settings/base.py | 5 +-- my_hr/__init__.py | 0 my_hr/migrations/0001_initial.py | 44 +++++++++++++++++++++++++ my_hr/migrations/__init__.py | 0 my_hr/models.py | 31 +++++++++++++++++ payroll/__init__.py | 0 payroll/migrations/0001_initial.py | 41 +++++++++++++++++++++++ payroll/migrations/__init__.py | 0 payroll/models.py | 29 ++++++++++++++++ transactions/__init__.py | 0 transactions/migrations/0001_initial.py | 33 +++++++++++++++++++ transactions/migrations/__init__.py | 0 transactions/models.py | 20 +++++++++++ 13 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 my_hr/__init__.py create mode 100644 my_hr/migrations/0001_initial.py create mode 100644 my_hr/migrations/__init__.py create mode 100644 my_hr/models.py create mode 100644 payroll/__init__.py create mode 100644 payroll/migrations/0001_initial.py create mode 100644 payroll/migrations/__init__.py create mode 100644 payroll/models.py create mode 100644 transactions/__init__.py create mode 100644 transactions/migrations/0001_initial.py create mode 100644 transactions/migrations/__init__.py create mode 100644 transactions/models.py diff --git a/config/settings/base.py b/config/settings/base.py index 128b380b1..80cc373e1 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -17,11 +17,9 @@ import environ from dbt_copilot_python.database import database_url_from_env from dbt_copilot_python.utility import is_copilot -from django.urls import reverse_lazy from django_log_formatter_asim import ASIMFormatter from django_log_formatter_ecs import ECSFormatter - BASE_DIR = Path(__file__).resolve().parent.parent.parent env = environ.Env() @@ -77,6 +75,9 @@ "simple_history", "axes", "django_chunk_upload_handlers", + "transactions", + "my_hr", + "payroll" ] ROOT_URLCONF = "config.urls" diff --git a/my_hr/__init__.py b/my_hr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/my_hr/migrations/0001_initial.py b/my_hr/migrations/0001_initial.py new file mode 100644 index 000000000..a017f81da --- /dev/null +++ b/my_hr/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 5.1 on 2024-08-28 20:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Employee', + fields=[ + ('employee_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('group', models.CharField(max_length=100)), + ('directorate', models.CharField(max_length=100)), + ('cost_centre', models.CharField(max_length=100)), + ('cost_centre_name', models.CharField(max_length=100)), + ('last_name', models.CharField(max_length=100)), + ('first_name', models.CharField(max_length=100)), + ('se_no', models.CharField(max_length=100, unique=True)), + ('salary', models.DecimalField(decimal_places=2, max_digits=10)), + ('grade', models.CharField(max_length=100)), + ('employee_location_city_name', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('assignment_status', models.CharField(max_length=100)), + ('appointment_status', models.CharField(max_length=100)), + ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), + ('fte', models.DecimalField(decimal_places=2, max_digits=4)), + ('wmi_person', models.CharField(max_length=100)), + ('wmi', models.CharField(max_length=100)), + ('actual_group', models.CharField(max_length=100)), + ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), + ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), + ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ('total', models.DecimalField(decimal_places=2, max_digits=10)), + ('costing_cc', models.CharField(max_length=100)), + ('return_field', models.CharField(max_length=100)), + ], + ), + ] diff --git a/my_hr/migrations/__init__.py b/my_hr/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/my_hr/models.py b/my_hr/models.py new file mode 100644 index 000000000..6a2fb6dc8 --- /dev/null +++ b/my_hr/models.py @@ -0,0 +1,31 @@ +from django.db import models + +class Employee(models.Model): + employee_id = models.CharField(max_length=100, unique=True, primary_key=True) + group = models.CharField(max_length=100) + directorate = models.CharField(max_length=100) + cost_centre = models.CharField(max_length=100) + cost_centre_name = models.CharField(max_length=100) + last_name = models.CharField(max_length=100) + first_name = models.CharField(max_length=100) + se_no = models.CharField(max_length=100, unique=True) + salary = models.DecimalField(max_digits=10, decimal_places=2) + grade = models.CharField(max_length=100) + employee_location_city_name = models.CharField(max_length=100) + person_type = models.CharField(max_length=100) + assignment_status = models.CharField(max_length=100) + appointment_status = models.CharField(max_length=100) + working_hours = models.DecimalField(max_digits=5, decimal_places=2) + fte = models.DecimalField(max_digits=4, decimal_places=2) + wmi_person = models.CharField(max_length=100) + wmi = models.CharField(max_length=100) + actual_group = models.CharField(max_length=100) + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) + total = models.DecimalField(max_digits=10, decimal_places=2) + costing_cc = models.CharField(max_length=100) + return_field = models.CharField(max_length=100) + + def __str__(self): + return self.employee_id \ No newline at end of file diff --git a/payroll/__init__.py b/payroll/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py new file mode 100644 index 000000000..3308536d7 --- /dev/null +++ b/payroll/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 5.1 on 2024-08-28 20:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Payroll', + fields=[ + ('payroll_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('business_unit_number', models.CharField(max_length=100)), + ('business_unit_name', models.CharField(max_length=100)), + ('cost_center_number', models.CharField(max_length=100)), + ('cost_center_name', models.CharField(max_length=100)), + ('employee_name', models.CharField(max_length=100)), + ('employee_number', models.CharField(max_length=100)), + ('assignment_number', models.CharField(max_length=100)), + ('payroll_name', models.CharField(max_length=100)), + ('employee_organization', models.CharField(max_length=100)), + ('employee_location', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('employee_category', models.CharField(max_length=100)), + ('assignment_type', models.CharField(max_length=100)), + ('position', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('account_code', models.CharField(max_length=100)), + ('account_name', models.CharField(max_length=100)), + ('pay_element_name', models.CharField(max_length=100)), + ('effective_date', models.DateField()), + ('debit_amount', models.DecimalField(decimal_places=2, max_digits=10)), + ('credit_amount', models.DecimalField(decimal_places=2, max_digits=10)), + ], + ), + ] diff --git a/payroll/migrations/__init__.py b/payroll/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/payroll/models.py b/payroll/models.py new file mode 100644 index 000000000..f0dad012a --- /dev/null +++ b/payroll/models.py @@ -0,0 +1,29 @@ +from django.db import models + + +class Payroll(models.Model): + payroll_id = models.CharField(max_length=100, unique=True, primary_key=True) + business_unit_number = models.CharField(max_length=100) + business_unit_name = models.CharField(max_length=100) + cost_center_number = models.CharField(max_length=100) + cost_center_name = models.CharField(max_length=100) + employee_name = models.CharField(max_length=100) + employee_number = models.CharField(max_length=100) + assignment_number = models.CharField(max_length=100) + payroll_name = models.CharField(max_length=100) + employee_organization = models.CharField(max_length=100) + employee_location = models.CharField(max_length=100) + person_type = models.CharField(max_length=100) + employee_category = models.CharField(max_length=100) + assignment_type = models.CharField(max_length=100) + position = models.CharField(max_length=100) + grade = models.CharField(max_length=100) + account_code = models.CharField(max_length=100) + account_name = models.CharField(max_length=100) + pay_element_name = models.CharField(max_length=100) + effective_date = models.DateField() + debit_amount = models.DecimalField(max_digits=10, decimal_places=2) + credit_amount = models.DecimalField(max_digits=10, decimal_places=2) + + def __str__(self): + return self.payroll_id diff --git a/transactions/__init__.py b/transactions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/transactions/migrations/0001_initial.py b/transactions/migrations/0001_initial.py new file mode 100644 index 000000000..eb268a2b5 --- /dev/null +++ b/transactions/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 5.1 on 2024-08-28 20:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Transactions', + fields=[ + ('transaction_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('source', models.CharField(max_length=100)), + ('entity', models.CharField(max_length=100)), + ('cost_centre', models.CharField(max_length=100)), + ('group', models.CharField(max_length=100)), + ('account', models.CharField(max_length=100)), + ('programme', models.CharField(max_length=100)), + ('line_description', models.TextField()), + ('net', models.DecimalField(decimal_places=2, max_digits=10)), + ('fiscal_period', models.CharField(max_length=100)), + ('date_of_journal', models.DateField()), + ('purchase_order_number', models.CharField(max_length=100)), + ('supplier_name', models.CharField(max_length=100)), + ('level4_code', models.CharField(max_length=100)), + ], + ), + ] diff --git a/transactions/migrations/__init__.py b/transactions/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/transactions/models.py b/transactions/models.py new file mode 100644 index 000000000..f3f76501a --- /dev/null +++ b/transactions/models.py @@ -0,0 +1,20 @@ +from django.db import models + +class Transactions(models.Model): + transaction_id = models.CharField(max_length=100, unique=True, primary_key=True) + source = models.CharField(max_length=100) + entity = models.CharField(max_length=100) + cost_centre = models.CharField(max_length=100) + group = models.CharField(max_length=100) + account = models.CharField(max_length=100) + programme = models.CharField(max_length=100) + line_description = models.TextField() + net = models.DecimalField(max_digits=10, decimal_places=2) + fiscal_period = models.CharField(max_length=100) + date_of_journal = models.DateField() + purchase_order_number = models.CharField(max_length=100) + supplier_name = models.CharField(max_length=100) + level4_code = models.CharField(max_length=100) + + def __str__(self): + return self.transaction_id \ No newline at end of file From cc6a333611aa7cb724b16ce2e591e48fd5120930 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Fri, 30 Aug 2024 21:27:51 +0100 Subject: [PATCH 02/23] refactor:added new features to the project Refactored module and directory names for consistency and clarity. Added new modules for CSV and Excel file processing, S3 output, services, and serializers. Improved exception handling and logging functionality across the project. --- {my_hr => app_layer}/__init__.py | 0 app_layer/adapters/csv_file_processor.py | 33 ++ app_layer/adapters/excel_file_processor.py | 16 + app_layer/adapters/s3_output_bucket.py | 7 + app_layer/admin.py | 3 + app_layer/apps.py | 6 + app_layer/exception.py | 130 +++++++ app_layer/layer/service.py | 50 +++ app_layer/log.py | 337 ++++++++++++++++++ {my_hr => app_layer}/migrations/__init__.py | 0 app_layer/models.py | 3 + app_layer/ports/file_processor.py | 60 ++++ app_layer/ports/output_service.py | 45 +++ app_layer/serializers.py | 7 + app_layer/tests.py | 3 + app_layer/urls.py | 7 + app_layer/views.py | 57 +++ chartofaccountDIT/views.py | 4 +- config/settings/base.py | 7 +- {transactions => myhr}/__init__.py | 0 myhr/admin.py | 3 + myhr/apps.py | 6 + {my_hr => myhr}/migrations/0001_initial.py | 2 +- {transactions => myhr}/migrations/__init__.py | 0 {my_hr => myhr}/models.py | 1 + myhr/tests.py | 3 + myhr/views.py | 3 + payroll/admin.py | 3 + payroll/apps.py | 6 + payroll/migrations/0001_initial.py | 3 +- .../0002_alter_payroll_created_at.py | 18 + payroll/models.py | 1 + payroll/tests.py | 3 + payroll/views.py | 3 + requirements.txt | 16 + transaction/__init__.py | 0 transaction/admin.py | 4 + transaction/apps.py | 6 + .../migrations/0001_initial.py | 4 +- transaction/migrations/__init__.py | 0 {transactions => transaction}/models.py | 3 +- transaction/serializers.py | 9 + transaction/tests.py | 3 + transaction/views.py | 3 + 44 files changed, 868 insertions(+), 10 deletions(-) rename {my_hr => app_layer}/__init__.py (100%) create mode 100644 app_layer/adapters/csv_file_processor.py create mode 100644 app_layer/adapters/excel_file_processor.py create mode 100644 app_layer/adapters/s3_output_bucket.py create mode 100644 app_layer/admin.py create mode 100644 app_layer/apps.py create mode 100644 app_layer/exception.py create mode 100644 app_layer/layer/service.py create mode 100644 app_layer/log.py rename {my_hr => app_layer}/migrations/__init__.py (100%) create mode 100644 app_layer/models.py create mode 100644 app_layer/ports/file_processor.py create mode 100644 app_layer/ports/output_service.py create mode 100644 app_layer/serializers.py create mode 100644 app_layer/tests.py create mode 100644 app_layer/urls.py create mode 100644 app_layer/views.py rename {transactions => myhr}/__init__.py (100%) create mode 100644 myhr/admin.py create mode 100644 myhr/apps.py rename {my_hr => myhr}/migrations/0001_initial.py (97%) rename {transactions => myhr}/migrations/__init__.py (100%) rename {my_hr => myhr}/models.py (96%) create mode 100644 myhr/tests.py create mode 100644 myhr/views.py create mode 100644 payroll/admin.py create mode 100644 payroll/apps.py create mode 100644 payroll/migrations/0002_alter_payroll_created_at.py create mode 100644 payroll/tests.py create mode 100644 payroll/views.py create mode 100644 transaction/__init__.py create mode 100644 transaction/admin.py create mode 100644 transaction/apps.py rename {transactions => transaction}/migrations/0001_initial.py (93%) create mode 100644 transaction/migrations/__init__.py rename {transactions => transaction}/models.py (90%) create mode 100644 transaction/serializers.py create mode 100644 transaction/tests.py create mode 100644 transaction/views.py diff --git a/my_hr/__init__.py b/app_layer/__init__.py similarity index 100% rename from my_hr/__init__.py rename to app_layer/__init__.py diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py new file mode 100644 index 000000000..30253c7dc --- /dev/null +++ b/app_layer/adapters/csv_file_processor.py @@ -0,0 +1,33 @@ +import os + +from app_layer.log import LogService +from app_layer.ports.file_processor import FileProcessor + + +class CsvFileProcessor(FileProcessor): + """ + Class CsvFileProcessor + + This class is a subclass of FileProcessor and is responsible for processing CSV files and sending the processed content to an output adapter. + + Methods: + - process_file(file_path: str) -> str: + This method takes a file path as input and reads the CSV file. It processes the DataFrame if needed and returns the string representation of the DataFrame. + + - send_to_output(output_adapter, file_path: str, content: str): + This method takes an output adapter, file path, and content as input. It sends the content to the output adapter using the send() method of the adapter. + + """ + + def process_file(self, log: LogService, file_path: str): + log.deb(f'processing csv file: {file_path}...') + is_valid = os.path.basename(file_path.lower()).startswith(('actualsdbt', + 'hrdbt', + 'payrolldbt', + 'transactionsdbt', + 'budget')) + log.deb(f'processing csv file: {file_path} - valid[{is_valid}]') + return is_valid + + def send_to_output(self, log: LogService, output_adapter, file_path: str, content: str): + output_adapter.send(file_path, content) diff --git a/app_layer/adapters/excel_file_processor.py b/app_layer/adapters/excel_file_processor.py new file mode 100644 index 000000000..567f8fc1e --- /dev/null +++ b/app_layer/adapters/excel_file_processor.py @@ -0,0 +1,16 @@ +# adapters/excel_file_processor.py + +import os + +from app_layer.ports.file_processor import FileProcessor + + +class ExcelFileProcessor(FileProcessor): + def process_file(self, log, file_path: str): + log.deb(f'processing excel file: {file_path}...') + is_valid = os.path.basename(file_path).startswith('ActualsDBT') + log.deb(f'processing excel file: {file_path} - valid[{is_valid}]') + return is_valid + + def send_to_output(self, log, output_adapter, file_path: str, content: str): + output_adapter.send(file_path, content) diff --git a/app_layer/adapters/s3_output_bucket.py b/app_layer/adapters/s3_output_bucket.py new file mode 100644 index 000000000..2babd093d --- /dev/null +++ b/app_layer/adapters/s3_output_bucket.py @@ -0,0 +1,7 @@ +from app_layer.ports.output_service import OutputService + + +class S3OutputBucket(OutputService): + def send(self, log, output_adapter, file_path: str, content: str): + # Email sending logic goes here + print(f'sending email with content: {content} for file: {file_path}') diff --git a/app_layer/admin.py b/app_layer/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/app_layer/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/app_layer/apps.py b/app_layer/apps.py new file mode 100644 index 000000000..eabf69ab6 --- /dev/null +++ b/app_layer/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AppLayerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'app_layer' diff --git a/app_layer/exception.py b/app_layer/exception.py new file mode 100644 index 000000000..8181993e0 --- /dev/null +++ b/app_layer/exception.py @@ -0,0 +1,130 @@ +# Pixel Code +# Email: haresh@pixelcode.uk +# +# Copyright (c) 2024. +# +# All rights reserved. +# +# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# For permission requests, write to the author, at the email address above. + +""" +exception.py contains custom exception classes for the service. + +The ServiceException class is a custom exception class for the service. + +Attributes +---------- +message (str) + The message of the exception + +Methods +---------- +append_message(message) -> ServiceException + Append a message to the exception +to_dict() -> dict + Convert the exception to a dictionary + +Custom exceptions are defined for: + - Empty or None Value + - Unknown Value + - Value Exists + - Key Not Found +""" +__version__: str = '0.0.1' + + +class ServiceException(Exception): + """ + A custom exception class for service + + Attributes + ---------- + message (str) + The message of the exception + + Methods + ---------- + append_message(message) -> ServiceException + Append a message to the exception + to_dict() -> dict + Convert the exception to a dictionary + + """ + + message = None + + def __init__(self, message: str): + """ + Constructs all the necessary attributes for the ServiceException object. + + Parameters + ---------- + message : str + The message of the exception + """ + self.message = message + super().__init__(self.message) + + @classmethod + def append_message(cls, message) -> 'ServiceException': + """ + Append a message to the exception + + Parameters + ---------- + message : str + The message to append to the exception + + Returns + ---------- + ServiceException + The exception with the appended message + """ + cls.message = message if cls.message is None else cls.message + f" - {message}" + return cls(cls.message) + + def to_dict(self) -> dict: + """ + Convert the exception to a dictionary + + Returns + ---------- + dict + The exception as a dictionary + """ + return { + "message": self.message + } + + +# Custom exception for empty or none value not permitted +class EmptyOrNoneValueException(ServiceException): + def __init__(self, message="empty or none value not permitted"): + super().__init__(message) + + +# Custom exception for unknown and/or not recognised value +class UnknownValueException(ServiceException): + def __init__(self, message="unknown and/or not recognised value"): + super().__init__(message) + + +# Custom exception for value already exists +class ValueExistsException(ServiceException): + def __init__(self, message="value already exists"): + super().__init__(message) + + +# Custom exception for key not found +class KeyNotFoundException(ServiceException): + def __init__(self, message="key not found"): + super().__init__(message) diff --git a/app_layer/layer/service.py b/app_layer/layer/service.py new file mode 100644 index 000000000..53ad563c4 --- /dev/null +++ b/app_layer/layer/service.py @@ -0,0 +1,50 @@ +from typing import Any +from botocore.client import BaseClient +from app_layer.adapters.csv_file_processor import CsvFileProcessor +from app_layer.adapters.excel_file_processor import ExcelFileProcessor +from app_layer.adapters.s3_output_bucket import S3OutputBucket + + +def process_data_input_source_files(log, ssm_client: BaseClient, file_paths: [str]) -> dict[str, Any]: + # Basic validation check + if not file_paths: + log.war('no files to process. exiting...') + return {'status_code': 204, 'message': 'no files to process'} + + # Get file types from all file paths + file_types = [file_path.split('.')[-1].lower() for file_path in file_paths] + + processors = { + 'csv': CsvFileProcessor(), + 'xlsx': ExcelFileProcessor(), + 'xls': ExcelFileProcessor() + } + + # Dictionary to store valid data file processors + valid_file_processors = {} + + # Process each file and check if it is valid + log.deb('processing data input source files...') + for file_path, file_type in zip(file_paths, file_types): + processor = processors[file_type] + valid_file_processors[file_path] = processor if processor.process_file(log, file_path) else None + + # If we don't have 5 data file processors, then return + if len(valid_file_processors) < 5: + log.deb('insufficient number of data files. unable to start etl service') + return {'status_code': 400, 'message': 'insufficient number of data files'} + + if len(valid_file_processors) > 5: + log.war('more than 5 data files found in the input source. unable to start etl service') + return {'status_code': 413, 'message': 'more than 5 oracle processor files found in the input'} + + # We can now send these data file processors to the etl process. + log.deb('all oracle processor files are valid. handover to etl service...') + + + + # When ETL service is done, send the processed content to the output adapter + s3_output_bucket = S3OutputBucket() + + return {'status_code': 200, 'message': 'all oracle processor files processed'} + diff --git a/app_layer/log.py b/app_layer/log.py new file mode 100644 index 000000000..8a9db24fc --- /dev/null +++ b/app_layer/log.py @@ -0,0 +1,337 @@ +# Pixel Code +# Email: haresh@pixelcode.uk +# +# Copyright (c) 2024. +# +# All rights reserved. +# +# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# For permission requests, write to the author, at the email address above. + +""" +log.py is a utility module that provides logging functionality for the runtime. + +The module provides a LogService class that can be used to log messages at different log levels. +The LogService class uses the aws_lambda_powertools library to provide logging functionality. + +The LogLevel enumeration defines the different log levels that can be used in the logger. + +The LogService class provides methods to log messages at different log levels, such as INFO, DEBUG, WARNING, ERROR, and + CRITICAL. +The class also provides a method to set the log level of the logger. + +The LogService class uses the inspect and traceback modules to build log messages with additional information such as + the function name, line number, and exception details. + +The LogService class is designed to be used as a utility class to provide logging functionality in the runtime code. + +Example usage: + + log_service = LogService() + log_service.build("MyService", LogLevel.INFO) + log_service.inf("This is an info message") + log_service.deb("This is a debug message") + log_service.war("This is a warning message") + log_service.err("This is an error message") + log_service.cri("This is a critical message") + log_service.set_log_level(LogLevel.DEBUG) + log_service.deb("This is a debug message with log level DEBUG") +""" + +import inspect +import traceback + +from enum import Enum +from aws_lambda_powertools import Logger +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEvent + +from runtime.chalicelib.exception import EmptyOrNoneValueException + + +class LogLevel(Enum): + """ + LogLevel is an enumeration of the different log levels that can be used in the logger. + """ + DEBUG = 10 + INFO = 20 + WARNING = 30 + ERROR = 40 + CRITICAL = 50 + + +def _log_message_builder(log_level: LogLevel, frame, message: str, exception: Exception = None, + add_trace: bool = False) -> dict: + """ + Builds a log message with the given log level, frame, and message. + + Parameters + ---------- + log_level : LogLevel + The log level of the message (unused but kept for future use). + frame : frame + The frame object of the calling function. + message : str + The message to log. + exception : Exception + The exception to log. + add_trace : bool + Flag to add the traceback to the message. + + Returns + ------- + dict + The log message as a dictionary. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + + # create a dictionary to map, frame.f_code.co_name, frame.f_lineno + + stack_info = { + 'frame': frame.f_code.co_name, + 'line': frame.f_lineno + } + + if message is None or message == "": + message = "" + + stack_info['message'] = message + + + if exception is not None: + stack_info['exception'] = str(exception) + + if add_trace: + stack_info['trace'] = traceback.format_exc() + + # create a json from dictionary + return stack_info + + +class LogService(object): + """ + LogService is a utility class that provides logging functionality. + """ + + def __init__(self): + self._service_name = None + self._logger = None + + def build(self, service_name: str, log_level: LogLevel = LogLevel.INFO) -> 'LogService': + """ + Builds a logger with the given service name and log level. + + Parameters + ---------- + service_name : str + The name of the service. + log_level : LogLevel + The log level of the logger. + + Raises + ------ + EmptyOrNoneValueException + If the service_name argument is empty or None. + """ + + if service_name is None or service_name == "": + raise EmptyOrNoneValueException.append_message("service_name argument cannot be empty (or none)") + + self._logger = Logger(service=service_name, level=log_level.value, log_record_order=["timestamp", "level", "message"]) + return self + + def inject_lambda_context(self, lambda_handler): + """ + Decorator to inject the Lambda context into the logger. + + Parameters + ---------- + lambda_handler : function + The lambda handler function. + + Returns + ------- + function + The decorated lambda handler function. + + Example usage + ------------- + @log_service.inject_lambda_context + def lambda_handler(event, context): + log_service.inf("This is an info message") + return "Hello, World!" + """ + + def decorate(event, context: LambdaContext): + self._logger.structure_logs(append=True, cold_start=self._logger.cold_start, lambda_context=context) + return lambda_handler(event, context) + + return decorate + + def set_correlation_id_from_event(self, event: dict): + """ + Sets the correlation id for the logger based on the event. + + Parameters + ---------- + event : dict + The event dictionary. + """ + if event is None: + self._logger.warn("event argument cannot be None. unable to set correlation id for logger") + return + + request = APIGatewayProxyEvent(event) + self._logger.set_correlation_id(request.request_context.request_id) + + def get_logger(self) -> Logger: + """ + Returns the logger instance. + + Returns + ------- + Logger + The logger instance. + """ + + return self._logger + + def inf(self, message: str): + """ + Logs an INFO message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.INFO, + inspect.currentframe().f_back, + message) + self._logger.info(log_message) + + def deb(self, message: str): + """ + Logs a DEBUG message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.DEBUG, + inspect.currentframe().f_back, + message) + self._logger.debug(log_message) + + def war(self, message: str): + """ + Logs a WARNING message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.WARNING, + inspect.currentframe().f_back, + message) + self._logger.warning(log_message) + + def err(self, message: str): + """ + Logs an ERROR message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.ERROR, + inspect.currentframe().f_back, + message) + self._logger.error(log_message) + + def exc(self, message: str, exception: Exception): + """ + Logs an ERROR message with an exception. + + Parameters + ---------- + message : str + The message to log. + exception : Exception + The exception to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.ERROR, + inspect.currentframe().f_back, + message, + exception) + self._logger.exception(log_message) + + def cri(self, message: str): + """ + Logs a CRITICAL message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.CRITICAL, + inspect.currentframe().f_back, + message) + self._logger.critical(log_message) + + def set_log_level(self, log_level: LogLevel): + """ + Sets the log level of the logger. + + Parameters + ---------- + log_level : LogLevel + The log level to set. + """ + self._logger.setLevel(log_level.__str__()) diff --git a/my_hr/migrations/__init__.py b/app_layer/migrations/__init__.py similarity index 100% rename from my_hr/migrations/__init__.py rename to app_layer/migrations/__init__.py diff --git a/app_layer/models.py b/app_layer/models.py new file mode 100644 index 000000000..71a836239 --- /dev/null +++ b/app_layer/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/app_layer/ports/file_processor.py b/app_layer/ports/file_processor.py new file mode 100644 index 000000000..f3a1f5682 --- /dev/null +++ b/app_layer/ports/file_processor.py @@ -0,0 +1,60 @@ +from abc import ABCMeta, abstractmethod + + +class FileProcessor(metaclass=ABCMeta): + """ + This code defines an abstract base class called FileProcessor. + + It is designed to be subclassed by other classes that will implement the functionality of processing a file and + sending the processed content to an output adapter. + + The class has two abstract methods: + + 1. process_file(self, file_path: str): + - This method is responsible for processing the content of a file located at the given file_path. + - Subclasses must override this method and provide their own implementation. + + 2. send_to_output(self, output_adapter, file_path: str, content: str): + - This method is responsible for sending the processed content to an output adapter. + - It takes an output_adapter object, the file_path of the processed file, and the content that needs to be sent. + - Subclasses must override this method and provide their own implementation. + + This class should not be instantiated directly. Instead, it should be used as a base class for implementing specific + file processing functionality. + """ + @abstractmethod + def process_file(self, log, file_path: str): + """ + This method is an abstract method that should be overridden by a subclass. + + Parameters: + - log: An instance of the LogService class used for logging. + - file_path: The path of the file to be processed. + - file_path: str + + Returns: + None + + This method is an abstract method and must be implemented by subclasses. + """ + pass + + @abstractmethod + def send_to_output(self, log, output_adapter, file_path: str, content: str): + """ + send_to_output(self, log: LogService, output_adapter, file_path: str, content: str) + + Sends the specified content to the output using the provided output adapter. + + Parameters: + - log (LogService): The log service to use for logging events and errors. + - output_adapter: The output adapter that handles the specific output mechanism. + - file_path (str): The path to the file where the content will be sent. + - content (str): The content to send. + + Returns: + None + + This method is an abstract method and must be implemented by subclasses. + """ + pass \ No newline at end of file diff --git a/app_layer/ports/output_service.py b/app_layer/ports/output_service.py new file mode 100644 index 000000000..b08ab5f5b --- /dev/null +++ b/app_layer/ports/output_service.py @@ -0,0 +1,45 @@ +# ports/output_service.py + +from abc import ABCMeta, abstractmethod + +from app_layer.log import LogService + + +class OutputService(metaclass=ABCMeta): + """ + This is the documentation for the `OutputService` class. + + Class: OutputService + -------------------- + This is an abstract base class that defines the interface for sending output. It is meant to be subclassed and + implemented by concrete output service classes. + + Methods + ------- + send(file_path: str, content: str) + This is an abstract method that subclasses must implement. It takes two parameters: + - file_path: A string representing the file path where the output should be sent. + - content: A string representing the content of the output. + + Note: + - This method should be overridden in subclasses with the specific implementation logic for sending the output. + + Exceptions + ---------- + None + + Example usage + ------------- + ``` + class MyOutputService(OutputService): + def send(self, log: LogService, ssm_client: BaseClient, file_path: str, content: str): + # Implement the logic to send the output to the specified file path + pass + + output_service = MyOutputService() + output_service.send('/path/to/file.txt', 'Hello, World!') + ``` + """ + @abstractmethod + def send(self, log: LogService, ssm_client, file_path: str, content: str): + pass diff --git a/app_layer/serializers.py b/app_layer/serializers.py new file mode 100644 index 000000000..a3f7fb0ae --- /dev/null +++ b/app_layer/serializers.py @@ -0,0 +1,7 @@ +# app_layer/serializers.py +from rest_framework import serializers + + +class S3EventSerializer(serializers.Serializer): + bucket = serializers.CharField() + key = serializers.CharField() \ No newline at end of file diff --git a/app_layer/tests.py b/app_layer/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/app_layer/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/app_layer/urls.py b/app_layer/urls.py new file mode 100644 index 000000000..46e35d28c --- /dev/null +++ b/app_layer/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from app_layer.views import s3_event, add_transaction + +urlpatterns = [ + path('api/s3-event/', s3_event, name='s3_event'), + path('api/transactions/', add_transaction, name='add_transaction'), +] \ No newline at end of file diff --git a/app_layer/views.py b/app_layer/views.py new file mode 100644 index 000000000..01cd13736 --- /dev/null +++ b/app_layer/views.py @@ -0,0 +1,57 @@ +import boto3 +from rest_framework import status +from rest_framework.decorators import api_view +from rest_framework.response import Response + +from .layer.service import process_data_input_source_files +from .log import LogService, LogLevel +from .serializers import S3EventSerializer +from transaction.serializers import TransactionSerializer +from django.db import transaction as db_transaction + +app_name = "fat" +log = LogService().build(app_name, LogLevel.DEBUG) + +# SSM client +ssm_client = boto3.client('ssm', region_name='us-west-1') + +@api_view(['POST']) +def s3_event(request): + serializer = S3EventSerializer(data=request.data) + if serializer.is_valid(): + bucket = serializer.validated_data['bucket'] + key = serializer.validated_data['key'] + + # Get all objects from S3 bucket + s3 = boto3.client('s3', region_name='us-west-1') + objects = s3.list_objects_v2(Bucket=bucket) + if not objects.get('Contents'): + log.deb(f'no objects found in bucket: {bucket}. will not proceed') + return + + log.deb(f'objects found in bucket: {bucket}. processing...') + + # Get all file paths + file_paths = [obj['Key'] for obj in objects['Contents']] + + # Process the oracle input source files + result = process_data_input_source_files(log, ssm_client, file_paths) + + return + + +# External API endpoint +# --------------------- +# To allow data workspace to contact fft for data + +@api_view(['GET']) +@db_transaction.atomic +def get_latest(request): + if request.method == 'GET': + # Below is an example of how to use the TransactionSerializer, not the actual implementation + # of the get_latest function. + serializer = TransactionSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) \ No newline at end of file diff --git a/chartofaccountDIT/views.py b/chartofaccountDIT/views.py index a4af2d409..2de42283e 100644 --- a/chartofaccountDIT/views.py +++ b/chartofaccountDIT/views.py @@ -59,7 +59,7 @@ def get_context_data(self, **kwargs): "This field tells us what we are " "spending the money on. " "The structure follows the Treasury Common Chart of " - "Accounts and groups a set of transactions " + "Accounts and groups a set of transaction " "into a clearly defined category." ) return context @@ -211,7 +211,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["section_name"] = self.name context["section_description"] = ( - "This field is used to identify transactions with " + "This field is used to identify transaction with " "Other Government Departments (OGDs) / Bodies which " "is needed for the year-end accounts. To be used " "when setting up Purchase Orders and / or " diff --git a/config/settings/base.py b/config/settings/base.py index 80cc373e1..9cd6545ae 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -75,9 +75,10 @@ "simple_history", "axes", "django_chunk_upload_handlers", - "transactions", - "my_hr", - "payroll" + "myhr", + "payroll", + "transaction", + "app_layer" ] ROOT_URLCONF = "config.urls" diff --git a/transactions/__init__.py b/myhr/__init__.py similarity index 100% rename from transactions/__init__.py rename to myhr/__init__.py diff --git a/myhr/admin.py b/myhr/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/myhr/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/myhr/apps.py b/myhr/apps.py new file mode 100644 index 000000000..d3c8a2f3c --- /dev/null +++ b/myhr/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class MyhrConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'myhr' diff --git a/my_hr/migrations/0001_initial.py b/myhr/migrations/0001_initial.py similarity index 97% rename from my_hr/migrations/0001_initial.py rename to myhr/migrations/0001_initial.py index a017f81da..ea63f59aa 100644 --- a/my_hr/migrations/0001_initial.py +++ b/myhr/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-08-28 20:53 +# Generated by Django 5.1 on 2024-08-29 10:18 from django.db import migrations, models diff --git a/transactions/migrations/__init__.py b/myhr/migrations/__init__.py similarity index 100% rename from transactions/migrations/__init__.py rename to myhr/migrations/__init__.py diff --git a/my_hr/models.py b/myhr/models.py similarity index 96% rename from my_hr/models.py rename to myhr/models.py index 6a2fb6dc8..0b450a27d 100644 --- a/my_hr/models.py +++ b/myhr/models.py @@ -2,6 +2,7 @@ class Employee(models.Model): employee_id = models.CharField(max_length=100, unique=True, primary_key=True) + created_at = models.DateTimeField(auto_now=True) group = models.CharField(max_length=100) directorate = models.CharField(max_length=100) cost_centre = models.CharField(max_length=100) diff --git a/myhr/tests.py b/myhr/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/myhr/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/myhr/views.py b/myhr/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/myhr/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/payroll/admin.py b/payroll/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/payroll/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/payroll/apps.py b/payroll/apps.py new file mode 100644 index 000000000..45203bfb9 --- /dev/null +++ b/payroll/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PayrollConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'payroll' diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py index 3308536d7..aac2e35d7 100644 --- a/payroll/migrations/0001_initial.py +++ b/payroll/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-08-28 20:53 +# Generated by Django 5.1 on 2024-08-29 11:14 from django.db import migrations, models @@ -15,6 +15,7 @@ class Migration(migrations.Migration): name='Payroll', fields=[ ('payroll_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('created_at', models.DateTimeField()), ('business_unit_number', models.CharField(max_length=100)), ('business_unit_name', models.CharField(max_length=100)), ('cost_center_number', models.CharField(max_length=100)), diff --git a/payroll/migrations/0002_alter_payroll_created_at.py b/payroll/migrations/0002_alter_payroll_created_at.py new file mode 100644 index 000000000..cd158a5af --- /dev/null +++ b/payroll/migrations/0002_alter_payroll_created_at.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2024-08-29 11:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('payroll', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='payroll', + name='created_at', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/payroll/models.py b/payroll/models.py index f0dad012a..fd40aedf8 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -3,6 +3,7 @@ class Payroll(models.Model): payroll_id = models.CharField(max_length=100, unique=True, primary_key=True) + created_at = models.DateTimeField(auto_now=True) business_unit_number = models.CharField(max_length=100) business_unit_name = models.CharField(max_length=100) cost_center_number = models.CharField(max_length=100) diff --git a/payroll/tests.py b/payroll/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/payroll/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/payroll/views.py b/payroll/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/payroll/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/requirements.txt b/requirements.txt index 86f60b406..5cc2fa794 100644 --- a/requirements.txt +++ b/requirements.txt @@ -110,3 +110,19 @@ xlwt==1.3.0 ; python_version >= "3.12" and python_version < "4.0" zipp==3.19.2 ; python_version >= "3.12" and python_version < "4.0" zope-event==5.0 ; python_version >= "3.12" and python_version < "4.0" zope-interface==6.4.post2 ; python_version >= "3.12" and python_version < "4.0" + +Django~=4.2.15 +openpyxl~=3.1.5 +boto3~=1.34.144 +botocore~=1.34.162 +django-filter~=24.2 +django-tables2~=2.7.0 +django-simple-history~=3.7.0 +django-guardian~=2.4.0 +sentry-sdk~=2.8.0 +django-environ~=0.11.2 +celery~=5.4.0 +psycogreen~=1.0.2 +djangorestframework~=3.15.2 +mohawk~=1.1.0 +django_chunk_upload_handlers~=0.0.14 \ No newline at end of file diff --git a/transaction/__init__.py b/transaction/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/transaction/admin.py b/transaction/admin.py new file mode 100644 index 000000000..1821c7038 --- /dev/null +++ b/transaction/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from .models import Transactions + +admin.site.register(Transactions) \ No newline at end of file diff --git a/transaction/apps.py b/transaction/apps.py new file mode 100644 index 000000000..26d1c705d --- /dev/null +++ b/transaction/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class TransactionsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'transaction' diff --git a/transactions/migrations/0001_initial.py b/transaction/migrations/0001_initial.py similarity index 93% rename from transactions/migrations/0001_initial.py rename to transaction/migrations/0001_initial.py index eb268a2b5..f2bd540d7 100644 --- a/transactions/migrations/0001_initial.py +++ b/transaction/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-08-28 20:52 +# Generated by Django 5.1 on 2024-08-29 10:22 from django.db import migrations, models @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='Transactions', + name='Transaction', fields=[ ('transaction_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), ('source', models.CharField(max_length=100)), diff --git a/transaction/migrations/__init__.py b/transaction/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/transactions/models.py b/transaction/models.py similarity index 90% rename from transactions/models.py rename to transaction/models.py index f3f76501a..a47c43ea4 100644 --- a/transactions/models.py +++ b/transaction/models.py @@ -1,7 +1,8 @@ from django.db import models -class Transactions(models.Model): +class Transaction(models.Model): transaction_id = models.CharField(max_length=100, unique=True, primary_key=True) + created_at = models.DateTimeField(auto_now=True) source = models.CharField(max_length=100) entity = models.CharField(max_length=100) cost_centre = models.CharField(max_length=100) diff --git a/transaction/serializers.py b/transaction/serializers.py new file mode 100644 index 000000000..fc5d41f4e --- /dev/null +++ b/transaction/serializers.py @@ -0,0 +1,9 @@ +# transaction/serializers.py +from rest_framework import serializers +from .models import Transaction + + +class TransactionSerializer(serializers.ModelSerializer): + class Meta: + model = Transaction + fields = ['transaction_id', 'amount', 'timestamp', 'description'] \ No newline at end of file diff --git a/transaction/tests.py b/transaction/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/transaction/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/transaction/views.py b/transaction/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/transaction/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 5a68ad2049f81fdeb59dbfe01f2a660d64577f25 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Tue, 3 Sep 2024 19:31:11 +0100 Subject: [PATCH 03/23] refactor: HR module and refactor CSV processing Rename `myhr` module to `hr`, update model definitions, and implement CSV parsing methods. Revise URL routing to add `sns_notification` endpoint and update file processing logic. Adjust configurations and settings, ensuring compatibility with new structure and functionalities. --- app_layer/adapters/csv_file_processor.py | 50 +++++++++++++++--- app_layer/layer/service.py | 37 ++++++------- app_layer/urls.py | 6 +-- app_layer/views.py | 64 ++++++++++++++--------- config/settings/base.py | 8 +-- config/settings/local.py | 7 +-- core/models.py | 2 +- {myhr => hr}/__init__.py | 0 {myhr => hr}/admin.py | 0 {myhr => hr}/apps.py | 4 +- hr/migrations/0001_initial.py | 44 ++++++++++++++++ {myhr => hr}/migrations/__init__.py | 0 hr/models.py | 66 ++++++++++++++++++++++++ {myhr => hr}/tests.py | 0 {myhr => hr}/views.py | 0 myhr/migrations/0001_initial.py | 44 ---------------- myhr/models.py | 32 ------------ payroll/models.py | 29 +++++++++++ requirements.txt | 3 +- transaction/admin.py | 4 +- transaction/models.py | 21 ++++++++ 21 files changed, 281 insertions(+), 140 deletions(-) rename {myhr => hr}/__init__.py (100%) rename {myhr => hr}/admin.py (100%) rename {myhr => hr}/apps.py (66%) create mode 100644 hr/migrations/0001_initial.py rename {myhr => hr}/migrations/__init__.py (100%) create mode 100644 hr/models.py rename {myhr => hr}/tests.py (100%) rename {myhr => hr}/views.py (100%) delete mode 100644 myhr/migrations/0001_initial.py delete mode 100644 myhr/models.py diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py index 30253c7dc..b028c91bd 100644 --- a/app_layer/adapters/csv_file_processor.py +++ b/app_layer/adapters/csv_file_processor.py @@ -3,6 +3,10 @@ from app_layer.log import LogService from app_layer.ports.file_processor import FileProcessor +from hr import models as hr_models +from payroll import models as payroll_models +from transaction import models as transaction_models +from zero_transaction import models as zero_transaction_models class CsvFileProcessor(FileProcessor): """ @@ -21,12 +25,46 @@ class CsvFileProcessor(FileProcessor): def process_file(self, log: LogService, file_path: str): log.deb(f'processing csv file: {file_path}...') - is_valid = os.path.basename(file_path.lower()).startswith(('actualsdbt', - 'hrdbt', - 'payrolldbt', - 'transactionsdbt', - 'budget')) - log.deb(f'processing csv file: {file_path} - valid[{is_valid}]') + is_valid = os.path.basename(file_path.lower()).startswith(('actualsauto', + 'budgetauto', + 'hrauto', + 'payrollauto', + 'transactionsauto', + 'zeroauto')) + # Extract file_name from file_path + file_name = os.path.basename(file_path) + + if not is_valid: + log.err(f'invalid csv file: {file_path}. will not process.') + else: + log.deb(f'valid csv file: {file_path}. will process.') + log.deb(f'processing file: {file_name}...') + + # If file_name == hrauto then use parse the file and store in a dictionary using hr model from hr.models.py + if file_name.lower().startswith('hrauto'): + hr_model = hr_models.HR() + hr_model.parse_csv(file_path) + + # If file_name == payrollauto then use parse the file and store in a dictionary using payroll model from hr.models.py + elif file_name.lower().startswith('payrollauto'): + payroll_model = payroll_models.Payroll() + payroll_model.parse_csv(file_path) + + # If file_name == transactionsauto then use parse the file and store in a dictionary using transactions model from transaction.models.py + elif file_name.lower().startswith('transactionsauto'): + transactions_model = transaction_models.Transaction() + transactions_model.parse_csv(file_path) + + # If file_name == zeroauto then use parse the file and store in a dictionary using budget model from zero_transaction.models.py + elif file_name.lower().startswith('zeroauto'): + zero_model = zero_transaction_models.ZeroTransaction() + zero_model.parse_csv(file_path) + + else: + log.deb(f'unknown file: {file_name}. will not continue processing.') + return False + + log.deb(f'processed file: {file_name}') return is_valid def send_to_output(self, log: LogService, output_adapter, file_path: str, content: str): diff --git a/app_layer/layer/service.py b/app_layer/layer/service.py index 53ad563c4..274df2a22 100644 --- a/app_layer/layer/service.py +++ b/app_layer/layer/service.py @@ -1,18 +1,24 @@ from typing import Any -from botocore.client import BaseClient from app_layer.adapters.csv_file_processor import CsvFileProcessor from app_layer.adapters.excel_file_processor import ExcelFileProcessor from app_layer.adapters.s3_output_bucket import S3OutputBucket +from app_layer.log import LogService -def process_data_input_source_files(log, ssm_client: BaseClient, file_paths: [str]) -> dict[str, Any]: +def process_data_input_source_files(log: LogService, file_paths: [str]) -> dict[str, Any]: # Basic validation check if not file_paths: log.war('no files to process. exiting...') return {'status_code': 204, 'message': 'no files to process'} - # Get file types from all file paths - file_types = [file_path.split('.')[-1].lower() for file_path in file_paths] + buckets = {} + for path in file_paths: + bucket_name, file_name = path.split('/', 1) + file_type = file_name.split('.')[-1].lower() + + if bucket_name not in buckets: + buckets[bucket_name] = {} + buckets[bucket_name][file_name] = file_type processors = { 'csv': CsvFileProcessor(), @@ -25,26 +31,21 @@ def process_data_input_source_files(log, ssm_client: BaseClient, file_paths: [st # Process each file and check if it is valid log.deb('processing data input source files...') - for file_path, file_type in zip(file_paths, file_types): - processor = processors[file_type] - valid_file_processors[file_path] = processor if processor.process_file(log, file_path) else None - - # If we don't have 5 data file processors, then return - if len(valid_file_processors) < 5: - log.deb('insufficient number of data files. unable to start etl service') - return {'status_code': 400, 'message': 'insufficient number of data files'} + for bucket_name, files in buckets.items(): + for file_name, file_type in files.items(): + file_path = f"{bucket_name}/{file_name}" + processor = processors.get(file_type) - if len(valid_file_processors) > 5: - log.war('more than 5 data files found in the input source. unable to start etl service') - return {'status_code': 413, 'message': 'more than 5 oracle processor files found in the input'} + if processor: + valid_file_processors[file_path] = processor if processor.process_file(log, file_path) else None + else: + log.err(f'no processor found for file: {file_name}') # We can now send these data file processors to the etl process. log.deb('all oracle processor files are valid. handover to etl service...') - - # When ETL service is done, send the processed content to the output adapter s3_output_bucket = S3OutputBucket() - return {'status_code': 200, 'message': 'all oracle processor files processed'} + return {'status_code': 200, 'message': 'all input processor files processed'} diff --git a/app_layer/urls.py b/app_layer/urls.py index 46e35d28c..85dd1322c 100644 --- a/app_layer/urls.py +++ b/app_layer/urls.py @@ -1,7 +1,7 @@ from django.urls import path -from app_layer.views import s3_event, add_transaction +from app_layer.views import sns_notification urlpatterns = [ - path('api/s3-event/', s3_event, name='s3_event'), - path('api/transactions/', add_transaction, name='add_transaction'), + # SNS notification endpoint + path('sns-notification/', sns_notification, name='sns_notification'), ] \ No newline at end of file diff --git a/app_layer/views.py b/app_layer/views.py index 01cd13736..67ba5b678 100644 --- a/app_layer/views.py +++ b/app_layer/views.py @@ -1,43 +1,57 @@ -import boto3 +import json +import requests +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response - -from .layer.service import process_data_input_source_files -from .log import LogService, LogLevel -from .serializers import S3EventSerializer +from layer.service import process_data_input_source_files +from log import LogService, LogLevel from transaction.serializers import TransactionSerializer from django.db import transaction as db_transaction app_name = "fat" +region_name = 'us-west-1' log = LogService().build(app_name, LogLevel.DEBUG) -# SSM client -ssm_client = boto3.client('ssm', region_name='us-west-1') -@api_view(['POST']) -def s3_event(request): - serializer = S3EventSerializer(data=request.data) - if serializer.is_valid(): - bucket = serializer.validated_data['bucket'] - key = serializer.validated_data['key'] +@csrf_exempt +def sns_notification(request): + if request.method == 'POST': + message = json.loads(request.body.decode('utf-8')) + sns_message_type = request.META.get('HTTP_X_AMZ_SNS_MESSAGE_TYPE') + + if sns_message_type == 'SubscriptionConfirmation': + # Handle subscription confirmation + confirm_url = message['SubscribeURL'] + # Fetch the URL to confirm the subscription + requests.get(confirm_url) + return JsonResponse({'status': 'subscription confirmed'}) + + elif sns_message_type == 'Notification': + # Process the S3 event notification + s3_info = json.loads(message['Message']) - # Get all objects from S3 bucket - s3 = boto3.client('s3', region_name='us-west-1') - objects = s3.list_objects_v2(Bucket=bucket) - if not objects.get('Contents'): - log.deb(f'no objects found in bucket: {bucket}. will not proceed') - return + # Parse the S3 event notification + buckets = {} + for record in s3_info['Records']: + bucket_name = record['s3']['bucket']['name'] + file_key = record['s3']['object']['key'] + file_name = file_key.split('/')[-1] # Get the file name from the key + file_type = file_name.split('.')[-1].lower() # Get the file type - log.deb(f'objects found in bucket: {bucket}. processing...') + # Store the file information in the buckets dictionary + if bucket_name not in buckets: + buckets[bucket_name] = {} + buckets[bucket_name][file_name] = file_type - # Get all file paths - file_paths = [obj['Key'] for obj in objects['Contents']] + # Process the data input source files + result = process_data_input_source_files(log, buckets) - # Process the oracle input source files - result = process_data_input_source_files(log, ssm_client, file_paths) + # Return the result + return JsonResponse(result) - return + return JsonResponse({'status': 'not allowed'}, status=405) # External API endpoint diff --git a/config/settings/base.py b/config/settings/base.py index 9cd6545ae..7afa2c9dd 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -75,10 +75,11 @@ "simple_history", "axes", "django_chunk_upload_handlers", - "myhr", + "hr", "payroll", "transaction", - "app_layer" + "app_layer", + "zero_transaction" ] ROOT_URLCONF = "config.urls" @@ -167,7 +168,8 @@ def FILTERS_VERBOSE_LOOKUPS(): AUTHBROKER_CLIENT_SECRET = env("AUTHBROKER_CLIENT_SECRET", default=None) AUTHBROKER_SCOPES = "read write" -LOGIN_URL = reverse_lazy("authbroker_client:login") +# LOGIN_URL = reverse_lazy("authbroker_client:login") +LOGIN_URL = "/admin" # This is a workaround for the authbroker_client:login not working. Do not commit this change. LOGIN_REDIRECT_URL = "index" GIT_COMMIT = env("GIT_COMMIT", default=None) diff --git a/config/settings/local.py b/config/settings/local.py index 702c7c19d..a094a6f0a 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -4,8 +4,9 @@ CAN_ELEVATE_SSO_USER_PERMISSIONS = True CAN_CREATE_TEST_USER = True -AUTHENTICATION_BACKENDS += [ - "user.backends.CustomAuthbrokerBackend", -] +# Do not commit this file to version control. Use it for local settings only. +# AUTHENTICATION_BACKENDS += [ +# "user.backends.CustomAuthbrokerBackend", +# ] ASYNC_FILE_UPLOAD = False diff --git a/core/models.py b/core/models.py index ee7ce1a75..bf315a136 100644 --- a/core/models.py +++ b/core/models.py @@ -60,7 +60,7 @@ def future_year_dictionary(self): .order_by("financial_year") ) - +# NOTE: This model is used in the upload budgets form class FinancialYear(BaseModel): """Key and representation of the financial year""" diff --git a/myhr/__init__.py b/hr/__init__.py similarity index 100% rename from myhr/__init__.py rename to hr/__init__.py diff --git a/myhr/admin.py b/hr/admin.py similarity index 100% rename from myhr/admin.py rename to hr/admin.py diff --git a/myhr/apps.py b/hr/apps.py similarity index 66% rename from myhr/apps.py rename to hr/apps.py index d3c8a2f3c..339a3b683 100644 --- a/myhr/apps.py +++ b/hr/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class MyhrConfig(AppConfig): +class HRConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'myhr' + name = 'hr' diff --git a/hr/migrations/0001_initial.py b/hr/migrations/0001_initial.py new file mode 100644 index 000000000..5167ab34b --- /dev/null +++ b/hr/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 5.1 on 2024-09-03 17:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='HR', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('group', models.CharField(max_length=255)), + ('directorate', models.CharField(max_length=255)), + ('cc', models.CharField(max_length=255)), + ('cc_name', models.CharField(max_length=255)), + ('last_name', models.CharField(max_length=255)), + ('first_name', models.CharField(max_length=255)), + ('se_no', models.CharField(max_length=50)), + ('salary', models.DecimalField(decimal_places=2, max_digits=10)), + ('grade', models.CharField(max_length=10)), + ('employee_location_city_name', models.CharField(max_length=255)), + ('person_type', models.CharField(max_length=255)), + ('assignment_status', models.CharField(max_length=255)), + ('appointment_status', models.CharField(max_length=255)), + ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), + ('fte', models.DecimalField(decimal_places=2, max_digits=4)), + ('wmi_person', models.CharField(blank=True, max_length=255, null=True)), + ('wmi', models.CharField(blank=True, max_length=255, null=True)), + ('actual_group', models.CharField(max_length=255)), + ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), + ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), + ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ('total', models.DecimalField(decimal_places=2, max_digits=10)), + ('costing_cc', models.CharField(max_length=255)), + ('return_field', models.CharField(max_length=255)), + ], + ), + ] diff --git a/myhr/migrations/__init__.py b/hr/migrations/__init__.py similarity index 100% rename from myhr/migrations/__init__.py rename to hr/migrations/__init__.py diff --git a/hr/models.py b/hr/models.py new file mode 100644 index 000000000..7a9af9823 --- /dev/null +++ b/hr/models.py @@ -0,0 +1,66 @@ +import csv +from django.db import models + + +class HR(models.Model): + group = models.CharField(max_length=255) + directorate = models.CharField(max_length=255) + cc = models.CharField(max_length=255) + cc_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + first_name = models.CharField(max_length=255) + se_no = models.CharField(max_length=50) + salary = models.DecimalField(max_digits=10, decimal_places=2) + grade = models.CharField(max_length=10) + employee_location_city_name = models.CharField(max_length=255) + person_type = models.CharField(max_length=255) + assignment_status = models.CharField(max_length=255) + appointment_status = models.CharField(max_length=255) + working_hours = models.DecimalField(max_digits=5, decimal_places=2) + fte = models.DecimalField(max_digits=4, decimal_places=2) # Full-Time Equivalent + wmi_person = models.CharField(max_length=255, blank=True, null=True) + wmi = models.CharField(max_length=255, blank=True, null=True) + actual_group = models.CharField(max_length=255) + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) # Employer's National Insurance Contribution + total = models.DecimalField(max_digits=10, decimal_places=2) + costing_cc = models.CharField(max_length=255) + return_field = models.CharField(max_length=255) # Assuming 'Return' is a field name; rename if necessary + + def __str__(self): + return f"{self.first_name} {self.last_name} ({self.se_no})" + + def parse_csv(self, file_path): + with open(file_path, mode='r', encoding='utf-8-sig') as file: + reader = csv.DictReader(file) + for row in reader: + HR.objects.create( + group=row['group'], + directorate=row['directorate'], + cc=row['cc'], + cc_name=row['cc_name'], + last_name=row['last_name'], + first_name=row['first_name'], + se_no=row['se_no'], + salary=row['salary'], + grade=row['grade'], + employee_location_city_name=row['employee_location_city_name'], + person_type=row['person_type'], + assignment_status=row['assignment_status'], + appointment_status=row['appointment_status'], + working_hours=row['working_hours'], + fte=row['fte'], + wmi_person=row.get('wmi_person', ''), + wmi=row.get('wmi', ''), + actual_group=row['actual_group'], + basic_pay=row['basic_pay'], + superannuation=row['superannuation'], + ernic=row['ernic'], + total=row['total'], + costing_cc=row['costing_cc'], + return_field=row['return_field'] + ) + class Meta: + verbose_name = "HR" + verbose_name_plural = "HR Records" diff --git a/myhr/tests.py b/hr/tests.py similarity index 100% rename from myhr/tests.py rename to hr/tests.py diff --git a/myhr/views.py b/hr/views.py similarity index 100% rename from myhr/views.py rename to hr/views.py diff --git a/myhr/migrations/0001_initial.py b/myhr/migrations/0001_initial.py deleted file mode 100644 index ea63f59aa..000000000 --- a/myhr/migrations/0001_initial.py +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by Django 5.1 on 2024-08-29 10:18 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Employee', - fields=[ - ('employee_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), - ('group', models.CharField(max_length=100)), - ('directorate', models.CharField(max_length=100)), - ('cost_centre', models.CharField(max_length=100)), - ('cost_centre_name', models.CharField(max_length=100)), - ('last_name', models.CharField(max_length=100)), - ('first_name', models.CharField(max_length=100)), - ('se_no', models.CharField(max_length=100, unique=True)), - ('salary', models.DecimalField(decimal_places=2, max_digits=10)), - ('grade', models.CharField(max_length=100)), - ('employee_location_city_name', models.CharField(max_length=100)), - ('person_type', models.CharField(max_length=100)), - ('assignment_status', models.CharField(max_length=100)), - ('appointment_status', models.CharField(max_length=100)), - ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), - ('fte', models.DecimalField(decimal_places=2, max_digits=4)), - ('wmi_person', models.CharField(max_length=100)), - ('wmi', models.CharField(max_length=100)), - ('actual_group', models.CharField(max_length=100)), - ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), - ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), - ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), - ('total', models.DecimalField(decimal_places=2, max_digits=10)), - ('costing_cc', models.CharField(max_length=100)), - ('return_field', models.CharField(max_length=100)), - ], - ), - ] diff --git a/myhr/models.py b/myhr/models.py deleted file mode 100644 index 0b450a27d..000000000 --- a/myhr/models.py +++ /dev/null @@ -1,32 +0,0 @@ -from django.db import models - -class Employee(models.Model): - employee_id = models.CharField(max_length=100, unique=True, primary_key=True) - created_at = models.DateTimeField(auto_now=True) - group = models.CharField(max_length=100) - directorate = models.CharField(max_length=100) - cost_centre = models.CharField(max_length=100) - cost_centre_name = models.CharField(max_length=100) - last_name = models.CharField(max_length=100) - first_name = models.CharField(max_length=100) - se_no = models.CharField(max_length=100, unique=True) - salary = models.DecimalField(max_digits=10, decimal_places=2) - grade = models.CharField(max_length=100) - employee_location_city_name = models.CharField(max_length=100) - person_type = models.CharField(max_length=100) - assignment_status = models.CharField(max_length=100) - appointment_status = models.CharField(max_length=100) - working_hours = models.DecimalField(max_digits=5, decimal_places=2) - fte = models.DecimalField(max_digits=4, decimal_places=2) - wmi_person = models.CharField(max_length=100) - wmi = models.CharField(max_length=100) - actual_group = models.CharField(max_length=100) - basic_pay = models.DecimalField(max_digits=10, decimal_places=2) - superannuation = models.DecimalField(max_digits=10, decimal_places=2) - ernic = models.DecimalField(max_digits=10, decimal_places=2) - total = models.DecimalField(max_digits=10, decimal_places=2) - costing_cc = models.CharField(max_length=100) - return_field = models.CharField(max_length=100) - - def __str__(self): - return self.employee_id \ No newline at end of file diff --git a/payroll/models.py b/payroll/models.py index fd40aedf8..f8739c629 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -26,5 +26,34 @@ class Payroll(models.Model): debit_amount = models.DecimalField(max_digits=10, decimal_places=2) credit_amount = models.DecimalField(max_digits=10, decimal_places=2) + def parse_csv(self, file_path): + with open(file_path, mode='r', encoding='utf-8-sig') as file: + reader = csv.DictReader(file) + for row in reader: + Payroll.objects.create( + payroll_id=row['payroll_id'], + business_unit_number=row['business_unit_number'], + business_unit_name=row['business_unit_name'], + cost_center_number=row['cost_center_number'], + cost_center_name=row['cost_center_name'], + employee_name=row['employee_name'], + employee_number=row['employee_number'], + assignment_number=row['assignment_number'], + payroll_name=row['payroll_name'], + employee_organization=row['employee_organization'], + employee_location=row['employee_location'], + person_type=row['person_type'], + employee_category=row['employee_category'], + assignment_type=row['assignment_type'], + position=row['position'], + grade=row['grade'], + account_code=row['account_code'], + account_name=row['account_name'], + pay_element_name=row['pay_element_name'], + effective_date=row['effective_date'], + debit_amount=row['debit_amount'], + credit_amount=row['credit_amount'], + ) + def __str__(self): return self.payroll_id diff --git a/requirements.txt b/requirements.txt index 5cc2fa794..ca0f63a16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -125,4 +125,5 @@ celery~=5.4.0 psycogreen~=1.0.2 djangorestframework~=3.15.2 mohawk~=1.1.0 -django_chunk_upload_handlers~=0.0.14 \ No newline at end of file +django_chunk_upload_handlers~=0.0.14 +locust~=2.31.5 \ No newline at end of file diff --git a/transaction/admin.py b/transaction/admin.py index 1821c7038..de4554a2c 100644 --- a/transaction/admin.py +++ b/transaction/admin.py @@ -1,4 +1,4 @@ from django.contrib import admin -from .models import Transactions +from .models import Transaction -admin.site.register(Transactions) \ No newline at end of file +admin.site.register(Transaction) \ No newline at end of file diff --git a/transaction/models.py b/transaction/models.py index a47c43ea4..9cf76db7e 100644 --- a/transaction/models.py +++ b/transaction/models.py @@ -17,5 +17,26 @@ class Transaction(models.Model): supplier_name = models.CharField(max_length=100) level4_code = models.CharField(max_length=100) + def parse_csv(self, file_path): + with open(file_path, mode='r', encoding='utf-8-sig') as file: + reader = csv.DictReader(file) + for row in reader: + Transaction.objects.create( + transaction_id=row['transaction_id'], + source=row['source'], + entity=row['entity'], + cost_centre=row['cost_centre'], + group=row['group'], + account=row['account'], + programme=row['programme'], + line_description=row['line_description'], + net=row['net'], + fiscal_period=row['fiscal_period'], + date_of_journal=row['date_of_journal'], + purchase_order_number=row['purchase_order_number'], + supplier_name=row['supplier_name'], + level4_code=row['level4_code'], + ) + def __str__(self): return self.transaction_id \ No newline at end of file From 4309ecb132e376c702bafbaf10d830c54319bdd2 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Tue, 3 Sep 2024 19:31:54 +0100 Subject: [PATCH 04/23] chore:added ZeroTransaction model and initial migration Introduce the ZeroTransaction model with fields for financial entries and a CSV parser method. Also, add necessary initial Django app configuration and placeholder files for admin, views, and tests. --- zero_transaction/__init__.py | 0 zero_transaction/admin.py | 3 + zero_transaction/apps.py | 6 ++ zero_transaction/migrations/0001_initial.py | 46 ++++++++++++++ zero_transaction/migrations/__init__.py | 0 zero_transaction/models.py | 67 +++++++++++++++++++++ zero_transaction/tests.py | 3 + zero_transaction/views.py | 3 + 8 files changed, 128 insertions(+) create mode 100644 zero_transaction/__init__.py create mode 100644 zero_transaction/admin.py create mode 100644 zero_transaction/apps.py create mode 100644 zero_transaction/migrations/0001_initial.py create mode 100644 zero_transaction/migrations/__init__.py create mode 100644 zero_transaction/models.py create mode 100644 zero_transaction/tests.py create mode 100644 zero_transaction/views.py diff --git a/zero_transaction/__init__.py b/zero_transaction/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zero_transaction/admin.py b/zero_transaction/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/zero_transaction/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/zero_transaction/apps.py b/zero_transaction/apps.py new file mode 100644 index 000000000..7d960518a --- /dev/null +++ b/zero_transaction/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ZeroTransactionsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'zero_transaction' diff --git a/zero_transaction/migrations/0001_initial.py b/zero_transaction/migrations/0001_initial.py new file mode 100644 index 000000000..ea786c996 --- /dev/null +++ b/zero_transaction/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 5.1 on 2024-09-03 15:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ZeroTransaction', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entity', models.CharField(max_length=100)), + ('cost_centre', models.CharField(max_length=100)), + ('account', models.CharField(max_length=100)), + ('programme', models.CharField(max_length=100)), + ('analysis_1', models.CharField(max_length=100)), + ('spare_1', models.CharField(blank=True, max_length=100, null=True)), + ('spare_2', models.CharField(blank=True, max_length=100, null=True)), + ('apr', models.FloatField()), + ('may', models.FloatField()), + ('jun', models.FloatField()), + ('jul', models.FloatField()), + ('aug', models.FloatField()), + ('sep', models.FloatField()), + ('oct', models.FloatField()), + ('nov', models.FloatField()), + ('dec', models.FloatField()), + ('jan', models.FloatField()), + ('feb', models.FloatField()), + ('mar', models.FloatField()), + ('adj1', models.FloatField(blank=True, null=True)), + ('adj2', models.FloatField(blank=True, null=True)), + ('adj3', models.FloatField(blank=True, null=True)), + ('total', models.FloatField()), + ], + options={ + 'verbose_name_plural': 'Zero Transaction Entries', + }, + ), + ] diff --git a/zero_transaction/migrations/__init__.py b/zero_transaction/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/zero_transaction/models.py b/zero_transaction/models.py new file mode 100644 index 000000000..4303b523f --- /dev/null +++ b/zero_transaction/models.py @@ -0,0 +1,67 @@ +from django.db import models + + +class ZeroTransaction(models.Model): + entity = models.CharField(max_length=100) + cost_centre = models.CharField(max_length=100) + account = models.CharField(max_length=100) + programme = models.CharField(max_length=100) + analysis_1 = models.CharField(max_length=100) + spare_1 = models.CharField(max_length=100, blank=True, null=True) + spare_2 = models.CharField(max_length=100, blank=True, null=True) + + apr = models.FloatField() + may = models.FloatField() + jun = models.FloatField() + jul = models.FloatField() + aug = models.FloatField() + sep = models.FloatField() + oct = models.FloatField() + nov = models.FloatField() + dec = models.FloatField() + jan = models.FloatField() + feb = models.FloatField() + mar = models.FloatField() + + adj1 = models.FloatField(blank=True, null=True) + adj2 = models.FloatField(blank=True, null=True) + adj3 = models.FloatField(blank=True, null=True) + + total = models.FloatField() + + def parse_csv(self, file_path): + with open(file_path, mode='r', encoding='utf-8-sig') as file: + reader = csv.DictReader(file) + for row in reader: + ZeroTransactionEntry.objects.create( + entity=row['entity'], + cost_centre=row['cost_centre'], + account=row['account'], + programme=row['programme'], + analysis_1=row['analysis_1'], + spare_1=row.get('spare_1', ''), + spare_2=row.get('spare_2', ''), + apr=row['apr'], + may=row['may'], + jun=row['jun'], + jul=row['jul'], + aug=row['aug'], + sep=row['sep'], + oct=row['oct'], + nov=row['nov'], + dec=row['dec'], + jan=row['jan'], + feb=row['feb'], + mar=row['mar'], + adj1=row.get('adj1', ''), + adj2=row.get('adj2', ''), + adj3=row.get('adj3', ''), + total=row['total'], + ) + + class Meta: + verbose_name_plural = "Zero Transaction Entries" + + def __str__(self): + return f"{self.entity} - {self.cost_centre} - {self.account}" + diff --git a/zero_transaction/tests.py b/zero_transaction/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/zero_transaction/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/zero_transaction/views.py b/zero_transaction/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/zero_transaction/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 8b3f86ce59145a112ab44e663a6154bf4e04801b Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 7 Sep 2024 23:02:07 +0100 Subject: [PATCH 05/23] chore:temp remove app_layer module and its associated files This commit deletes the entire app_layer module, including admin configuration, apps settings, file processors for CSV and Excel, custom exceptions, logging services, models, serializers, service layer, URL routes, and views. The removal streamlines the codebase by eliminating unused or deprecated components. --- app_layer/__init__.py | 0 app_layer/adapters/csv_file_processor.py | 71 ---- app_layer/adapters/excel_file_processor.py | 16 - app_layer/adapters/s3_output_bucket.py | 7 - app_layer/admin.py | 3 - app_layer/apps.py | 6 - app_layer/exception.py | 130 ------- app_layer/layer/service.py | 51 --- app_layer/log.py | 337 ------------------ app_layer/migrations/__init__.py | 0 app_layer/models.py | 3 - app_layer/ports/file_processor.py | 60 ---- app_layer/ports/output_service.py | 45 --- app_layer/serializers.py | 7 - app_layer/tests.py | 3 - app_layer/urls.py | 7 - app_layer/views.py | 71 ---- config/settings/base.py | 2 +- config/urls.py | 1 + core/templates/base_generic.html | 69 ++-- front_end/src/Apps/CostCentrePayroll.jsx | 10 + front_end/src/Apps/Payroll.jsx | 14 + .../CostCentrePayrollList/index.jsx | 79 ++++ .../src/Components/EditPayroll/index.jsx | 248 +++++++++++++ .../src/Components/PayrollTable/index.jsx | 180 ++++++++++ front_end/src/Util.js | 71 +++- front_end/src/index.jsx | 6 + hr/models.py | 25 +- payroll/edit_payroll.py | 63 ++++ payroll/edit_select_cost_centre.py | 100 ++++++ payroll/models.py | 35 +- payroll/serialisers.py | 43 +++ payroll/templates/payroll/edit/edit.html | 42 +++ .../templates/payroll/edit/payroll_base.html | 28 ++ .../payroll/edit/select_cost_centre.html | 26 ++ payroll/templates/payroll/list/list.html | 15 + .../templates/payroll/list/payroll_base.html | 9 + payroll/urls.py | 10 + payroll/views.py | 20 +- requirements.txt | 4 +- transaction/models.py | 23 +- zero_transaction/models.py | 25 +- 42 files changed, 1093 insertions(+), 872 deletions(-) delete mode 100644 app_layer/__init__.py delete mode 100644 app_layer/adapters/csv_file_processor.py delete mode 100644 app_layer/adapters/excel_file_processor.py delete mode 100644 app_layer/adapters/s3_output_bucket.py delete mode 100644 app_layer/admin.py delete mode 100644 app_layer/apps.py delete mode 100644 app_layer/exception.py delete mode 100644 app_layer/layer/service.py delete mode 100644 app_layer/log.py delete mode 100644 app_layer/migrations/__init__.py delete mode 100644 app_layer/models.py delete mode 100644 app_layer/ports/file_processor.py delete mode 100644 app_layer/ports/output_service.py delete mode 100644 app_layer/serializers.py delete mode 100644 app_layer/tests.py delete mode 100644 app_layer/urls.py delete mode 100644 app_layer/views.py create mode 100644 front_end/src/Apps/CostCentrePayroll.jsx create mode 100644 front_end/src/Apps/Payroll.jsx create mode 100644 front_end/src/Components/CostCentrePayrollList/index.jsx create mode 100644 front_end/src/Components/EditPayroll/index.jsx create mode 100644 front_end/src/Components/PayrollTable/index.jsx create mode 100644 payroll/edit_payroll.py create mode 100644 payroll/edit_select_cost_centre.py create mode 100644 payroll/serialisers.py create mode 100644 payroll/templates/payroll/edit/edit.html create mode 100644 payroll/templates/payroll/edit/payroll_base.html create mode 100644 payroll/templates/payroll/edit/select_cost_centre.html create mode 100644 payroll/templates/payroll/list/list.html create mode 100644 payroll/templates/payroll/list/payroll_base.html create mode 100644 payroll/urls.py diff --git a/app_layer/__init__.py b/app_layer/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py deleted file mode 100644 index b028c91bd..000000000 --- a/app_layer/adapters/csv_file_processor.py +++ /dev/null @@ -1,71 +0,0 @@ -import os - -from app_layer.log import LogService -from app_layer.ports.file_processor import FileProcessor - -from hr import models as hr_models -from payroll import models as payroll_models -from transaction import models as transaction_models -from zero_transaction import models as zero_transaction_models - -class CsvFileProcessor(FileProcessor): - """ - Class CsvFileProcessor - - This class is a subclass of FileProcessor and is responsible for processing CSV files and sending the processed content to an output adapter. - - Methods: - - process_file(file_path: str) -> str: - This method takes a file path as input and reads the CSV file. It processes the DataFrame if needed and returns the string representation of the DataFrame. - - - send_to_output(output_adapter, file_path: str, content: str): - This method takes an output adapter, file path, and content as input. It sends the content to the output adapter using the send() method of the adapter. - - """ - - def process_file(self, log: LogService, file_path: str): - log.deb(f'processing csv file: {file_path}...') - is_valid = os.path.basename(file_path.lower()).startswith(('actualsauto', - 'budgetauto', - 'hrauto', - 'payrollauto', - 'transactionsauto', - 'zeroauto')) - # Extract file_name from file_path - file_name = os.path.basename(file_path) - - if not is_valid: - log.err(f'invalid csv file: {file_path}. will not process.') - else: - log.deb(f'valid csv file: {file_path}. will process.') - log.deb(f'processing file: {file_name}...') - - # If file_name == hrauto then use parse the file and store in a dictionary using hr model from hr.models.py - if file_name.lower().startswith('hrauto'): - hr_model = hr_models.HR() - hr_model.parse_csv(file_path) - - # If file_name == payrollauto then use parse the file and store in a dictionary using payroll model from hr.models.py - elif file_name.lower().startswith('payrollauto'): - payroll_model = payroll_models.Payroll() - payroll_model.parse_csv(file_path) - - # If file_name == transactionsauto then use parse the file and store in a dictionary using transactions model from transaction.models.py - elif file_name.lower().startswith('transactionsauto'): - transactions_model = transaction_models.Transaction() - transactions_model.parse_csv(file_path) - - # If file_name == zeroauto then use parse the file and store in a dictionary using budget model from zero_transaction.models.py - elif file_name.lower().startswith('zeroauto'): - zero_model = zero_transaction_models.ZeroTransaction() - zero_model.parse_csv(file_path) - - else: - log.deb(f'unknown file: {file_name}. will not continue processing.') - return False - - log.deb(f'processed file: {file_name}') - return is_valid - - def send_to_output(self, log: LogService, output_adapter, file_path: str, content: str): - output_adapter.send(file_path, content) diff --git a/app_layer/adapters/excel_file_processor.py b/app_layer/adapters/excel_file_processor.py deleted file mode 100644 index 567f8fc1e..000000000 --- a/app_layer/adapters/excel_file_processor.py +++ /dev/null @@ -1,16 +0,0 @@ -# adapters/excel_file_processor.py - -import os - -from app_layer.ports.file_processor import FileProcessor - - -class ExcelFileProcessor(FileProcessor): - def process_file(self, log, file_path: str): - log.deb(f'processing excel file: {file_path}...') - is_valid = os.path.basename(file_path).startswith('ActualsDBT') - log.deb(f'processing excel file: {file_path} - valid[{is_valid}]') - return is_valid - - def send_to_output(self, log, output_adapter, file_path: str, content: str): - output_adapter.send(file_path, content) diff --git a/app_layer/adapters/s3_output_bucket.py b/app_layer/adapters/s3_output_bucket.py deleted file mode 100644 index 2babd093d..000000000 --- a/app_layer/adapters/s3_output_bucket.py +++ /dev/null @@ -1,7 +0,0 @@ -from app_layer.ports.output_service import OutputService - - -class S3OutputBucket(OutputService): - def send(self, log, output_adapter, file_path: str, content: str): - # Email sending logic goes here - print(f'sending email with content: {content} for file: {file_path}') diff --git a/app_layer/admin.py b/app_layer/admin.py deleted file mode 100644 index 8c38f3f3d..000000000 --- a/app_layer/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/app_layer/apps.py b/app_layer/apps.py deleted file mode 100644 index eabf69ab6..000000000 --- a/app_layer/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class AppLayerConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'app_layer' diff --git a/app_layer/exception.py b/app_layer/exception.py deleted file mode 100644 index 8181993e0..000000000 --- a/app_layer/exception.py +++ /dev/null @@ -1,130 +0,0 @@ -# Pixel Code -# Email: haresh@pixelcode.uk -# -# Copyright (c) 2024. -# -# All rights reserved. -# -# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# For permission requests, write to the author, at the email address above. - -""" -exception.py contains custom exception classes for the service. - -The ServiceException class is a custom exception class for the service. - -Attributes ----------- -message (str) - The message of the exception - -Methods ----------- -append_message(message) -> ServiceException - Append a message to the exception -to_dict() -> dict - Convert the exception to a dictionary - -Custom exceptions are defined for: - - Empty or None Value - - Unknown Value - - Value Exists - - Key Not Found -""" -__version__: str = '0.0.1' - - -class ServiceException(Exception): - """ - A custom exception class for service - - Attributes - ---------- - message (str) - The message of the exception - - Methods - ---------- - append_message(message) -> ServiceException - Append a message to the exception - to_dict() -> dict - Convert the exception to a dictionary - - """ - - message = None - - def __init__(self, message: str): - """ - Constructs all the necessary attributes for the ServiceException object. - - Parameters - ---------- - message : str - The message of the exception - """ - self.message = message - super().__init__(self.message) - - @classmethod - def append_message(cls, message) -> 'ServiceException': - """ - Append a message to the exception - - Parameters - ---------- - message : str - The message to append to the exception - - Returns - ---------- - ServiceException - The exception with the appended message - """ - cls.message = message if cls.message is None else cls.message + f" - {message}" - return cls(cls.message) - - def to_dict(self) -> dict: - """ - Convert the exception to a dictionary - - Returns - ---------- - dict - The exception as a dictionary - """ - return { - "message": self.message - } - - -# Custom exception for empty or none value not permitted -class EmptyOrNoneValueException(ServiceException): - def __init__(self, message="empty or none value not permitted"): - super().__init__(message) - - -# Custom exception for unknown and/or not recognised value -class UnknownValueException(ServiceException): - def __init__(self, message="unknown and/or not recognised value"): - super().__init__(message) - - -# Custom exception for value already exists -class ValueExistsException(ServiceException): - def __init__(self, message="value already exists"): - super().__init__(message) - - -# Custom exception for key not found -class KeyNotFoundException(ServiceException): - def __init__(self, message="key not found"): - super().__init__(message) diff --git a/app_layer/layer/service.py b/app_layer/layer/service.py deleted file mode 100644 index 274df2a22..000000000 --- a/app_layer/layer/service.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Any -from app_layer.adapters.csv_file_processor import CsvFileProcessor -from app_layer.adapters.excel_file_processor import ExcelFileProcessor -from app_layer.adapters.s3_output_bucket import S3OutputBucket -from app_layer.log import LogService - - -def process_data_input_source_files(log: LogService, file_paths: [str]) -> dict[str, Any]: - # Basic validation check - if not file_paths: - log.war('no files to process. exiting...') - return {'status_code': 204, 'message': 'no files to process'} - - buckets = {} - for path in file_paths: - bucket_name, file_name = path.split('/', 1) - file_type = file_name.split('.')[-1].lower() - - if bucket_name not in buckets: - buckets[bucket_name] = {} - buckets[bucket_name][file_name] = file_type - - processors = { - 'csv': CsvFileProcessor(), - 'xlsx': ExcelFileProcessor(), - 'xls': ExcelFileProcessor() - } - - # Dictionary to store valid data file processors - valid_file_processors = {} - - # Process each file and check if it is valid - log.deb('processing data input source files...') - for bucket_name, files in buckets.items(): - for file_name, file_type in files.items(): - file_path = f"{bucket_name}/{file_name}" - processor = processors.get(file_type) - - if processor: - valid_file_processors[file_path] = processor if processor.process_file(log, file_path) else None - else: - log.err(f'no processor found for file: {file_name}') - - # We can now send these data file processors to the etl process. - log.deb('all oracle processor files are valid. handover to etl service...') - - # When ETL service is done, send the processed content to the output adapter - s3_output_bucket = S3OutputBucket() - - return {'status_code': 200, 'message': 'all input processor files processed'} - diff --git a/app_layer/log.py b/app_layer/log.py deleted file mode 100644 index 8a9db24fc..000000000 --- a/app_layer/log.py +++ /dev/null @@ -1,337 +0,0 @@ -# Pixel Code -# Email: haresh@pixelcode.uk -# -# Copyright (c) 2024. -# -# All rights reserved. -# -# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# For permission requests, write to the author, at the email address above. - -""" -log.py is a utility module that provides logging functionality for the runtime. - -The module provides a LogService class that can be used to log messages at different log levels. -The LogService class uses the aws_lambda_powertools library to provide logging functionality. - -The LogLevel enumeration defines the different log levels that can be used in the logger. - -The LogService class provides methods to log messages at different log levels, such as INFO, DEBUG, WARNING, ERROR, and - CRITICAL. -The class also provides a method to set the log level of the logger. - -The LogService class uses the inspect and traceback modules to build log messages with additional information such as - the function name, line number, and exception details. - -The LogService class is designed to be used as a utility class to provide logging functionality in the runtime code. - -Example usage: - - log_service = LogService() - log_service.build("MyService", LogLevel.INFO) - log_service.inf("This is an info message") - log_service.deb("This is a debug message") - log_service.war("This is a warning message") - log_service.err("This is an error message") - log_service.cri("This is a critical message") - log_service.set_log_level(LogLevel.DEBUG) - log_service.deb("This is a debug message with log level DEBUG") -""" - -import inspect -import traceback - -from enum import Enum -from aws_lambda_powertools import Logger -from aws_lambda_powertools.utilities.typing import LambdaContext -from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEvent - -from runtime.chalicelib.exception import EmptyOrNoneValueException - - -class LogLevel(Enum): - """ - LogLevel is an enumeration of the different log levels that can be used in the logger. - """ - DEBUG = 10 - INFO = 20 - WARNING = 30 - ERROR = 40 - CRITICAL = 50 - - -def _log_message_builder(log_level: LogLevel, frame, message: str, exception: Exception = None, - add_trace: bool = False) -> dict: - """ - Builds a log message with the given log level, frame, and message. - - Parameters - ---------- - log_level : LogLevel - The log level of the message (unused but kept for future use). - frame : frame - The frame object of the calling function. - message : str - The message to log. - exception : Exception - The exception to log. - add_trace : bool - Flag to add the traceback to the message. - - Returns - ------- - dict - The log message as a dictionary. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - - # create a dictionary to map, frame.f_code.co_name, frame.f_lineno - - stack_info = { - 'frame': frame.f_code.co_name, - 'line': frame.f_lineno - } - - if message is None or message == "": - message = "" - - stack_info['message'] = message - - - if exception is not None: - stack_info['exception'] = str(exception) - - if add_trace: - stack_info['trace'] = traceback.format_exc() - - # create a json from dictionary - return stack_info - - -class LogService(object): - """ - LogService is a utility class that provides logging functionality. - """ - - def __init__(self): - self._service_name = None - self._logger = None - - def build(self, service_name: str, log_level: LogLevel = LogLevel.INFO) -> 'LogService': - """ - Builds a logger with the given service name and log level. - - Parameters - ---------- - service_name : str - The name of the service. - log_level : LogLevel - The log level of the logger. - - Raises - ------ - EmptyOrNoneValueException - If the service_name argument is empty or None. - """ - - if service_name is None or service_name == "": - raise EmptyOrNoneValueException.append_message("service_name argument cannot be empty (or none)") - - self._logger = Logger(service=service_name, level=log_level.value, log_record_order=["timestamp", "level", "message"]) - return self - - def inject_lambda_context(self, lambda_handler): - """ - Decorator to inject the Lambda context into the logger. - - Parameters - ---------- - lambda_handler : function - The lambda handler function. - - Returns - ------- - function - The decorated lambda handler function. - - Example usage - ------------- - @log_service.inject_lambda_context - def lambda_handler(event, context): - log_service.inf("This is an info message") - return "Hello, World!" - """ - - def decorate(event, context: LambdaContext): - self._logger.structure_logs(append=True, cold_start=self._logger.cold_start, lambda_context=context) - return lambda_handler(event, context) - - return decorate - - def set_correlation_id_from_event(self, event: dict): - """ - Sets the correlation id for the logger based on the event. - - Parameters - ---------- - event : dict - The event dictionary. - """ - if event is None: - self._logger.warn("event argument cannot be None. unable to set correlation id for logger") - return - - request = APIGatewayProxyEvent(event) - self._logger.set_correlation_id(request.request_context.request_id) - - def get_logger(self) -> Logger: - """ - Returns the logger instance. - - Returns - ------- - Logger - The logger instance. - """ - - return self._logger - - def inf(self, message: str): - """ - Logs an INFO message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.INFO, - inspect.currentframe().f_back, - message) - self._logger.info(log_message) - - def deb(self, message: str): - """ - Logs a DEBUG message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.DEBUG, - inspect.currentframe().f_back, - message) - self._logger.debug(log_message) - - def war(self, message: str): - """ - Logs a WARNING message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.WARNING, - inspect.currentframe().f_back, - message) - self._logger.warning(log_message) - - def err(self, message: str): - """ - Logs an ERROR message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.ERROR, - inspect.currentframe().f_back, - message) - self._logger.error(log_message) - - def exc(self, message: str, exception: Exception): - """ - Logs an ERROR message with an exception. - - Parameters - ---------- - message : str - The message to log. - exception : Exception - The exception to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.ERROR, - inspect.currentframe().f_back, - message, - exception) - self._logger.exception(log_message) - - def cri(self, message: str): - """ - Logs a CRITICAL message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.CRITICAL, - inspect.currentframe().f_back, - message) - self._logger.critical(log_message) - - def set_log_level(self, log_level: LogLevel): - """ - Sets the log level of the logger. - - Parameters - ---------- - log_level : LogLevel - The log level to set. - """ - self._logger.setLevel(log_level.__str__()) diff --git a/app_layer/migrations/__init__.py b/app_layer/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/app_layer/models.py b/app_layer/models.py deleted file mode 100644 index 71a836239..000000000 --- a/app_layer/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/app_layer/ports/file_processor.py b/app_layer/ports/file_processor.py deleted file mode 100644 index f3a1f5682..000000000 --- a/app_layer/ports/file_processor.py +++ /dev/null @@ -1,60 +0,0 @@ -from abc import ABCMeta, abstractmethod - - -class FileProcessor(metaclass=ABCMeta): - """ - This code defines an abstract base class called FileProcessor. - - It is designed to be subclassed by other classes that will implement the functionality of processing a file and - sending the processed content to an output adapter. - - The class has two abstract methods: - - 1. process_file(self, file_path: str): - - This method is responsible for processing the content of a file located at the given file_path. - - Subclasses must override this method and provide their own implementation. - - 2. send_to_output(self, output_adapter, file_path: str, content: str): - - This method is responsible for sending the processed content to an output adapter. - - It takes an output_adapter object, the file_path of the processed file, and the content that needs to be sent. - - Subclasses must override this method and provide their own implementation. - - This class should not be instantiated directly. Instead, it should be used as a base class for implementing specific - file processing functionality. - """ - @abstractmethod - def process_file(self, log, file_path: str): - """ - This method is an abstract method that should be overridden by a subclass. - - Parameters: - - log: An instance of the LogService class used for logging. - - file_path: The path of the file to be processed. - - file_path: str - - Returns: - None - - This method is an abstract method and must be implemented by subclasses. - """ - pass - - @abstractmethod - def send_to_output(self, log, output_adapter, file_path: str, content: str): - """ - send_to_output(self, log: LogService, output_adapter, file_path: str, content: str) - - Sends the specified content to the output using the provided output adapter. - - Parameters: - - log (LogService): The log service to use for logging events and errors. - - output_adapter: The output adapter that handles the specific output mechanism. - - file_path (str): The path to the file where the content will be sent. - - content (str): The content to send. - - Returns: - None - - This method is an abstract method and must be implemented by subclasses. - """ - pass \ No newline at end of file diff --git a/app_layer/ports/output_service.py b/app_layer/ports/output_service.py deleted file mode 100644 index b08ab5f5b..000000000 --- a/app_layer/ports/output_service.py +++ /dev/null @@ -1,45 +0,0 @@ -# ports/output_service.py - -from abc import ABCMeta, abstractmethod - -from app_layer.log import LogService - - -class OutputService(metaclass=ABCMeta): - """ - This is the documentation for the `OutputService` class. - - Class: OutputService - -------------------- - This is an abstract base class that defines the interface for sending output. It is meant to be subclassed and - implemented by concrete output service classes. - - Methods - ------- - send(file_path: str, content: str) - This is an abstract method that subclasses must implement. It takes two parameters: - - file_path: A string representing the file path where the output should be sent. - - content: A string representing the content of the output. - - Note: - - This method should be overridden in subclasses with the specific implementation logic for sending the output. - - Exceptions - ---------- - None - - Example usage - ------------- - ``` - class MyOutputService(OutputService): - def send(self, log: LogService, ssm_client: BaseClient, file_path: str, content: str): - # Implement the logic to send the output to the specified file path - pass - - output_service = MyOutputService() - output_service.send('/path/to/file.txt', 'Hello, World!') - ``` - """ - @abstractmethod - def send(self, log: LogService, ssm_client, file_path: str, content: str): - pass diff --git a/app_layer/serializers.py b/app_layer/serializers.py deleted file mode 100644 index a3f7fb0ae..000000000 --- a/app_layer/serializers.py +++ /dev/null @@ -1,7 +0,0 @@ -# app_layer/serializers.py -from rest_framework import serializers - - -class S3EventSerializer(serializers.Serializer): - bucket = serializers.CharField() - key = serializers.CharField() \ No newline at end of file diff --git a/app_layer/tests.py b/app_layer/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/app_layer/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/app_layer/urls.py b/app_layer/urls.py deleted file mode 100644 index 85dd1322c..000000000 --- a/app_layer/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.urls import path -from app_layer.views import sns_notification - -urlpatterns = [ - # SNS notification endpoint - path('sns-notification/', sns_notification, name='sns_notification'), -] \ No newline at end of file diff --git a/app_layer/views.py b/app_layer/views.py deleted file mode 100644 index 67ba5b678..000000000 --- a/app_layer/views.py +++ /dev/null @@ -1,71 +0,0 @@ -import json -import requests -from django.http import JsonResponse -from django.views.decorators.csrf import csrf_exempt -from rest_framework import status -from rest_framework.decorators import api_view -from rest_framework.response import Response -from layer.service import process_data_input_source_files -from log import LogService, LogLevel -from transaction.serializers import TransactionSerializer -from django.db import transaction as db_transaction - -app_name = "fat" -region_name = 'us-west-1' -log = LogService().build(app_name, LogLevel.DEBUG) - - -@csrf_exempt -def sns_notification(request): - if request.method == 'POST': - message = json.loads(request.body.decode('utf-8')) - sns_message_type = request.META.get('HTTP_X_AMZ_SNS_MESSAGE_TYPE') - - if sns_message_type == 'SubscriptionConfirmation': - # Handle subscription confirmation - confirm_url = message['SubscribeURL'] - # Fetch the URL to confirm the subscription - requests.get(confirm_url) - return JsonResponse({'status': 'subscription confirmed'}) - - elif sns_message_type == 'Notification': - # Process the S3 event notification - s3_info = json.loads(message['Message']) - - # Parse the S3 event notification - buckets = {} - for record in s3_info['Records']: - bucket_name = record['s3']['bucket']['name'] - file_key = record['s3']['object']['key'] - file_name = file_key.split('/')[-1] # Get the file name from the key - file_type = file_name.split('.')[-1].lower() # Get the file type - - # Store the file information in the buckets dictionary - if bucket_name not in buckets: - buckets[bucket_name] = {} - buckets[bucket_name][file_name] = file_type - - # Process the data input source files - result = process_data_input_source_files(log, buckets) - - # Return the result - return JsonResponse(result) - - return JsonResponse({'status': 'not allowed'}, status=405) - - -# External API endpoint -# --------------------- -# To allow data workspace to contact fft for data - -@api_view(['GET']) -@db_transaction.atomic -def get_latest(request): - if request.method == 'GET': - # Below is an example of how to use the TransactionSerializer, not the actual implementation - # of the get_latest function. - serializer = TransactionSerializer(data=request.data) - if serializer.is_valid(): - serializer.save() - return Response(serializer.data, status=status.HTTP_201_CREATED) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) \ No newline at end of file diff --git a/config/settings/base.py b/config/settings/base.py index 7afa2c9dd..a3fa43521 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -78,7 +78,7 @@ "hr", "payroll", "transaction", - "app_layer", + # "app_layer", "zero_transaction" ] diff --git a/config/urls.py b/config/urls.py index 9b85eacbd..193478e7e 100644 --- a/config/urls.py +++ b/config/urls.py @@ -35,6 +35,7 @@ path("data-lake/", include("data_lake.urls")), path("oscar_return/", include("oscar_return.urls")), path("upload_split_file/", include("upload_split_file.urls")), + path("payroll/", include("payroll.urls")), path("admin/", admin.site.urls), # TODO - split below out into develop only? path( diff --git a/core/templates/base_generic.html b/core/templates/base_generic.html index 61a1b6f0c..e7f7bcd6b 100644 --- a/core/templates/base_generic.html +++ b/core/templates/base_generic.html @@ -1,3 +1,4 @@ + {% load util %} {% load breadcrumbs %} {% load forecast_permissions %} @@ -51,13 +52,13 @@ {% if not settings.DEBUG %} +{% endif %} -{% endif %} + {% vite_dev_client %} + {% vite_js 'src/index.jsx' %} +{% endblock %} \ No newline at end of file diff --git a/payroll/templates/payroll/edit/payroll_base.html b/payroll/templates/payroll/edit/payroll_base.html new file mode 100644 index 000000000..d28b1dc9f --- /dev/null +++ b/payroll/templates/payroll/edit/payroll_base.html @@ -0,0 +1,28 @@ +{% extends "wide_content.html" %} + +{% block content %} +
+

{{ view.title }}

+ +
+
+
+ Group +
+ {{ view.cost_centre_details.group }} ({{ view.cost_centre_details.group_code }}) +
+
+
+ Directorate +
+ + {{ view.cost_centre_details.directorate }} + ({{ view.cost_centre_details.directorate_code }}) + +
+
+ + {% block page_content %} + {% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/payroll/templates/payroll/edit/select_cost_centre.html b/payroll/templates/payroll/edit/select_cost_centre.html new file mode 100644 index 000000000..148575e35 --- /dev/null +++ b/payroll/templates/payroll/edit/select_cost_centre.html @@ -0,0 +1,26 @@ +{% extends "wide_content.html" %} +{% load util vite %} + +{% load breadcrumbs %} + +{% block breadcrumbs %} + {{ block.super }} + {% breadcrumb "Select cost centre" "select_cost_centre" %} +{% endblock %} + +{% block title %}Edit Payroll - Select Cost Centre{% endblock %} + +{% block content %} +

My cost centres

+
+{% endblock %} +{% block scripts %} + + {% vite_dev_client %} + {% vite_js 'src/index.jsx' %} +{% endblock %} diff --git a/payroll/templates/payroll/list/list.html b/payroll/templates/payroll/list/list.html new file mode 100644 index 000000000..f3961a34c --- /dev/null +++ b/payroll/templates/payroll/list/list.html @@ -0,0 +1,15 @@ +{% extends "payroll/list/payroll_base.html" %} +{% load util vite %} +{% load breadcrumbs %} + +{% block breadcrumbs %} + {{ block.super }} + {% breadcrumb "View Payroll" "payroll_list" %} +{% endblock %} + +{% block title %}View Payroll{% endblock %} + +{% block content %} +

View Payroll

+

This is the View Payroll page.

+{% endblock %} \ No newline at end of file diff --git a/payroll/templates/payroll/list/payroll_base.html b/payroll/templates/payroll/list/payroll_base.html new file mode 100644 index 000000000..dc1cda9ad --- /dev/null +++ b/payroll/templates/payroll/list/payroll_base.html @@ -0,0 +1,9 @@ +{% extends "wide_content.html" %} + +{% block content %} +
+

{{ view.title }}

+ {% block page_content %} + {% endblock %} +
+{% endblock %} \ No newline at end of file diff --git a/payroll/urls.py b/payroll/urls.py new file mode 100644 index 000000000..21dbdb0fa --- /dev/null +++ b/payroll/urls.py @@ -0,0 +1,10 @@ +from django.urls import path +from . import views +from .edit_payroll import EditPayrollView +from .edit_select_cost_centre import SelectCostCentreView + +urlpatterns = [ + path('list/', views.payroll_list, name='payroll_list'), + path("edit/select-cost-centre/", SelectCostCentreView.as_view(), name="select_cost_centre"), + path("edit///",EditPayrollView.as_view(), name="edit_payroll"), +] \ No newline at end of file diff --git a/payroll/views.py b/payroll/views.py index 91ea44a21..a7f4007ce 100644 --- a/payroll/views.py +++ b/payroll/views.py @@ -1,3 +1,21 @@ from django.shortcuts import render +from django.views.generic import TemplateView +from .models import Payroll -# Create your views here. +def payroll_list(request): + return render(request, 'payroll/list/list.html') + +def payroll_edit(request): + return render(request, 'payroll/edit/edit.html') + +class EditPayrollView(TemplateView): + template_name = 'payroll/edit/edit.html' + + def get_payroll_data(self): + # Fetch payroll data logic here + return Payroll.objects.all() + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['payroll_data'] = self.get_payroll_data() + return context \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca0f63a16..f7ff50b84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ asgiref==3.8.1 ; python_version >= "3.12" and python_version < "4.0" backoff==2.2.1 ; python_version >= "3.12" and python_version < "4.0" billiard==4.2.0 ; python_version >= "3.12" and python_version < "4.0" boto3==1.34.144 ; python_version >= "3.12" and python_version < "4.0" -botocore==1.34.144 ; python_version >= "3.12" and python_version < "4.0" celery==5.4.0 ; python_version >= "3.12" and python_version < "4.0" certifi==2024.7.4 ; python_version >= "3.12" and python_version < "4" cffi==1.16.0 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_version >= "3.12" and python_version < "4.0" @@ -126,4 +125,5 @@ psycogreen~=1.0.2 djangorestframework~=3.15.2 mohawk~=1.1.0 django_chunk_upload_handlers~=0.0.14 -locust~=2.31.5 \ No newline at end of file +locust~=2.31.5 +requests~=2.32.3 \ No newline at end of file diff --git a/transaction/models.py b/transaction/models.py index 9cf76db7e..29cb96443 100644 --- a/transaction/models.py +++ b/transaction/models.py @@ -1,3 +1,7 @@ +import csv +from io import StringIO + +import boto3 from django.db import models class Transaction(models.Model): @@ -17,9 +21,21 @@ class Transaction(models.Model): supplier_name = models.CharField(max_length=100) level4_code = models.CharField(max_length=100) - def parse_csv(self, file_path): - with open(file_path, mode='r', encoding='utf-8-sig') as file: + def parse_csv(self, bucket_name: str, file_path: str): + try: + # Initialize S3 client + s3 = boto3.client('s3') + + # Get the file from S3 + s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) + + # Read the file content + file_content = s3_object['Body'].read().decode('utf-8-sig') + + # Use StringIO to read the content as a CSV + file = StringIO(file_content) reader = csv.DictReader(file) + for row in reader: Transaction.objects.create( transaction_id=row['transaction_id'], @@ -37,6 +53,9 @@ def parse_csv(self, file_path): supplier_name=row['supplier_name'], level4_code=row['level4_code'], ) + except Exception as e: + # log.exc('an error occurred while parsing the CSV file', e) + raise e def __str__(self): return self.transaction_id \ No newline at end of file diff --git a/zero_transaction/models.py b/zero_transaction/models.py index 4303b523f..7d196e5b7 100644 --- a/zero_transaction/models.py +++ b/zero_transaction/models.py @@ -1,4 +1,8 @@ +import csv +import boto3 +from io import StringIO from django.db import models +# from app_layer.log import LogService class ZeroTransaction(models.Model): @@ -29,11 +33,23 @@ class ZeroTransaction(models.Model): total = models.FloatField() - def parse_csv(self, file_path): - with open(file_path, mode='r', encoding='utf-8-sig') as file: + def parse_csv(self, bucket_name: str, file_path: str): + try: + # Initialize S3 client + s3 = boto3.client('s3') + + # Get the file from S3 + s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) + + # Read the file content + file_content = s3_object['Body'].read().decode('utf-8-sig') + + # Use StringIO to read the content as a CSV + file = StringIO(file_content) reader = csv.DictReader(file) + for row in reader: - ZeroTransactionEntry.objects.create( + ZeroTransaction.objects.create( entity=row['entity'], cost_centre=row['cost_centre'], account=row['account'], @@ -58,6 +74,9 @@ def parse_csv(self, file_path): adj3=row.get('adj3', ''), total=row['total'], ) + except Exception as e: + # log.exc('an error occurred while parsing the CSV file', e) + raise e class Meta: verbose_name_plural = "Zero Transaction Entries" From 5e6b3132e2069a02b4f8880d94f3a216537df3cb Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sun, 8 Sep 2024 10:28:56 +0100 Subject: [PATCH 06/23] chore:add console logging for debug and refactor payroll models Added console.log statements for debugging table_data and payroll_data in respective components. Refactored the payroll models by introducing new models and updating serializers accordingly, while cleaning up unnecessary code in templates and serialization logic. --- .../src/Components/EditForecast/index.jsx | 1 + .../src/Components/EditPayroll/index.jsx | 10 +++- .../src/Components/PayrollTable/index.jsx | 31 ++---------- front_end/src/Util.js | 38 ++++++--------- payroll/edit_payroll.py | 23 +++++---- payroll/models.py | 40 ++++++++++++++++ payroll/serialisers.py | 48 +++++-------------- payroll/templates/payroll/edit/edit.html | 5 -- 8 files changed, 91 insertions(+), 105 deletions(-) diff --git a/front_end/src/Components/EditForecast/index.jsx b/front_end/src/Components/EditForecast/index.jsx index 575b528c5..c7ae38ec1 100644 --- a/front_end/src/Components/EditForecast/index.jsx +++ b/front_end/src/Components/EditForecast/index.jsx @@ -32,6 +32,7 @@ function EditForecast() { const timer = () => { setTimeout(() => { if (window.table_data) { + console.log('Table Data:', window.table_data); let rows = processForecastData(window.table_data) dispatch({ type: SET_CELLS, diff --git a/front_end/src/Components/EditPayroll/index.jsx b/front_end/src/Components/EditPayroll/index.jsx index 10504eaf8..02ce18831 100644 --- a/front_end/src/Components/EditPayroll/index.jsx +++ b/front_end/src/Components/EditPayroll/index.jsx @@ -28,6 +28,15 @@ function EditPayroll() { const [sheetUpdating, setSheetUpdating] = useState(false) + useEffect(() => { + if (window.payroll_data) { + console.log('Payroll Data empty?:', window.payroll_data.empty); + console.log('Payroll Data:', window.payroll_data); + } else { + console.log('Payroll Data is not available'); + } + }, []); + useEffect(() => { const timer = () => { setTimeout(() => { @@ -37,7 +46,6 @@ function EditPayroll() { type: SET_CELLS, cells: rows }) - } else { timer() } diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index 15683cb76..4b2685d03 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -73,18 +73,6 @@ function Table({rowData, sheetUpdating}) { Budget type EU/Non-EU Assignment status - Apr - May - Jun - Jul - Aug - Sep - Oct - Nov - Dec - Jan - Feb - Mar @@ -147,22 +135,11 @@ function Table({rowData, sheetUpdating}) { - {window.period_display.map((value, index) => { - return - })} - - - - - - - - - - - - + {/*{window.period_display.map((value, index) => {*/} + {/* return */} + {/*})}*/} + })} diff --git a/front_end/src/Util.js b/front_end/src/Util.js index c052eae0c..3ede572bd 100644 --- a/front_end/src/Util.js +++ b/front_end/src/Util.js @@ -83,7 +83,7 @@ export async function postData(url = '', data = {}) { export const processPayrollData = (payrollData) => { - let payrollEmployeeCols = [ + let employeePayrollCols = [ "name", "grade", "staff_number", @@ -94,38 +94,28 @@ export const processPayrollData = (payrollData) => { "assignment_status", ] - // Mapping of the data keys from API response to payrollEmployeeCols - const keyMapping = { - "employee_name": "name", - "grade": "grade", - "employee_number": "staff_number", - }; - let rows = []; payrollData.forEach(function (rowData, rowIndex) { - let cells = {}; - let colIndex = 0; - - for (const payrollEmployeeCol of payrollEmployeeCols) { - const apiFieldKey = Object.keys(keyMapping).find(key => keyMapping[key] === payrollEmployeeCol); - if (apiFieldKey) { - cells[payrollEmployeeCol] = { - rowIndex: rowIndex, - colIndex: colIndex, - key: payrollEmployeeCol, - value: rowData[apiFieldKey], - isEditable: false - }; - - colIndex++; + let cells = {} + let colIndex = 0 + + // eslint-disable-next-line + for (const employeePayrollCol of employeePayrollCols) { + cells[employeePayrollCol] = { + rowIndex: rowIndex, + colIndex: colIndex, + key: employeePayrollCol, + value: rowData[employeePayrollCol], + isEditable: false } + + colIndex++ } rows.push(cells); }); - return rows; } diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 094bc02ce..be5d34022 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -10,17 +10,12 @@ from forecast.views.base import ( CostCentrePermissionTest, ) -from payroll.models import Payroll +from payroll.models import Payroll, EmployeePayroll +from payroll.serialisers import PayrollSerializer logger = logging.getLogger(__name__) -class PayrollSerializer(serializers.ModelSerializer): - class Meta: - model = Payroll - fields = ['payroll_id', 'created_at', 'business_unit_number', 'business_unit_name', 'cost_center_number', 'cost_center_name', 'employee_name', 'employee_number', 'assignment_number', 'payroll_name', 'employee_organization', 'employee_location', 'person_type', 'employee_category', 'assignment_type', 'position', 'grade', 'account_code', 'account_name', 'pay_element_name', 'effective_date', 'debit_amount', 'credit_amount'] - - class EditPayrollView( CostCentrePermissionTest, TemplateView, @@ -43,16 +38,20 @@ def cost_centre_details(self): "cost_centre_code": cost_centre.cost_centre_code, } - def get_payroll_data(self): - # Fetch payroll data logic here - payroll_data = list(Payroll.objects.all().values()) - return payroll_data + def get_payroll_serialiser(self): + get_all_employee_data = EmployeePayroll.objects.all() + payroll_serialiser = PayrollSerializer(get_all_employee_data, many=True) + return payroll_serialiser def get_context_data(self, **kwargs): + payroll_serialiser = self.get_payroll_serialiser() + serialiser_data = payroll_serialiser.data + payroll_data = json.dumps(serialiser_data) + self.title = "Edit payroll forecast" context = super().get_context_data(**kwargs) - context['payroll_data'] = self.get_payroll_data() + context['payroll_data'] = payroll_data return context diff --git a/payroll/models.py b/payroll/models.py index eccece69a..cb83198a9 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -5,6 +5,46 @@ # from app_layer.log import LogService + +class EmployeePayroll(models.Model): + name = models.CharField(max_length=100) + grade = models.CharField(max_length=100) + staff_number = models.CharField(max_length=100) + fte = models.DecimalField(max_digits=5, decimal_places=2) + programme_code = models.CharField(max_length=100) + budget_type = models.CharField(max_length=100) + eu_non_eu = models.CharField(max_length=100) + assignment_status = models.CharField(max_length=100) + + class Meta: + abstract = False + +class NonEmployeePayroll(models.Model): + name = models.CharField(max_length=100) + grade = models.CharField(max_length=100) + staff_number = models.CharField(max_length=100) + fte = models.DecimalField(max_digits=5, decimal_places=2) + programme_code = models.CharField(max_length=100) + budget_type = models.CharField(max_length=100) + eu_non_eu = models.CharField(max_length=100) + assignment_status = models.CharField(max_length=100) + person_type = models.CharField(max_length=100) + + class Meta: + abstract = False + +class ForcastPayroll(models.Model): + name = models.CharField(max_length=100) + nac = models.CharField(max_length=100) + nac_description = models.CharField(max_length=100) + project_code = models.CharField(max_length=100) + programme_code = models.CharField(max_length=100) + budget_type = models.CharField(max_length=100) + + class Meta: + abstract = False + + class Payroll(models.Model): payroll_id = models.CharField(max_length=100, unique=True, primary_key=True) created_at = models.DateTimeField(auto_now=True) diff --git a/payroll/serialisers.py b/payroll/serialisers.py index 94eb03412..a95406ca8 100644 --- a/payroll/serialisers.py +++ b/payroll/serialisers.py @@ -1,43 +1,19 @@ -from payroll.models import Payroll from rest_framework import serializers +from payroll.models import EmployeePayroll -class PayrollEmployeeSerializer(serializers.ModelSerializer): + +class PayrollSerializer(serializers.ModelSerializer): class Meta: - model = Payroll + model = EmployeePayroll fields = [ - "business_unit_number", - "business_unit_name", - "cost_center_number", - "cost_center_name", - "employee_name", - "employee_number", - "assignment_number", - "payroll_name", - "employee_organization", - "employee_location", - "person_type", - "employee_category", - "assignment_type", - "position", + "name", "grade", - "account_code", - "account_name", - "pay_element_name", - "effective_date", - "debit_amount", - "credit_amount", - "apr", - "may", - "jun", - "jul", - "aug", - "sep", - "oct", - "nov", - "dec", - "jan", - "feb", - "mar", + "staff_number", + "fte", + "programme_code", + "budget_type", + "eu_non_eu", + "assignment_status", ] - read_only_fields = fields + read_only_fields = fields \ No newline at end of file diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index 941b1444b..12fc1a304 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -30,11 +30,6 @@ {% endblock %} {% block scripts %} {% vite_dev_client %} From 9091cd79a5bc15379ce0fe0f2eb0191ae3cd48d1 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 9 Sep 2024 00:32:22 +0100 Subject: [PATCH 07/23] chore:remove initial payroll migration and update frontend components Deleted the initial payroll migration files to tidy up database setup. Modified frontend components to include monthly payroll data handling, and introduced a new PayrollTableCell component for displaying monthly data in the PayrollTable. Enhanced backend data serializers and models to support these changes. --- .../src/Components/EditPayroll/index.jsx | 8 +- .../src/Components/PayrollTable/index.jsx | 47 ++- .../src/Components/PayrollTableCell/index.jsx | 275 ++++++++++++++++++ front_end/src/Util.js | 26 +- payroll/edit_payroll.py | 30 +- payroll/migrations/0001_initial.py | 42 --- .../0002_alter_payroll_created_at.py | 18 -- payroll/models.py | 28 ++ payroll/serialisers.py | 22 +- payroll/templates/payroll/edit/edit.html | 10 +- payroll/views.py | 14 +- 11 files changed, 402 insertions(+), 118 deletions(-) create mode 100644 front_end/src/Components/PayrollTableCell/index.jsx delete mode 100644 payroll/migrations/0001_initial.py delete mode 100644 payroll/migrations/0002_alter_payroll_created_at.py diff --git a/front_end/src/Components/EditPayroll/index.jsx b/front_end/src/Components/EditPayroll/index.jsx index 02ce18831..624b5244c 100644 --- a/front_end/src/Components/EditPayroll/index.jsx +++ b/front_end/src/Components/EditPayroll/index.jsx @@ -30,11 +30,17 @@ function EditPayroll() { useEffect(() => { if (window.payroll_data) { - console.log('Payroll Data empty?:', window.payroll_data.empty); console.log('Payroll Data:', window.payroll_data); } else { console.log('Payroll Data is not available'); } + + if (window.payroll_monthly_data) { + console.log('Payroll Monthly Data:', window.payroll_monthly_data); + } else { + console.log('Payroll Monthly Data is not available'); + } + }, []); useEffect(() => { diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index 4b2685d03..4ad804560 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -1,7 +1,7 @@ import React, {Fragment, memo } from 'react' import { useSelector, useDispatch } from 'react-redux' import { nanoid } from 'nanoid' -import TableCell from '../../Components/TableCell/index' +import PayrollTableCell from '../../Components/PayrollTableCell/index' import InfoCell from '../../Components/InfoCell/index' import CellValue from '../../Components/CellValue/index' import AggregateValue from '../../Components/AggregateValue/index' @@ -73,6 +73,18 @@ function Table({rowData, sheetUpdating}) { Budget type EU/Non-EU Assignment status + Apr + May + Jun + Jul + Aug + Sep + Oct + Nov + Dec + Jan + Feb + Mar @@ -88,7 +100,7 @@ function Table({rowData, sheetUpdating}) { "cellId": null }) ) - if (selectedRow === rowIndex) { + if (selectedRow === rowIndex) { dispatch( SET_SELECTED_ROW({ selectedRow: null @@ -134,10 +146,35 @@ function Table({rowData, sheetUpdating}) { + {Object.keys(window.payroll_monthly_data).map((dataKey, index) => { + const monthValues = window.payroll_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) - {/*{window.period_display.map((value, index) => {*/} - {/* return */} - {/*})}*/} + return ( + Object.keys(monthValues).map((monthKey) => { + const monthValue = monthValues[monthKey]; // Access the value for each month + console.log('monthValue', monthValue) + console.log('monthKey', monthKey) + return ( + + + {/**/} + + + ); + }) + ); + })} })} diff --git a/front_end/src/Components/PayrollTableCell/index.jsx b/front_end/src/Components/PayrollTableCell/index.jsx new file mode 100644 index 000000000..f6b8c4944 --- /dev/null +++ b/front_end/src/Components/PayrollTableCell/index.jsx @@ -0,0 +1,275 @@ +import React, {Fragment, useState, useEffect, memo } from 'react' +import { useSelector, useDispatch } from 'react-redux' +import { SET_EDITING_CELL } from '../../Reducers/Edit' +import { + postData, + processForecastData, + formatValue +} from '../../Util' +import { SET_ERROR } from '../../Reducers/Error' +import { SET_CELLS } from '../../Reducers/Cells' + +const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) => { + + let editing = false + let isEditable = true + + const checkValue = (val) => { + if (cellId === val) { + editing = true + return false + } else if (editing) { + // Turn off editing + editing = false + return false + } + + return true + } + + let selectChanged = false + + const checkSelectRow = (selectedRow) => { + if (selectedRow === rowIndex) { + selectChanged = true + return false + } else if (selectChanged) { + selectChanged = false + return false + } + + return true + } + + const dispatch = useDispatch(); + + const cells = useSelector(state => state.allCells.cells); + const cell = useSelector(state => state.allCells.cells[rowIndex][cellKey]); + const editCellId = useSelector(state => state.edit.cellId, checkValue); + + const [isUpdating, setIsUpdating] = useState(false) + + const selectedRow = useSelector(state => state.selected.selectedRow, checkSelectRow); + const allSelected = useSelector(state => state.selected.all); + + + + // Check for actual + // if (window.payroll_monthly_data.indexOf(cellKey) > -1) { + // isEditable = false + // } + + const getValue = () => { + return cellValue + } + + const [value, setValue] = useState(getValue()) + + useEffect(() => { + if (cell) { + setValue(cell) + } + }, [cell]); + + const isSelected = () => { + if (allSelected) { + return true + } + + return selectedRow === rowIndex + } + + const wasEdited = () => { + return isEditable; + } + + const getClasses = () => { + let editable = '' + + if (!isEditable) { + editable = ' not-editable' + } + + if (!cell) + return "govuk-table__cell payroll-month-cell figure-cell " + (isSelected() ? 'selected' : '') + editable + + // let negative = '' + + // if (cell.amount < 0) { + // negative = " negative" + // } + + return "govuk-table__cell payroll-month-cell figure-cell " + (wasEdited() ? 'edited ' : '') + (isSelected() ? 'selected' : '') + editable + negative + } + + const setContentState = (value) => { + var re = /^-?\d*\.?\d{0,12}$/; + var isValid = (value.match(re) !== null); + + if (!isValid) { + return + } + setValue(value) + } + + + const updateValue = () => { + return; + let newAmount = value * 100 + + if (newAmount > Number.MAX_SAFE_INTEGER) { + newAmount = Number.MAX_SAFE_INTEGER + } + + if (newAmount < Number.MIN_SAFE_INTEGER) { + newAmount = Number.MIN_SAFE_INTEGER + } + + let intAmount = parseInt(newAmount, 10) + + if (cell && intAmount === cell.amount) { + return + } + + if (!cell && intAmount === 0) { + return + } + + setIsUpdating(true) + + let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value + + let payload = new FormData() + payload.append("natural_account_code", cells[rowIndex]["natural_account_code"].value) + payload.append("programme_code", cells[rowIndex]["programme"].value) + payload.append("project_code", cells[rowIndex]["project_code"].value) + payload.append("analysis1_code", cells[rowIndex]["analysis1_code"].value) + payload.append("analysis2_code", cells[rowIndex]["analysis2_code"].value) + payload.append("csrfmiddlewaretoken", crsfToken) + payload.append("month", cellKey) + payload.append("amount", intAmount) + + postData( + `/forecast/update-forecast/${window.cost_centre}/${window.financial_year}`, + payload + ).then((response) => { + setIsUpdating(false) + if (response.status === 200) { + let rows = processForecastData(response.data) + dispatch({ + type: SET_CELLS, + cells: rows + }) + } else { + dispatch( + SET_ERROR({ + errorMessage: response.data.error + }) + ); + } + }) + } + + const handleBlur = (event) => { + updateValue() + dispatch( + SET_EDITING_CELL({ + "cellId": null + }) + ) + } + + const handleKeyDown = (event) => { + if (event.key === "Tab") { + updateValue() + } + } + + const handleKeyPress = (event) => { + if(event.key === 'Enter') { + updateValue() + event.preventDefault() + } + } + + const getId = () => { + if (!cell) + return + + if (isUpdating) { + return cellId + "_updating" + } + + return cellId + } + + const isCellUpdating = () => { + if (cell && !isEditable) + return false + + if (isUpdating) + return true + + if (sheetUpdating && isSelected()) { + return true + } + + return false + } + + const getCellContent = () => { + if (isCellUpdating()) { + return ( + + UPDATING... + + ) + } else { + if (editCellId === cellId) { + return ( + input && input.focus() } + id={cellId + "_input"} + className="cell-input" + type="text" + value={value} + onChange={e => setContentState(e.target.value)} + onKeyPress={handleKeyPress} + onKeyDown={handleKeyDown} + onBlur={handleBlur} + /> + ) + } else { + return {formatValue(getValue())} + } + } + } + + return ( + + { + if (isSelected()) { + dispatch( + SET_EDITING_CELL({ + "cellId": cellId + }) + ); + } + }} + > + {getCellContent()} + + + ); +} + +const comparisonFn = function(prevProps, nextProps) { + return ( + prevProps.sheetUpdating === nextProps.sheetUpdating + ) +}; + +export default memo(PayrollTableCell, comparisonFn); diff --git a/front_end/src/Util.js b/front_end/src/Util.js index 3ede572bd..f5543f25b 100644 --- a/front_end/src/Util.js +++ b/front_end/src/Util.js @@ -5,21 +5,6 @@ export const getCellId = (key, index) => { return "id_" + key + "_" + index; } -export const months = [ - "apr", - "may", - "jun", - "jul", - "aug", - "sep", - "oct", - "nov", - "dec", - "jan", - "feb", - "mar" -]; - function getCookie(name) { var cookieValue = null; if (document.cookie && document.cookie !== '') { @@ -94,12 +79,13 @@ export const processPayrollData = (payrollData) => { "assignment_status", ] + let rows = []; + let colIndex = 0 + let cells = {} - payrollData.forEach(function (rowData, rowIndex) { - let cells = {} - let colIndex = 0 + payrollData.forEach(function (rowData, rowIndex) { // eslint-disable-next-line for (const employeePayrollCol of employeePayrollCols) { cells[employeePayrollCol] = { @@ -109,13 +95,11 @@ export const processPayrollData = (payrollData) => { value: rowData[employeePayrollCol], isEditable: false } - colIndex++ } - - rows.push(cells); }); + rows.push(cells); return rows; } diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index be5d34022..8f296c7af 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -1,17 +1,14 @@ import json import logging -from django.core.serializers import serialize from django.views.generic.base import TemplateView -from rest_framework import serializers -from rest_framework.response import Response from costcentre.models import CostCentre from forecast.views.base import ( CostCentrePermissionTest, ) -from payroll.models import Payroll, EmployeePayroll -from payroll.serialisers import PayrollSerializer +from payroll.models import EmployeePayroll +from payroll.serialisers import EmployeePayrollSerializer, EmployeeMonthlyPayrollSerializer logger = logging.getLogger(__name__) @@ -38,20 +35,31 @@ def cost_centre_details(self): "cost_centre_code": cost_centre.cost_centre_code, } - def get_payroll_serialiser(self): + def get_employee_payroll_serialiser(self): get_all_employee_data = EmployeePayroll.objects.all() - payroll_serialiser = PayrollSerializer(get_all_employee_data, many=True) + payroll_serialiser = EmployeePayrollSerializer(get_all_employee_data, many=True) return payroll_serialiser + def get_employee_payroll_monthly_serialiser(self): + get_all_employee_data = EmployeePayroll.objects.all() + payroll_monthly_serialiser = EmployeeMonthlyPayrollSerializer(get_all_employee_data, many=True) + return payroll_monthly_serialiser + def get_context_data(self, **kwargs): - payroll_serialiser = self.get_payroll_serialiser() - serialiser_data = payroll_serialiser.data - payroll_data = json.dumps(serialiser_data) + employee_payroll_serialiser = self.get_employee_payroll_serialiser() + employee_payroll_serialiser_data = employee_payroll_serialiser.data + employee_payroll_data = json.dumps(employee_payroll_serialiser_data) + + employee_payroll_monthly_serialiser = self.get_employee_payroll_monthly_serialiser() + employee_payroll_monthly_serialiser_data = employee_payroll_monthly_serialiser.data + employee_payroll_monthly_data = json.dumps(employee_payroll_monthly_serialiser_data) + self.title = "Edit payroll forecast" context = super().get_context_data(**kwargs) - context['payroll_data'] = payroll_data + context['payroll_data'] = employee_payroll_data + context['payroll_monthly_data'] = employee_payroll_monthly_data return context diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py deleted file mode 100644 index aac2e35d7..000000000 --- a/payroll/migrations/0001_initial.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by Django 5.1 on 2024-08-29 11:14 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Payroll', - fields=[ - ('payroll_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), - ('created_at', models.DateTimeField()), - ('business_unit_number', models.CharField(max_length=100)), - ('business_unit_name', models.CharField(max_length=100)), - ('cost_center_number', models.CharField(max_length=100)), - ('cost_center_name', models.CharField(max_length=100)), - ('employee_name', models.CharField(max_length=100)), - ('employee_number', models.CharField(max_length=100)), - ('assignment_number', models.CharField(max_length=100)), - ('payroll_name', models.CharField(max_length=100)), - ('employee_organization', models.CharField(max_length=100)), - ('employee_location', models.CharField(max_length=100)), - ('person_type', models.CharField(max_length=100)), - ('employee_category', models.CharField(max_length=100)), - ('assignment_type', models.CharField(max_length=100)), - ('position', models.CharField(max_length=100)), - ('grade', models.CharField(max_length=100)), - ('account_code', models.CharField(max_length=100)), - ('account_name', models.CharField(max_length=100)), - ('pay_element_name', models.CharField(max_length=100)), - ('effective_date', models.DateField()), - ('debit_amount', models.DecimalField(decimal_places=2, max_digits=10)), - ('credit_amount', models.DecimalField(decimal_places=2, max_digits=10)), - ], - ), - ] diff --git a/payroll/migrations/0002_alter_payroll_created_at.py b/payroll/migrations/0002_alter_payroll_created_at.py deleted file mode 100644 index cd158a5af..000000000 --- a/payroll/migrations/0002_alter_payroll_created_at.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1 on 2024-08-29 11:15 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('payroll', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='payroll', - name='created_at', - field=models.DateTimeField(auto_now=True), - ), - ] diff --git a/payroll/models.py b/payroll/models.py index cb83198a9..f6c32cacc 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -15,6 +15,20 @@ class EmployeePayroll(models.Model): budget_type = models.CharField(max_length=100) eu_non_eu = models.CharField(max_length=100) assignment_status = models.CharField(max_length=100) + current_month = models.CharField(max_length=100) + current_year = models.CharField(max_length=100) + apr = models.IntegerField(editable=True, verbose_name='april') + may = models.IntegerField(editable=True, verbose_name='may') + jun = models.IntegerField(editable=True, verbose_name='june') + jul = models.IntegerField(editable=True, verbose_name='july') + aug = models.IntegerField(editable=True, verbose_name='august') + sep = models.IntegerField(editable=True, verbose_name='september') + oct = models.IntegerField(editable=True, verbose_name='october') + nov = models.IntegerField(editable=True, verbose_name='november') + dec = models.IntegerField(editable=True, verbose_name='december') + jan = models.IntegerField(editable=True, verbose_name='january') + feb = models.IntegerField(editable=True, verbose_name='february') + mar = models.IntegerField(editable=True, verbose_name='march') class Meta: abstract = False @@ -29,6 +43,20 @@ class NonEmployeePayroll(models.Model): eu_non_eu = models.CharField(max_length=100) assignment_status = models.CharField(max_length=100) person_type = models.CharField(max_length=100) + current_month = models.CharField(max_length=100) + current_year = models.CharField(max_length=100) + apr = models.IntegerField(editable=True, verbose_name='april') + may = models.IntegerField(editable=True, verbose_name='may') + jun = models.IntegerField(editable=True, verbose_name='june') + jul = models.IntegerField(editable=True, verbose_name='july') + aug = models.IntegerField(editable=True, verbose_name='august') + sep = models.IntegerField(editable=True, verbose_name='september') + oct = models.IntegerField(editable=True, verbose_name='october') + nov = models.IntegerField(editable=True, verbose_name='november') + dec = models.IntegerField(editable=True, verbose_name='december') + jan = models.IntegerField(editable=True, verbose_name='january') + feb = models.IntegerField(editable=True, verbose_name='february') + mar = models.IntegerField(editable=True, verbose_name='march') class Meta: abstract = False diff --git a/payroll/serialisers.py b/payroll/serialisers.py index a95406ca8..281a1e0c4 100644 --- a/payroll/serialisers.py +++ b/payroll/serialisers.py @@ -3,7 +3,27 @@ from payroll.models import EmployeePayroll -class PayrollSerializer(serializers.ModelSerializer): +class EmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): + class Meta: + model = EmployeePayroll + fields = [ + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", + "jan", + "feb", + "mar", + ] + read_only_fields = fields + + +class EmployeePayrollSerializer(serializers.ModelSerializer): class Meta: model = EmployeePayroll fields = [ diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index 12fc1a304..9e62209e4 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -17,20 +17,18 @@
+ +
{% endblock %} {% block scripts %} {% vite_dev_client %} {% vite_js 'src/index.jsx' %} diff --git a/payroll/views.py b/payroll/views.py index a7f4007ce..13c0a8c27 100644 --- a/payroll/views.py +++ b/payroll/views.py @@ -6,16 +6,4 @@ def payroll_list(request): return render(request, 'payroll/list/list.html') def payroll_edit(request): - return render(request, 'payroll/edit/edit.html') - -class EditPayrollView(TemplateView): - template_name = 'payroll/edit/edit.html' - - def get_payroll_data(self): - # Fetch payroll data logic here - return Payroll.objects.all() - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context['payroll_data'] = self.get_payroll_data() - return context \ No newline at end of file + return render(request, 'payroll/edit/edit.html') \ No newline at end of file From ba212a7e024c8f23820048dab99c7e269b5fb907 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 9 Sep 2024 00:32:42 +0100 Subject: [PATCH 08/23] chore:add initial payroll migration This commit introduces the initial migration for the payroll app, creating four models: EmployeePayroll, ForcastPayroll, NonEmployeePayroll, and Payroll. These models define the structure and fields for handling payroll data in the application. --- payroll/migrations/0001_initial.py | 123 +++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 payroll/migrations/0001_initial.py diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py new file mode 100644 index 000000000..d6cebb34e --- /dev/null +++ b/payroll/migrations/0001_initial.py @@ -0,0 +1,123 @@ +# Generated by Django 5.1 on 2024-09-08 17:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='EmployeePayroll', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('staff_number', models.CharField(max_length=100)), + ('fte', models.DecimalField(decimal_places=2, max_digits=5)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ('eu_non_eu', models.CharField(max_length=100)), + ('assignment_status', models.CharField(max_length=100)), + ('current_month', models.CharField(max_length=100)), + ('current_year', models.CharField(max_length=100)), + ('apr', models.IntegerField(verbose_name='april')), + ('may', models.IntegerField(verbose_name='may')), + ('jun', models.IntegerField(verbose_name='june')), + ('jul', models.IntegerField(verbose_name='july')), + ('aug', models.IntegerField(verbose_name='august')), + ('sep', models.IntegerField(verbose_name='september')), + ('oct', models.IntegerField(verbose_name='october')), + ('nov', models.IntegerField(verbose_name='november')), + ('dec', models.IntegerField(verbose_name='december')), + ('jan', models.IntegerField(verbose_name='january')), + ('feb', models.IntegerField(verbose_name='february')), + ('mar', models.IntegerField(verbose_name='march')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ForcastPayroll', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('nac', models.CharField(max_length=100)), + ('nac_description', models.CharField(max_length=100)), + ('project_code', models.CharField(max_length=100)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='NonEmployeePayroll', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('staff_number', models.CharField(max_length=100)), + ('fte', models.DecimalField(decimal_places=2, max_digits=5)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ('eu_non_eu', models.CharField(max_length=100)), + ('assignment_status', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('current_month', models.CharField(max_length=100)), + ('current_year', models.CharField(max_length=100)), + ('apr', models.IntegerField(verbose_name='april')), + ('may', models.IntegerField(verbose_name='may')), + ('jun', models.IntegerField(verbose_name='june')), + ('jul', models.IntegerField(verbose_name='july')), + ('aug', models.IntegerField(verbose_name='august')), + ('sep', models.IntegerField(verbose_name='september')), + ('oct', models.IntegerField(verbose_name='october')), + ('nov', models.IntegerField(verbose_name='november')), + ('dec', models.IntegerField(verbose_name='december')), + ('jan', models.IntegerField(verbose_name='january')), + ('feb', models.IntegerField(verbose_name='february')), + ('mar', models.IntegerField(verbose_name='march')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Payroll', + fields=[ + ('payroll_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('created_at', models.DateTimeField(auto_now=True)), + ('business_unit_number', models.CharField(max_length=100)), + ('business_unit_name', models.CharField(max_length=100)), + ('cost_center_number', models.CharField(max_length=100)), + ('cost_center_name', models.CharField(max_length=100)), + ('employee_name', models.CharField(max_length=100)), + ('employee_number', models.CharField(max_length=100)), + ('assignment_number', models.CharField(max_length=100)), + ('payroll_name', models.CharField(max_length=100)), + ('employee_organization', models.CharField(max_length=100)), + ('employee_location', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('employee_category', models.CharField(max_length=100)), + ('assignment_type', models.CharField(max_length=100)), + ('position', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('account_code', models.CharField(max_length=100)), + ('account_name', models.CharField(max_length=100)), + ('pay_element_name', models.CharField(max_length=100)), + ('effective_date', models.DateField()), + ('debit_amount', models.DecimalField(decimal_places=2, max_digits=10)), + ('credit_amount', models.DecimalField(decimal_places=2, max_digits=10)), + ], + options={ + 'abstract': False, + }, + ), + ] From 351390d41e67bd3d761fa02bb649fa3a5516a9b2 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 9 Sep 2024 01:50:11 +0100 Subject: [PATCH 09/23] refactor:rename Payroll components to Employee-specific names Updated various Payroll components and variables to be specific to employees. This includes renaming files, functions, and properties to reflect the new naming convention and improve code clarity. Additionally, removed unused imports and redundant code. --- .../Apps/{Payroll.jsx => PayrollEmployee.jsx} | 8 +++---- .../index.jsx | 16 ++++++------- .../src/Components/PayrollTable/index.jsx | 19 ++------------- .../src/Components/PayrollTableCell/index.jsx | 23 ++++++++++--------- front_end/src/index.jsx | 6 ++--- payroll/edit_payroll.py | 4 ++-- payroll/templates/payroll/edit/edit.html | 13 ++++------- payroll/urls.py | 1 + 8 files changed, 37 insertions(+), 53 deletions(-) rename front_end/src/Apps/{Payroll.jsx => PayrollEmployee.jsx} (50%) rename front_end/src/Components/{EditPayroll => EditPayrollEmployee}/index.jsx (96%) diff --git a/front_end/src/Apps/Payroll.jsx b/front_end/src/Apps/PayrollEmployee.jsx similarity index 50% rename from front_end/src/Apps/Payroll.jsx rename to front_end/src/Apps/PayrollEmployee.jsx index 778db8ed3..c1464f990 100644 --- a/front_end/src/Apps/Payroll.jsx +++ b/front_end/src/Apps/PayrollEmployee.jsx @@ -1,14 +1,14 @@ import React from 'react'; import { Provider } from 'react-redux'; import { store } from './../Store'; -import EditPayroll from './../Components/EditPayroll/index' +import EditPayrollEmployee from './../Components/EditPayrollEmployee/index' -function Payroll() { +function PayrollEmployee() { return ( - + ); } -export default Payroll; +export default PayrollEmployee; diff --git a/front_end/src/Components/EditPayroll/index.jsx b/front_end/src/Components/EditPayrollEmployee/index.jsx similarity index 96% rename from front_end/src/Components/EditPayroll/index.jsx rename to front_end/src/Components/EditPayrollEmployee/index.jsx index 624b5244c..106e96b93 100644 --- a/front_end/src/Components/EditPayroll/index.jsx +++ b/front_end/src/Components/EditPayrollEmployee/index.jsx @@ -15,7 +15,7 @@ import { } from '../../Util' -function EditPayroll() { +function EditPayrollEmployee() { console.log("EditPayroll component has been rendered"); const dispatch = useDispatch(); @@ -29,14 +29,14 @@ function EditPayroll() { const [sheetUpdating, setSheetUpdating] = useState(false) useEffect(() => { - if (window.payroll_data) { - console.log('Payroll Data:', window.payroll_data); + if (window.payroll_employee_data) { + console.log('Payroll Data:', window.payroll_employee_data); } else { console.log('Payroll Data is not available'); } - if (window.payroll_monthly_data) { - console.log('Payroll Monthly Data:', window.payroll_monthly_data); + if (window.payroll_employee_monthly_data) { + console.log('Payroll Monthly Data:', window.payroll_employee_monthly_data); } else { console.log('Payroll Monthly Data is not available'); } @@ -46,8 +46,8 @@ function EditPayroll() { useEffect(() => { const timer = () => { setTimeout(() => { - if (window.payroll_data) { - let rows = processPayrollData(window.payroll_data) + if (window.payroll_employee_data) { + let rows = processPayrollData(window.payroll_employee_data) dispatch({ type: SET_CELLS, cells: rows @@ -259,4 +259,4 @@ function EditPayroll() { ); } -export default EditPayroll +export default EditPayrollEmployee diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index 4ad804560..bb943ba78 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -4,16 +4,8 @@ import { nanoid } from 'nanoid' import PayrollTableCell from '../../Components/PayrollTableCell/index' import InfoCell from '../../Components/InfoCell/index' import CellValue from '../../Components/CellValue/index' -import AggregateValue from '../../Components/AggregateValue/index' -import VariancePercentage from '../../Components/VariancePercentage/index' import TableHeader from '../../Components/TableHeader/index' -import TotalCol from '../../Components/TotalCol/index' import ToggleCell from '../../Components/ToggleCell/index' -import TotalAggregate from '../../Components/TotalAggregate/index' -import TotalBudget from '../../Components/TotalBudget/index' -import OverspendUnderspend from '../../Components/OverspendUnderspend/index' -import TotalOverspendUnderspend from '../../Components/TotalOverspendUnderspend/index' -import TotalVariancePercentage from '../../Components/TotalVariancePercentage/index' import ActualsHeaderRow from '../../Components/ActualsHeaderRow/index' import { getCellId @@ -146,8 +138,8 @@ function Table({rowData, sheetUpdating}) { - {Object.keys(window.payroll_monthly_data).map((dataKey, index) => { - const monthValues = window.payroll_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) + {Object.keys(window.payroll_employee_monthly_data).map((dataKey, index) => { + const monthValues = window.payroll_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) return ( Object.keys(monthValues).map((monthKey) => { @@ -163,13 +155,6 @@ function Table({rowData, sheetUpdating}) { cellKey={monthKey} // Pass the monthKey (e.g., "apr") cellValue={monthValue} > - - {/**/} - ); }) diff --git a/front_end/src/Components/PayrollTableCell/index.jsx b/front_end/src/Components/PayrollTableCell/index.jsx index f6b8c4944..4e810c1c3 100644 --- a/front_end/src/Components/PayrollTableCell/index.jsx +++ b/front_end/src/Components/PayrollTableCell/index.jsx @@ -114,24 +114,25 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) const updateValue = () => { + // setValue(value) + console.log('cell value:', value) + console.log('cell key:', cellKey) + + return; - let newAmount = value * 100 + let newValue = 0 - if (newAmount > Number.MAX_SAFE_INTEGER) { - newAmount = Number.MAX_SAFE_INTEGER + if (value > 1) { + newValue = 1 } - if (newAmount < Number.MIN_SAFE_INTEGER) { - newAmount = Number.MIN_SAFE_INTEGER + if (value < 0) { + newValue = 0 } - let intAmount = parseInt(newAmount, 10) - - if (cell && intAmount === cell.amount) { - return - } + let intNewValue = parseInt(newValue, 10) - if (!cell && intAmount === 0) { + if (getValue() === intNewValue) { return } diff --git a/front_end/src/index.jsx b/front_end/src/index.jsx index e6ed53dc7..6128e4c39 100644 --- a/front_end/src/index.jsx +++ b/front_end/src/index.jsx @@ -2,15 +2,15 @@ import 'vite/modulepreload-polyfill' import React from 'react' import ReactDOM from 'react-dom' import Forecast from './Apps/Forecast' -import Payroll from './Apps/Payroll.jsx' import CostCentre from './Apps/CostCentre' import * as serviceWorker from './serviceWorker' import CostCentrePayroll from "./Apps/CostCentrePayroll.jsx"; +import PayrollEmployee from "./Apps/PayrollEmployee.jsx"; if (document.getElementById('forecast-app')) { ReactDOM.render(, document.getElementById('forecast-app')) -} else if (document.getElementById('payroll-app')) { - ReactDOM.render(, document.getElementById('payroll-app')) +} else if (document.getElementById('payroll-employee-app')) { + ReactDOM.render(, document.getElementById('payroll-employee-app')) } else if (document.getElementById('cost-centre-list-app')) { ReactDOM.render(, document.getElementById('cost-centre-list-app')) } else if (document.getElementById('cost-centre-payroll-list-app')) { diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 8f296c7af..1544f96ee 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -58,8 +58,8 @@ def get_context_data(self, **kwargs): self.title = "Edit payroll forecast" context = super().get_context_data(**kwargs) - context['payroll_data'] = employee_payroll_data - context['payroll_monthly_data'] = employee_payroll_monthly_data + context['payroll_employee_data'] = employee_payroll_data + context['payroll_employee_monthly_data'] = employee_payroll_monthly_data return context diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index 9e62209e4..4f0289778 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -12,23 +12,20 @@ {% block page_content %}
-
-
- +
- - -
+ +
{% endblock %} {% block scripts %} {% vite_dev_client %} {% vite_js 'src/index.jsx' %} diff --git a/payroll/urls.py b/payroll/urls.py index 21dbdb0fa..e01ec9a33 100644 --- a/payroll/urls.py +++ b/payroll/urls.py @@ -7,4 +7,5 @@ path('list/', views.payroll_list, name='payroll_list'), path("edit/select-cost-centre/", SelectCostCentreView.as_view(), name="select_cost_centre"), path("edit///",EditPayrollView.as_view(), name="edit_payroll"), + ] \ No newline at end of file From 12efce96e411d457384f0097bdc510d23652abc9 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 9 Sep 2024 02:29:52 +0100 Subject: [PATCH 10/23] chore:add Non-Employee Payroll Management Introduce a new component for managing non-employee payroll data. This update includes creating new components for editing and displaying non-employee payroll information, updating the context in the backend, and adjusting the frontend React entry points and templates. --- front_end/src/Apps/PayrollNonEmployee.jsx | 14 + .../EditPayrollNonEmployee/index.jsx | 262 ++++++++++++++++++ .../src/Components/PayrollNonTable/index.jsx | 177 ++++++++++++ .../src/Components/PayrollTable/index.jsx | 2 - front_end/src/index.jsx | 11 +- payroll/edit_payroll.py | 3 + payroll/templates/payroll/edit/edit.html | 16 +- 7 files changed, 478 insertions(+), 7 deletions(-) create mode 100644 front_end/src/Apps/PayrollNonEmployee.jsx create mode 100644 front_end/src/Components/EditPayrollNonEmployee/index.jsx create mode 100644 front_end/src/Components/PayrollNonTable/index.jsx diff --git a/front_end/src/Apps/PayrollNonEmployee.jsx b/front_end/src/Apps/PayrollNonEmployee.jsx new file mode 100644 index 000000000..1c74fa3bf --- /dev/null +++ b/front_end/src/Apps/PayrollNonEmployee.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { Provider } from 'react-redux'; +import { store } from './../Store'; +import EditPayrollNonEmployee from './../Components/EditPayrollNonEmployee/index' + +function PayrollNonEmployee() { + return ( + + + + ); +} + +export default PayrollNonEmployee; diff --git a/front_end/src/Components/EditPayrollNonEmployee/index.jsx b/front_end/src/Components/EditPayrollNonEmployee/index.jsx new file mode 100644 index 000000000..1e2aef64e --- /dev/null +++ b/front_end/src/Components/EditPayrollNonEmployee/index.jsx @@ -0,0 +1,262 @@ +import React, {Fragment, useEffect, useState } from 'react' +import { useSelector, useDispatch } from 'react-redux' +import Table from '../../Components/PayrollNonTable/index' +import { SET_EDITING_CELL } from '../../Reducers/Edit' +import { store } from '../../Store'; +import EditActionBar from '../../Components/EditActionBar/index' + +import { SET_ERROR } from '../../Reducers/Error' +import { SET_CELLS } from '../../Reducers/Cells' +import { OPEN_FILTER_IF_CLOSED } from '../../Reducers/Filter' +import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' +import { + getCellId, + processPayrollData, +} from '../../Util' + + +function EditPayrollNonEmployee() { + console.log("EditPayrollNon component has been rendered"); + const dispatch = useDispatch(); + + const errorMessage = useSelector(state => state.error.errorMessage) + const selectedRow = useSelector(state => state.selected.selectedRow) + const allSelected = useSelector(state => state.selected.all) + + const cells = useSelector(state => state.allCells.cells); + const editCellId = useSelector(state => state.edit.cellId); + + const [sheetUpdating, setSheetUpdating] = useState(false) + + useEffect(() => { + if (window.payroll_non_employee_data) { + console.log('Payroll Non Data:', window.payroll_non_employee_data); + } else { + console.log('Payroll Non Data is not available'); + } + + if (window.payroll_non_employee_monthly_data) { + console.log('Payroll Non Monthly Data:', window.payroll_non_employee_monthly_data); + } else { + console.log('Payroll Non Monthly Data is not available'); + } + + }, []); + + useEffect(() => { + const timer = () => { + setTimeout(() => { + if (window.payroll_non_employee_data) { + let rows = processPayrollData(window.payroll_non_employee_data) + dispatch({ + type: SET_CELLS, + cells: rows + }) + } else { + timer() + } + }, 100); + } + + timer() + }, [dispatch]) + + useEffect(() => { + const capturePaste = (event) => { + if (!event) + return + + if (selectedRow < 0 && !allSelected) { + return + } + + dispatch( + SET_ERROR({ + errorMessage: null + }) + ); + + let clipBoardContent = event.clipboardData.getData('text/plain') + let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value + + let payload = new FormData() + payload.append("paste_content", clipBoardContent) + payload.append("csrfmiddlewaretoken", crsfToken) + + if (allSelected) { + payload.append("all_selected", allSelected) + } else { + if (selectedRow > -1) { + payload.append("pasted_at_row", JSON.stringify(cells[selectedRow])) + } + } + + setSheetUpdating(true) + + // postData( + // `/forecast/paste-forecast/${window.cost_centre}/${window.financial_year}`, + // payload + // ).then((response) => { + // if (response.status === 200) { + // setSheetUpdating(false) + // let rows = processForecastData(response.data) + // dispatch({ + // type: SET_CELLS, + // cells: rows + // }) + // } else { + // setSheetUpdating(false) + // dispatch( + // SET_ERROR({ + // errorMessage: response.data.error + // }) + // ); + // } + // }) + } + + capturePaste() + document.addEventListener("paste", capturePaste) + + return () => { + document.removeEventListener("paste", capturePaste) + }; + }, [dispatch, cells, selectedRow, allSelected]); + + useEffect(() => { + const handleKeyDown = (event) => { + // This function puts editing cells into the tab order of the page + let lowestMonth = 0 + let body = document.getElementsByTagName("BODY")[0] + let skipLink = document.getElementsByClassName("govuk-skip-link")[0] + let filterOpenLink = document.getElementById("action-bar-switch") + let selectAll = document.getElementById("select_all") + + const state = store.getState(); + + if(event.key === 'Enter') { + if (document.activeElement.className === "link-button govuk-link") { + if (allSelected) { + dispatch( + UNSELECT_ALL() + ) + } else { + dispatch( + SELECT_ALL() + ) + } + + event.preventDefault() + } else if (document.activeElement.className === "select_row_btn govuk-link link-button") { + let parts = document.activeElement.id.split("_") + let targetRow = parseInt(parts[2], 10) + + if (selectedRow === targetRow) { + dispatch( + SET_SELECTED_ROW({ + selectedRow: -1 + }) + ) + } else { + dispatch( + SET_SELECTED_ROW({ + selectedRow: targetRow + }) + ) + } + } + } + + if (event.key === "Tab") { + // See if users has hit open filter link + if (document.activeElement === filterOpenLink) { + dispatch( + OPEN_FILTER_IF_CLOSED() + ); + return + } + // See if we need to open filter because of a backwards tab from select all + if (event.shiftKey && document.activeElement === selectAll) { + dispatch( + OPEN_FILTER_IF_CLOSED() + ); + return + } + + let targetRow = -1 + let nextId = null + + // Check for select button + if (editCellId) { + let parts = state.edit.cellId.split("_") + targetRow = parseInt(parts[1], 10) + } else if (document.activeElement.className === "select_row_btn govuk-link link-button") { + let parts = document.activeElement.id.split("_") + targetRow = parseInt(parts[2], 10) + } + + if (event.shiftKey && + editCellId === null && ( + document.activeElement === body || + document.activeElement === skipLink + )) { + targetRow = cells.length - 1 + + nextId = getCellId(targetRow, maxMonth) + + event.preventDefault() + document.activeElement.blur(); + + dispatch( + SET_EDITING_CELL({ + "cellId": nextId + }) + ); + } + } + } + + const handleMouseDown = (event) => { + let active = document.activeElement + + if (active.tagName !== "INPUT") { + dispatch( + SET_EDITING_CELL({ + "cellId": null + }) + ); + } + } + + window.addEventListener("mousedown", handleMouseDown); + window.addEventListener("keydown", handleKeyDown); + + return () => { + window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("mousedown", handleMouseDown); + }; + }, [dispatch, cells, editCellId, allSelected, selectedRow]); + + return ( + + {errorMessage != null && +
+

+ There is a problem +

+
+
    +
  • + {errorMessage} +
    • +
+
+
+ } + + + + ); +} + +export default EditPayrollNonEmployee diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx new file mode 100644 index 000000000..7e48c19eb --- /dev/null +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -0,0 +1,177 @@ +import React, {Fragment, memo } from 'react' +import { useSelector, useDispatch } from 'react-redux' +import { nanoid } from 'nanoid' +import PayrollTableCell from '../../Components/PayrollTableCell/index' +import InfoCell from '../../Components/InfoCell/index' +import CellValue from '../../Components/CellValue/index' +import TableHeader from '../../Components/TableHeader/index' +import ToggleCell from '../../Components/ToggleCell/index' +import ActualsHeaderRow from '../../Components/ActualsHeaderRow/index' +import { + getCellId +} from '../../Util' + +import { SET_EDITING_CELL } from '../../Reducers/Edit' +import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' + + +function Table({rowData, sheetUpdating}) { + const dispatch = useDispatch(); + const rows = useSelector(state => state.allCells.cells); + + const selectedRow = useSelector(state => state.selected.selectedRow); + const allSelected = useSelector(state => state.selected.all); + + return ( + +
+ + + + + + Name + Grade + Staff number + FTE + Programme Code + Budget type + EU/Non-EU + Assignment status + + + + + + + + + + + + + + + + {rows.map((cells, rowIndex) => { + return + + + + + + + + + + + + + + + + + + + + + + + + + + {Object.keys(window.payroll_non_employee_monthly_data).map((dataKey, index) => { + const monthValues = window.payroll_non_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) + + return ( + Object.keys(monthValues).map((monthKey) => { + const monthValue = monthValues[monthKey]; // Access the value for each month + return ( + + + ); + }) + ); + })} + + + })} + +
Non Payroll data
+ + AprMayJunJulAugSepOctNovDecJanFebMar
+ +
+ + ); +} + +const comparisonFn = function(prevProps, nextProps) { + return ( + prevProps.sheetUpdating === nextProps.sheetUpdating + ) +}; + + +export default memo(Table, comparisonFn); diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index bb943ba78..28b4b59f6 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -144,8 +144,6 @@ function Table({rowData, sheetUpdating}) { return ( Object.keys(monthValues).map((monthKey) => { const monthValue = monthValues[monthKey]; // Access the value for each month - console.log('monthValue', monthValue) - console.log('monthKey', monthKey) return ( , document.getElementById('forecast-app')) -} else if (document.getElementById('payroll-employee-app')) { - ReactDOM.render(, document.getElementById('payroll-employee-app')) } else if (document.getElementById('cost-centre-list-app')) { ReactDOM.render(, document.getElementById('cost-centre-list-app')) } else if (document.getElementById('cost-centre-payroll-list-app')) { ReactDOM.render(, document.getElementById('cost-centre-payroll-list-app')) } +if (document.getElementById('payroll-employee-app')) { + ReactDOM.render(, document.getElementById('payroll-employee-app')) +} + +if (document.getElementById('payroll-non-employee-app')) { + ReactDOM.render(, document.getElementById('payroll-non-employee-app')) +} + // If you want your app to work offline and load faster, you can change // unregister() to register() below. Note this comes with some pitfalls. // Learn more about service workers: https://bit.ly/CRA-PWA diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 1544f96ee..2d87c2333 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -60,6 +60,9 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['payroll_employee_data'] = employee_payroll_data context['payroll_employee_monthly_data'] = employee_payroll_monthly_data + + context['payroll_non_employee_data'] = employee_payroll_data + context['payroll_non_employee_monthly_data'] = employee_payroll_monthly_data return context diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index 4f0289778..b900e3f38 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -15,17 +15,27 @@
- + +
+ +
- -
+
+ +
+
+
{% endblock %} {% block scripts %} {% vite_dev_client %} {% vite_js 'src/index.jsx' %} From c3e4bbea76c3f48d9dbd3d0f801aa68f3aef3c05 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 9 Sep 2024 13:39:21 +0100 Subject: [PATCH 11/23] fix:row display logic in Payroll components Ensure PayrollTable and PayrollNonTable components map data correctly by matching rowIndex with data index. Added console logs in processPayrollData for debugging. --- .../src/Components/PayrollNonTable/index.jsx | 6 +++- .../src/Components/PayrollTable/index.jsx | 34 ++++++++++--------- front_end/src/Util.js | 11 ++++-- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx index 7e48c19eb..78743771d 100644 --- a/front_end/src/Components/PayrollNonTable/index.jsx +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -141,7 +141,8 @@ function Table({rowData, sheetUpdating}) { {Object.keys(window.payroll_non_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_non_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) - return ( + if (rowIndex === index) { + return ( Object.keys(monthValues).map((monthKey) => { const monthValue = monthValues[monthKey]; // Access the value for each month return ( @@ -157,6 +158,9 @@ function Table({rowData, sheetUpdating}) { ); }) ); + } + + })} diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index 28b4b59f6..aeb614be1 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -141,22 +141,24 @@ function Table({rowData, sheetUpdating}) { {Object.keys(window.payroll_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) - return ( - Object.keys(monthValues).map((monthKey) => { - const monthValue = monthValues[monthKey]; // Access the value for each month - return ( - - - ); - }) - ); + if (rowIndex === index) { + return ( + Object.keys(monthValues).map((monthKey) => { + const monthValue = monthValues[monthKey]; // Access the value for each month + return ( + + + ); + }) + ); + } })} diff --git a/front_end/src/Util.js b/front_end/src/Util.js index f5543f25b..702174d49 100644 --- a/front_end/src/Util.js +++ b/front_end/src/Util.js @@ -68,6 +68,7 @@ export async function postData(url = '', data = {}) { export const processPayrollData = (payrollData) => { + console.log('payrollData data', payrollData) let employeePayrollCols = [ "name", "grade", @@ -82,10 +83,10 @@ export const processPayrollData = (payrollData) => { let rows = []; let colIndex = 0 - let cells = {} - payrollData.forEach(function (rowData, rowIndex) { + let cells = {} + // eslint-disable-next-line for (const employeePayrollCol of employeePayrollCols) { cells[employeePayrollCol] = { @@ -96,10 +97,14 @@ export const processPayrollData = (payrollData) => { isEditable: false } colIndex++ + } + + rows.push(cells) }); - rows.push(cells); + + console.log('rows data', rows) return rows; } From 8e4f99e6717d82e92e8b01908fce054ff9c29c57 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Thu, 12 Sep 2024 14:41:49 +0100 Subject: [PATCH 12/23] chore:remove HR, Transaction, ZeroTransaction modules Deleted models, migrations, and related files for HR, Transaction, and ZeroTransaction modules. Consolidated functionalities into the payroll module, implemented new file processors, and updated relevant settings. --- {hr => app_layer}/__init__.py | 0 app_layer/adapters/csv_file_processor.py | 48 +++ app_layer/adapters/excel_file_processor.py | 17 + app_layer/adapters/s3_output_bucket.py | 7 + {hr => app_layer}/admin.py | 0 {hr => app_layer}/apps.py | 4 +- app_layer/exception.py | 130 +++++++ app_layer/layer/service.py | 54 +++ app_layer/log.py | 335 ++++++++++++++++++ app_layer/models.py | 3 + app_layer/ports/file_processor.py | 61 ++++ app_layer/ports/output_service.py | 45 +++ app_layer/serializers.py | 7 + {hr => app_layer}/tests.py | 0 app_layer/urls.py | 7 + app_layer/views.py | 71 ++++ config/settings/base.py | 5 +- .../src/Components/PayrollNonTable/index.jsx | 34 +- .../src/Components/PayrollTable/index.jsx | 2 - front_end/src/Util.js | 3 - hr/migrations/0001_initial.py | 44 --- hr/migrations/__init__.py | 0 hr/models.py | 85 ----- hr/views.py | 3 - payroll/PayrollLookup.csv | 140 ++++++++ payroll/models.py | 250 ++++++++++--- payroll/services.py | 104 ++++++ transaction/__init__.py | 0 transaction/admin.py | 4 - transaction/apps.py | 6 - transaction/migrations/0001_initial.py | 33 -- transaction/migrations/__init__.py | 0 transaction/models.py | 61 ---- transaction/serializers.py | 9 - transaction/tests.py | 3 - transaction/views.py | 3 - zero_transaction/__init__.py | 0 zero_transaction/admin.py | 3 - zero_transaction/apps.py | 6 - zero_transaction/migrations/0001_initial.py | 46 --- zero_transaction/migrations/__init__.py | 0 zero_transaction/models.py | 86 ----- zero_transaction/tests.py | 3 - zero_transaction/views.py | 3 - 44 files changed, 1251 insertions(+), 474 deletions(-) rename {hr => app_layer}/__init__.py (100%) create mode 100644 app_layer/adapters/csv_file_processor.py create mode 100644 app_layer/adapters/excel_file_processor.py create mode 100644 app_layer/adapters/s3_output_bucket.py rename {hr => app_layer}/admin.py (100%) rename {hr => app_layer}/apps.py (62%) create mode 100644 app_layer/exception.py create mode 100644 app_layer/layer/service.py create mode 100644 app_layer/log.py create mode 100644 app_layer/models.py create mode 100644 app_layer/ports/file_processor.py create mode 100644 app_layer/ports/output_service.py create mode 100644 app_layer/serializers.py rename {hr => app_layer}/tests.py (100%) create mode 100644 app_layer/urls.py create mode 100644 app_layer/views.py delete mode 100644 hr/migrations/0001_initial.py delete mode 100644 hr/migrations/__init__.py delete mode 100644 hr/models.py delete mode 100644 hr/views.py create mode 100644 payroll/PayrollLookup.csv create mode 100644 payroll/services.py delete mode 100644 transaction/__init__.py delete mode 100644 transaction/admin.py delete mode 100644 transaction/apps.py delete mode 100644 transaction/migrations/0001_initial.py delete mode 100644 transaction/migrations/__init__.py delete mode 100644 transaction/models.py delete mode 100644 transaction/serializers.py delete mode 100644 transaction/tests.py delete mode 100644 transaction/views.py delete mode 100644 zero_transaction/__init__.py delete mode 100644 zero_transaction/admin.py delete mode 100644 zero_transaction/apps.py delete mode 100644 zero_transaction/migrations/0001_initial.py delete mode 100644 zero_transaction/migrations/__init__.py delete mode 100644 zero_transaction/models.py delete mode 100644 zero_transaction/tests.py delete mode 100644 zero_transaction/views.py diff --git a/hr/__init__.py b/app_layer/__init__.py similarity index 100% rename from hr/__init__.py rename to app_layer/__init__.py diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py new file mode 100644 index 000000000..48d52e931 --- /dev/null +++ b/app_layer/adapters/csv_file_processor.py @@ -0,0 +1,48 @@ +import os +from typing import Optional, Any, List + +from app_layer.log import LogService +from app_layer.ports.file_processor import FileProcessor + +from payroll import models as payroll_models + +class CsvFileProcessor(FileProcessor): + """ + Class CsvFileProcessor + + This class is a subclass of FileProcessor and is responsible for processing CSV files and sending the processed content to an output adapter. + + Methods: + - process_file(bucket_name: str, file_path: str, results: List[Optional[Any]] = None) -> bool + - send_to_output(log: LogService, output_adapter, file_path: str, content: str) + """ + + def process_file(self, bucket_name: str, file_path: str, results: List[Optional[Any]] = None): + # log.deb(f'processing csv file: {file_path}...') + is_valid = os.path.basename(file_path.lower()).startswith(('hrauto', + 'payrollauto')) + + + if not is_valid: + # log.err(f'invalid csv file: {file_path}. will not process.') + return False + + # Extract file_name from file_path + file_name = os.path.basename(file_path) + if file_name.lower().startswith('hrauto'): + hr_model = payroll_models.HR() + hr_model.parse_csv(bucket_name, file_path) + elif file_name.lower().startswith('payrollauto'): + payroll_model = payroll_models.Payroll() + payroll_data = payroll_model.parse_csv(bucket_name, file_path) + if results is not None: + results.append(payroll_data) + else: + # log.deb(f'unknown file: {file_name}. will not continue processing.') + return False + + # log.deb(f'processed file: {file_name}') + return True + + def send_to_output(self, log: LogService, output_adapter, file_path: str, content: str): + output_adapter.send(file_path, content) diff --git a/app_layer/adapters/excel_file_processor.py b/app_layer/adapters/excel_file_processor.py new file mode 100644 index 000000000..661a3719a --- /dev/null +++ b/app_layer/adapters/excel_file_processor.py @@ -0,0 +1,17 @@ +# adapters/excel_file_processor.py + +import os +from typing import Optional, Any + +from app_layer.ports.file_processor import FileProcessor + + +class ExcelFileProcessor(FileProcessor): + def process_file(self, bucket_name, file_path: str, results: Optional[Any] = None): + # log.deb(f'processing excel file: {file_path}...') + is_valid = os.path.basename(file_path).startswith('ActualsDBT') + # log.deb(f'processing excel file: {file_path} - valid[{is_valid}]') + return is_valid + + def send_to_output(self, log, output_adapter, file_path: str, content: str): + output_adapter.send(file_path, content) diff --git a/app_layer/adapters/s3_output_bucket.py b/app_layer/adapters/s3_output_bucket.py new file mode 100644 index 000000000..2babd093d --- /dev/null +++ b/app_layer/adapters/s3_output_bucket.py @@ -0,0 +1,7 @@ +from app_layer.ports.output_service import OutputService + + +class S3OutputBucket(OutputService): + def send(self, log, output_adapter, file_path: str, content: str): + # Email sending logic goes here + print(f'sending email with content: {content} for file: {file_path}') diff --git a/hr/admin.py b/app_layer/admin.py similarity index 100% rename from hr/admin.py rename to app_layer/admin.py diff --git a/hr/apps.py b/app_layer/apps.py similarity index 62% rename from hr/apps.py rename to app_layer/apps.py index 339a3b683..eabf69ab6 100644 --- a/hr/apps.py +++ b/app_layer/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class HRConfig(AppConfig): +class AppLayerConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'hr' + name = 'app_layer' diff --git a/app_layer/exception.py b/app_layer/exception.py new file mode 100644 index 000000000..8181993e0 --- /dev/null +++ b/app_layer/exception.py @@ -0,0 +1,130 @@ +# Pixel Code +# Email: haresh@pixelcode.uk +# +# Copyright (c) 2024. +# +# All rights reserved. +# +# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# For permission requests, write to the author, at the email address above. + +""" +exception.py contains custom exception classes for the service. + +The ServiceException class is a custom exception class for the service. + +Attributes +---------- +message (str) + The message of the exception + +Methods +---------- +append_message(message) -> ServiceException + Append a message to the exception +to_dict() -> dict + Convert the exception to a dictionary + +Custom exceptions are defined for: + - Empty or None Value + - Unknown Value + - Value Exists + - Key Not Found +""" +__version__: str = '0.0.1' + + +class ServiceException(Exception): + """ + A custom exception class for service + + Attributes + ---------- + message (str) + The message of the exception + + Methods + ---------- + append_message(message) -> ServiceException + Append a message to the exception + to_dict() -> dict + Convert the exception to a dictionary + + """ + + message = None + + def __init__(self, message: str): + """ + Constructs all the necessary attributes for the ServiceException object. + + Parameters + ---------- + message : str + The message of the exception + """ + self.message = message + super().__init__(self.message) + + @classmethod + def append_message(cls, message) -> 'ServiceException': + """ + Append a message to the exception + + Parameters + ---------- + message : str + The message to append to the exception + + Returns + ---------- + ServiceException + The exception with the appended message + """ + cls.message = message if cls.message is None else cls.message + f" - {message}" + return cls(cls.message) + + def to_dict(self) -> dict: + """ + Convert the exception to a dictionary + + Returns + ---------- + dict + The exception as a dictionary + """ + return { + "message": self.message + } + + +# Custom exception for empty or none value not permitted +class EmptyOrNoneValueException(ServiceException): + def __init__(self, message="empty or none value not permitted"): + super().__init__(message) + + +# Custom exception for unknown and/or not recognised value +class UnknownValueException(ServiceException): + def __init__(self, message="unknown and/or not recognised value"): + super().__init__(message) + + +# Custom exception for value already exists +class ValueExistsException(ServiceException): + def __init__(self, message="value already exists"): + super().__init__(message) + + +# Custom exception for key not found +class KeyNotFoundException(ServiceException): + def __init__(self, message="key not found"): + super().__init__(message) diff --git a/app_layer/layer/service.py b/app_layer/layer/service.py new file mode 100644 index 000000000..422f6c43c --- /dev/null +++ b/app_layer/layer/service.py @@ -0,0 +1,54 @@ +from typing import Any, Optional, List +from app_layer.adapters.csv_file_processor import CsvFileProcessor +from app_layer.adapters.excel_file_processor import ExcelFileProcessor +from app_layer.adapters.s3_output_bucket import S3OutputBucket +from app_layer.log import LogService +from payroll import models + + +def process_data_input_source_files(log: LogService, file_paths: [str]) -> dict[str, Any]: + # Basic validation check + if not file_paths: + log.war('no files to process. exiting...') + return {'status_code': 204, 'message': 'no files to process'} + + buckets = {} + for path in file_paths: + bucket_name, file_name = path.split('/', 1) + file_type = file_name.split('.')[-1].lower() + + if bucket_name not in buckets: + buckets[bucket_name] = {} + buckets[bucket_name][file_name] = file_type + + processors = { + 'csv': CsvFileProcessor(), + 'xlsx': ExcelFileProcessor(), + 'xls': ExcelFileProcessor() + } + + # Dictionary to store valid data file processors + valid_file_processors = {} + + # Process each file and store the valid data file processors into database + log.deb('processing data input source files...') + results: List[Optional[Any]] = [] # only contains payroll data (we do not store this object into the database) + for bucket_name, files in buckets.items(): + for file_name, file_type in files.items(): + file_path = f"{bucket_name}/{file_name}" + processor = processors.get(file_type) + + if processor: + valid_file_processors[file_path] = processor if processor.process_file(bucket_name, file_path, results) else None + else: + log.err(f'no processor found for file: {file_name}') + + # HR service + hr_model = models.HR() + hr_model.update_records_with_basic_pay_superannuation_ernic_values(results.pop()) + + # When ETL service is done, send the processed content to the output adapter + s3_output_bucket = S3OutputBucket() + + return {'status_code': 200, 'message': 'all input processor files processed'} + diff --git a/app_layer/log.py b/app_layer/log.py new file mode 100644 index 000000000..3e7bcea05 --- /dev/null +++ b/app_layer/log.py @@ -0,0 +1,335 @@ +# Pixel Code +# Email: haresh@pixelcode.uk +# +# Copyright (c) 2024. +# +# All rights reserved. +# +# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# For permission requests, write to the author, at the email address above. + +""" +log.py is a utility module that provides logging functionality for the runtime. + +The module provides a LogService class that can be used to log messages at different log levels. +The LogService class uses the aws_lambda_powertools library to provide logging functionality. + +The LogLevel enumeration defines the different log levels that can be used in the logger. + +The LogService class provides methods to log messages at different log levels, such as INFO, DEBUG, WARNING, ERROR, and + CRITICAL. +The class also provides a method to set the log level of the logger. + +The LogService class uses the inspect and traceback modules to build log messages with additional information such as + the function name, line number, and exception details. + +The LogService class is designed to be used as a utility class to provide logging functionality in the runtime code. + +Example usage: + + log_service = LogService() + log_service.build("MyService", LogLevel.INFO) + log_service.inf("This is an info message") + log_service.deb("This is a debug message") + log_service.war("This is a warning message") + log_service.err("This is an error message") + log_service.cri("This is a critical message") + log_service.set_log_level(LogLevel.DEBUG) + log_service.deb("This is a debug message with log level DEBUG") +""" + +import inspect +import traceback +from enum import Enum + +from aws_lambda_powertools import Logger + +from .exception import EmptyOrNoneValueException + + +class LogLevel(Enum): + """ + LogLevel is an enumeration of the different log levels that can be used in the logger. + """ + DEBUG = 10 + INFO = 20 + WARNING = 30 + ERROR = 40 + CRITICAL = 50 + + +def _log_message_builder(log_level: LogLevel, frame, message: str, exception: Exception = None, + add_trace: bool = False) -> dict: + """ + Builds a log message with the given log level, frame, and message. + + Parameters + ---------- + log_level : LogLevel + The log level of the message (unused but kept for future use). + frame : frame + The frame object of the calling function. + message : str + The message to log. + exception : Exception + The exception to log. + add_trace : bool + Flag to add the traceback to the message. + + Returns + ------- + dict + The log message as a dictionary. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + + # create a dictionary to map, frame.f_code.co_name, frame.f_lineno + + stack_info = { + 'frame': frame.f_code.co_name, + 'line': frame.f_lineno + } + + if message is None or message == "": + message = "" + + stack_info['message'] = message + + + if exception is not None: + stack_info['exception'] = str(exception) + + if add_trace: + stack_info['trace'] = traceback.format_exc() + + # create a json from dictionary + return stack_info + + +class LogService(object): + """ + LogService is a utility class that provides logging functionality. + """ + + def __init__(self): + self._service_name = None + self._logger = None + + def build(self, service_name: str, log_level: LogLevel = LogLevel.INFO) -> 'LogService': + """ + Builds a logger with the given service name and log level. + + Parameters + ---------- + service_name : str + The name of the service. + log_level : LogLevel + The log level of the logger. + + Raises + ------ + EmptyOrNoneValueException + If the service_name argument is empty or None. + """ + + if service_name is None or service_name == "": + raise EmptyOrNoneValueException.append_message("service_name argument cannot be empty (or none)") + + self._logger = Logger(service=service_name, level=log_level.value, log_record_order=["timestamp", "level", "message"]) + return self + + def inject_lambda_context(self, lambda_handler): + """ + Decorator to inject the Lambda context into the logger. + + Parameters + ---------- + lambda_handler : function + The lambda handler function. + + Returns + ------- + function + The decorated lambda handler function. + + Example usage + ------------- + @log_service.inject_lambda_context + def lambda_handler(event, context): + log_service.inf("This is an info message") + return "Hello, World!" + """ + + def decorate(event, context: LambdaContext): + self._logger.structure_logs(append=True, cold_start=self._logger.cold_start, lambda_context=context) + return lambda_handler(event, context) + + return decorate + + def set_correlation_id_from_event(self, event: dict): + """ + Sets the correlation id for the logger based on the event. + + Parameters + ---------- + event : dict + The event dictionary. + """ + if event is None: + self._logger.warn("event argument cannot be None. unable to set correlation id for logger") + return + + request = APIGatewayProxyEvent(event) + self._logger.set_correlation_id(request.request_context.request_id) + + def get_logger(self) -> Logger: + """ + Returns the logger instance. + + Returns + ------- + Logger + The logger instance. + """ + + return self._logger + + def inf(self, message: str): + """ + Logs an INFO message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.INFO, + inspect.currentframe().f_back, + message) + self._logger.info(log_message) + + def deb(self, message: str): + """ + Logs a DEBUG message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.DEBUG, + inspect.currentframe().f_back, + message) + self._logger.debug(log_message) + + def war(self, message: str): + """ + Logs a WARNING message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.WARNING, + inspect.currentframe().f_back, + message) + self._logger.warning(log_message) + + def err(self, message: str): + """ + Logs an ERROR message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.ERROR, + inspect.currentframe().f_back, + message) + self._logger.error(log_message) + + def exc(self, message: str, exception: Exception): + """ + Logs an ERROR message with an exception. + + Parameters + ---------- + message : str + The message to log. + exception : Exception + The exception to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.ERROR, + inspect.currentframe().f_back, + message, + exception) + self._logger.exception(log_message) + + def cri(self, message: str): + """ + Logs a CRITICAL message. + + Parameters + ---------- + message : str + The message to log. + + Raises + ------ + EmptyOrNoneValueException + If the message argument is empty or None. + """ + log_message = _log_message_builder(LogLevel.CRITICAL, + inspect.currentframe().f_back, + message) + self._logger.critical(log_message) + + def set_log_level(self, log_level: LogLevel): + """ + Sets the log level of the logger. + + Parameters + ---------- + log_level : LogLevel + The log level to set. + """ + self._logger.setLevel(log_level.__str__()) diff --git a/app_layer/models.py b/app_layer/models.py new file mode 100644 index 000000000..71a836239 --- /dev/null +++ b/app_layer/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/app_layer/ports/file_processor.py b/app_layer/ports/file_processor.py new file mode 100644 index 000000000..275b939a5 --- /dev/null +++ b/app_layer/ports/file_processor.py @@ -0,0 +1,61 @@ +from abc import ABCMeta, abstractmethod +from typing import List, Optional, Any + + +class FileProcessor(metaclass=ABCMeta): + """ + This code defines an abstract base class called FileProcessor. + + It is designed to be subclassed by other classes that will implement the functionality of processing a file and + sending the processed content to an output adapter. + + The class has two abstract methods: + + 1. process_file(self, file_path: str): + - This method is responsible for processing the content of a file located at the given file_path. + - Subclasses must override this method and provide their own implementation. + + 2. send_to_output(self, output_adapter, file_path: str, content: str): + - This method is responsible for sending the processed content to an output adapter. + - It takes an output_adapter object, the file_path of the processed file, and the content that needs to be sent. + - Subclasses must override this method and provide their own implementation. + + This class should not be instantiated directly. Instead, it should be used as a base class for implementing specific + file processing functionality. + """ + @abstractmethod + def process_file(self, bucket_name: str, file_path: str, results: List[Optional[Any]] = None): + """ + This method is an abstract method that should be overridden by a subclass. + + Parameters: + - log: An instance of the LogService class used for logging. + - file_path: The path of the file to be processed. + - file_path: str + + Returns: + None + + This method is an abstract method and must be implemented by subclasses. + """ + pass + + @abstractmethod + def send_to_output(self, log, output_adapter, file_path: str, content: str): + """ + send_to_output(self, log: LogService, output_adapter, file_path: str, content: str) + + Sends the specified content to the output using the provided output adapter. + + Parameters: + - log (LogService): The log service to use for logging events and errors. + - output_adapter: The output adapter that handles the specific output mechanism. + - file_path (str): The path to the file where the content will be sent. + - content (str): The content to send. + + Returns: + None + + This method is an abstract method and must be implemented by subclasses. + """ + pass \ No newline at end of file diff --git a/app_layer/ports/output_service.py b/app_layer/ports/output_service.py new file mode 100644 index 000000000..b08ab5f5b --- /dev/null +++ b/app_layer/ports/output_service.py @@ -0,0 +1,45 @@ +# ports/output_service.py + +from abc import ABCMeta, abstractmethod + +from app_layer.log import LogService + + +class OutputService(metaclass=ABCMeta): + """ + This is the documentation for the `OutputService` class. + + Class: OutputService + -------------------- + This is an abstract base class that defines the interface for sending output. It is meant to be subclassed and + implemented by concrete output service classes. + + Methods + ------- + send(file_path: str, content: str) + This is an abstract method that subclasses must implement. It takes two parameters: + - file_path: A string representing the file path where the output should be sent. + - content: A string representing the content of the output. + + Note: + - This method should be overridden in subclasses with the specific implementation logic for sending the output. + + Exceptions + ---------- + None + + Example usage + ------------- + ``` + class MyOutputService(OutputService): + def send(self, log: LogService, ssm_client: BaseClient, file_path: str, content: str): + # Implement the logic to send the output to the specified file path + pass + + output_service = MyOutputService() + output_service.send('/path/to/file.txt', 'Hello, World!') + ``` + """ + @abstractmethod + def send(self, log: LogService, ssm_client, file_path: str, content: str): + pass diff --git a/app_layer/serializers.py b/app_layer/serializers.py new file mode 100644 index 000000000..a3f7fb0ae --- /dev/null +++ b/app_layer/serializers.py @@ -0,0 +1,7 @@ +# app_layer/serializers.py +from rest_framework import serializers + + +class S3EventSerializer(serializers.Serializer): + bucket = serializers.CharField() + key = serializers.CharField() \ No newline at end of file diff --git a/hr/tests.py b/app_layer/tests.py similarity index 100% rename from hr/tests.py rename to app_layer/tests.py diff --git a/app_layer/urls.py b/app_layer/urls.py new file mode 100644 index 000000000..85dd1322c --- /dev/null +++ b/app_layer/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from app_layer.views import sns_notification + +urlpatterns = [ + # SNS notification endpoint + path('sns-notification/', sns_notification, name='sns_notification'), +] \ No newline at end of file diff --git a/app_layer/views.py b/app_layer/views.py new file mode 100644 index 000000000..67ba5b678 --- /dev/null +++ b/app_layer/views.py @@ -0,0 +1,71 @@ +import json +import requests +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from rest_framework import status +from rest_framework.decorators import api_view +from rest_framework.response import Response +from layer.service import process_data_input_source_files +from log import LogService, LogLevel +from transaction.serializers import TransactionSerializer +from django.db import transaction as db_transaction + +app_name = "fat" +region_name = 'us-west-1' +log = LogService().build(app_name, LogLevel.DEBUG) + + +@csrf_exempt +def sns_notification(request): + if request.method == 'POST': + message = json.loads(request.body.decode('utf-8')) + sns_message_type = request.META.get('HTTP_X_AMZ_SNS_MESSAGE_TYPE') + + if sns_message_type == 'SubscriptionConfirmation': + # Handle subscription confirmation + confirm_url = message['SubscribeURL'] + # Fetch the URL to confirm the subscription + requests.get(confirm_url) + return JsonResponse({'status': 'subscription confirmed'}) + + elif sns_message_type == 'Notification': + # Process the S3 event notification + s3_info = json.loads(message['Message']) + + # Parse the S3 event notification + buckets = {} + for record in s3_info['Records']: + bucket_name = record['s3']['bucket']['name'] + file_key = record['s3']['object']['key'] + file_name = file_key.split('/')[-1] # Get the file name from the key + file_type = file_name.split('.')[-1].lower() # Get the file type + + # Store the file information in the buckets dictionary + if bucket_name not in buckets: + buckets[bucket_name] = {} + buckets[bucket_name][file_name] = file_type + + # Process the data input source files + result = process_data_input_source_files(log, buckets) + + # Return the result + return JsonResponse(result) + + return JsonResponse({'status': 'not allowed'}, status=405) + + +# External API endpoint +# --------------------- +# To allow data workspace to contact fft for data + +@api_view(['GET']) +@db_transaction.atomic +def get_latest(request): + if request.method == 'GET': + # Below is an example of how to use the TransactionSerializer, not the actual implementation + # of the get_latest function. + serializer = TransactionSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) \ No newline at end of file diff --git a/config/settings/base.py b/config/settings/base.py index a3fa43521..ff8384ff2 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -75,11 +75,8 @@ "simple_history", "axes", "django_chunk_upload_handlers", - "hr", "payroll", - "transaction", - # "app_layer", - "zero_transaction" + "app_layer", ] ROOT_URLCONF = "config.urls" diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx index 78743771d..1204ef774 100644 --- a/front_end/src/Components/PayrollNonTable/index.jsx +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -140,30 +140,26 @@ function Table({rowData, sheetUpdating}) { {Object.keys(window.payroll_non_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_non_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) - if (rowIndex === index) { return ( - Object.keys(monthValues).map((monthKey) => { - const monthValue = monthValues[monthKey]; // Access the value for each month - return ( - - - ); - }) - ); + Object.keys(monthValues).map((monthKey) => { + const monthValue = monthValues[monthKey]; // Access the value for each month + return ( + + + ); + }) + ); } - - })} - })} diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index aeb614be1..8ca10ce52 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -140,7 +140,6 @@ function Table({rowData, sheetUpdating}) { {Object.keys(window.payroll_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) - if (rowIndex === index) { return ( Object.keys(monthValues).map((monthKey) => { @@ -161,7 +160,6 @@ function Table({rowData, sheetUpdating}) { } })} - })} diff --git a/front_end/src/Util.js b/front_end/src/Util.js index 702174d49..fe1079370 100644 --- a/front_end/src/Util.js +++ b/front_end/src/Util.js @@ -1,6 +1,3 @@ -import TableHeader from "./Components/TableHeader/index.jsx"; -import React from "react"; - export const getCellId = (key, index) => { return "id_" + key + "_" + index; } diff --git a/hr/migrations/0001_initial.py b/hr/migrations/0001_initial.py deleted file mode 100644 index 5167ab34b..000000000 --- a/hr/migrations/0001_initial.py +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by Django 5.1 on 2024-09-03 17:59 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='HR', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('group', models.CharField(max_length=255)), - ('directorate', models.CharField(max_length=255)), - ('cc', models.CharField(max_length=255)), - ('cc_name', models.CharField(max_length=255)), - ('last_name', models.CharField(max_length=255)), - ('first_name', models.CharField(max_length=255)), - ('se_no', models.CharField(max_length=50)), - ('salary', models.DecimalField(decimal_places=2, max_digits=10)), - ('grade', models.CharField(max_length=10)), - ('employee_location_city_name', models.CharField(max_length=255)), - ('person_type', models.CharField(max_length=255)), - ('assignment_status', models.CharField(max_length=255)), - ('appointment_status', models.CharField(max_length=255)), - ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), - ('fte', models.DecimalField(decimal_places=2, max_digits=4)), - ('wmi_person', models.CharField(blank=True, max_length=255, null=True)), - ('wmi', models.CharField(blank=True, max_length=255, null=True)), - ('actual_group', models.CharField(max_length=255)), - ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), - ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), - ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), - ('total', models.DecimalField(decimal_places=2, max_digits=10)), - ('costing_cc', models.CharField(max_length=255)), - ('return_field', models.CharField(max_length=255)), - ], - ), - ] diff --git a/hr/migrations/__init__.py b/hr/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/hr/models.py b/hr/models.py deleted file mode 100644 index 624cabc2f..000000000 --- a/hr/models.py +++ /dev/null @@ -1,85 +0,0 @@ -import csv -from io import StringIO - -import boto3 -from django.db import models - - -class HR(models.Model): - group = models.CharField(max_length=255) - directorate = models.CharField(max_length=255) - cc = models.CharField(max_length=255) - cc_name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255) - first_name = models.CharField(max_length=255) - se_no = models.CharField(max_length=50) - salary = models.DecimalField(max_digits=10, decimal_places=2) - grade = models.CharField(max_length=10) - employee_location_city_name = models.CharField(max_length=255) - person_type = models.CharField(max_length=255) - assignment_status = models.CharField(max_length=255) - appointment_status = models.CharField(max_length=255) - working_hours = models.DecimalField(max_digits=5, decimal_places=2) - fte = models.DecimalField(max_digits=4, decimal_places=2) # Full-Time Equivalent - wmi_person = models.CharField(max_length=255, blank=True, null=True) - wmi = models.CharField(max_length=255, blank=True, null=True) - actual_group = models.CharField(max_length=255) - basic_pay = models.DecimalField(max_digits=10, decimal_places=2) - superannuation = models.DecimalField(max_digits=10, decimal_places=2) - ernic = models.DecimalField(max_digits=10, decimal_places=2) # Employer's National Insurance Contribution - total = models.DecimalField(max_digits=10, decimal_places=2) - costing_cc = models.CharField(max_length=255) - return_field = models.CharField(max_length=255) # Assuming 'Return' is a field name; rename if necessary - - def __str__(self): - return f"{self.first_name} {self.last_name} ({self.se_no})" - - def parse_csv(self, bucket_name: str, file_path: str): - try: - # Initialize S3 client - s3 = boto3.client('s3') - - # Get the file from S3 - s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) - - # Read the file content - file_content = s3_object['Body'].read().decode('utf-8-sig') - - # Use StringIO to read the content as a CSV - file = StringIO(file_content) - reader = csv.DictReader(file) - - for row in reader: - HR.objects.create( - group=row['group'], - directorate=row['directorate'], - cc=row['cc'], - cc_name=row['cc_name'], - last_name=row['last_name'], - first_name=row['first_name'], - se_no=row['se_no'], - salary=row['salary'], - grade=row['grade'], - employee_location_city_name=row['employee_location_city_name'], - person_type=row['person_type'], - assignment_status=row['assignment_status'], - appointment_status=row['appointment_status'], - working_hours=row['working_hours'], - fte=row['fte'], - wmi_person=row.get('wmi_person', ''), - wmi=row.get('wmi', ''), - actual_group=row['actual_group'], - basic_pay=row['basic_pay'], - superannuation=row['superannuation'], - ernic=row['ernic'], - total=row['total'], - costing_cc=row['costing_cc'], - return_field=row['return_field'] - ) - except Exception as e: - # log.exc('an error occurred while parsing the HR CSV file', e) - raise e - - class Meta: - verbose_name = "HR" - verbose_name_plural = "HR Records" diff --git a/hr/views.py b/hr/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/hr/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/payroll/PayrollLookup.csv b/payroll/PayrollLookup.csv new file mode 100644 index 000000000..23801c15b --- /dev/null +++ b/payroll/PayrollLookup.csv @@ -0,0 +1,140 @@ +AccountCode,AccountName,PayElementName,ToolTypePayment,CabinetOfficeReturns +52241001,Other Goods/Services,NI Employer,NI Employer,Y +51131001,Basic Salary - Ministers,BIS Private Secretary,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Basic Pay,Basic Pay,Y +51112001,Employer's Social Security Costs - Permanent UK Staff,NI Employer,NI Employer,Y +51113001,Employer's Pension Costs - Permanent UK Staff,BIS O ALPHA ERS OCP Contribution,Superannuation,Y +51114003,Covid 19 Allowances - Permanent UK Staff,Covid19 WFH Expenses Non Taxable,Do Not Use In Forecast,N (Taxable Recovery No Cost To Department) +52121009,Child Care,BIS Childcare Voucher Admin Fee,Do Not Use In Forecast,N +51113001,Employer's Pension Costs - Permanent UK Staff,BIS PTN LEGALGEN ERS OCP Contribution,NI Employer,Y +51113001,Employer's Pension Costs - Permanent UK Staff,BIS PTN Mini ASLC,NI Employer,Y +51114001,Allowances - Permanent UK Staff,BIS Temp Cover Supplement,Basic Pay,Only when Ongoing Date Confirmed +51114001,Allowances - Permanent UK Staff,BIS Enhancement Prog and Prjt Mgmt,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Arrears Basic Pay,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OSP Deduction,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC OSP Payment,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Statutory Sick Pay,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Career Break Leave,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Non Consolidated Bonus Non Pen,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Temp Cover Supplement,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC Arrears OSP Deduction New,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC Arrears OSP Payment New,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Analysts,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Arrears Enhancement Analysts,Basic Pay,Y +51113001,Employer's Pension Costs - Permanent UK Staff,BIS O PREMIUM ERS OCP Contribution,Superannuation,Y +51114001,Allowances - Permanent UK Staff,BIS Salary Enhancement,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Recr Resp Allowance Pen,Basic Pay,Y +51116001,Bonuses - Permanent UK Staff,BIS Special Merit Bonus,Do Not Use In Forecast,N +51113001,Employer's Pension Costs - Permanent UK Staff,BIS O CLASSIC ERS OCP Contribution,Superannuation,Y +51114001,Allowances - Permanent UK Staff,BIS Arrears Enhancement Prog and Prjt Mgmt,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,BIS Mark Time Pen,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Arrears Statutory Sick Pay,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Misc Taxable Allowance,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Cash Pens,Basic Pay,Y +51113001,Employer's Pension Costs - Permanent UK Staff,BIS O NUVOS ERS OCP Contribution,Superannuation,Y +51114001,Allowances - Permanent UK Staff,BIS London Enhancement Statistician,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Arrears Career Break Leave,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Flexible Benefit,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Arrears Unpaid Special Leave,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement FDECC Economist,Basic Pay,Y +51111004,Salaries - Permanent - Annual Leave Deductions,BIS Arrears Annual Leave Payment,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Fast Stream Payment,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS London Enhancement Operational Reseach,Basic Pay,Y +44825028,GSSC ASPP Recoverable,GSSC SHPP Birth Statutory Pay,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OShPP Birth Deduction,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OShPP Birth Payment,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC ShPP Birth Recovery RTI Balance Feed,Do Not Use In Forecast,N +51111004,Salaries - Permanent - Annual Leave Deductions,BIS Annual Leave Payment,Do Not Use In Forecast,N +44825028,GSSC ASPP Recoverable,GSSC SMP Recoverable,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC OMP Deduction,Basic Pay,Y +51112002,GSSC Statutory Maternity Pay,GSSC Statutory Maternity Pay,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Arrears Mark Time Pen,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OMP Payment,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Arrears Extra Duty Weekend,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Saturday 125,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Sunday 125,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Weekday 100,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS On Call Allowance,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement Science and Engineering,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Bank Holiday Overtime 125,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Saturday 200,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Sunday 200,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Overtime Sunday 125,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Procurement,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Arrears Bank Holiday Overtime 150,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Overtime Saturday 125,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Scientific,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Extra Duty Weekend,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Enhancement Cash Pens,Do Not Use In Forecast,N +51113001,Employer's Pension Costs - Permanent UK Staff,BIS O CLASSIC PLUS ERS OCP Contribution,Superannuation,Y +51114001,Allowances - Permanent UK Staff,BIS Private Office Enhancement Allowance,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Overtime Sunday 200,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Overtime Saturday 200,Do Not Use In Forecast,N +52114005,"Honoraria, Fees and Expenses",BIS Honorarium Recurring,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement FDECC Operational Research,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS London Enhancement Economist,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Misc Tax and Pen,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Arrears Overtime Weekday 150,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Electronic Engineers,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Cost of Living Additional No Tax No NI,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Unpaid Special Leave,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement Communications,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Arrears Extra Duty,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement FDECC Statistician,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement Electrical Engineering Inspector,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement Accountants,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Arrears Accountancy allowance,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Accountancy allowance,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Unauthorised Leave Deduction,Do Not Use In Forecast,N +51171007,Other Allowances,BIS Arrears Passport Claims Gross Up,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Emergency Response Allowance,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Enhancement Science and Engineering,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Arrears Bank Holiday Overtime 200,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Overtime Weekday 100,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Traveltime Weekday 100,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OPP Deduction,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC OPP Payment,Basic Pay,Y +51112004,GSSC Statutory Paternity Pay,GSSC Statutory Paternity Pay Birth,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Recr Resp Allowance Pen,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Enhancement Environmental Inspector Manager,Basic Pay,Y +52121015,Staff Transfer/Detached Duty Costs,BIS Offshore Allowance Full Days,Basic Pay,Y +52121015,Staff Transfer/Detached Duty Costs,BIS Offshore Allowance Half Days,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Retention Pay Additional,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement Environmental Investigator,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,GSSC Arrears OMP Deduction,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC Arrears OMP Payment,Do Not Use In Forecast,N +51112002,GSSC Statutory Maternity Pay,GSSC Arrears Statutory Maternity Pay,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS London Enhancement Accountant,Basic Pay,Y +51115001,Overtime - Permanent UK Staff,BIS Bank Holiday Overtime 200,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Pivotal Role Allowance,Basic Pay,Y +51171007,Other Allowances,BIS Car Allowance,Do Not Use In Forecast,N +52121009,Child Care,BIS Childcare Voucher Admin Fee Employer,Do Not Use In Forecast,N +52121009,Child Care,BIS Childcare Vouchers Employer Funded,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS London Enhancement Librarianship,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Arrears Special Merit Bonus,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OAP Deduction,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC OAP Payment,Do Not Use In Forecast,N +51112002,GSSC Statutory Maternity Pay,GSSC Statutory Adoption Pay,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,BIS Childcare Vouchers Adjust,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Keep in Touch Days New,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,BIS Flexi Hours Pay,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Arrears Retention Pay Additional,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,BIS Arrears Non Consolidated Perf Non Pens,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,GSSC Arrears Unpaid Parental Leave,Do Not Use In Forecast,N +51113001,Employer's Pension Costs - Permanent UK Staff,BIS Pension Payment,Superannuation,Y +51114001,Allowances - Permanent UK Staff,BIS New Arrears Private Office Enhancement Allowance,Do Not Use In Forecast,N +51115001,Overtime - Permanent UK Staff,BIS Overtime Weekday 150,Do Not Use In Forecast,N +51111001,Salaries - Permanent UK Staff,BIS OMP Payment for CCVs,Basic Pay,Y +51171007,Other Allowances,BIS BNFL Payment,Do Not Use In Forecast,N +51171007,Other Allowances,BIS Pension RDA Payment,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Misc Taxable Allowance,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Typing Skills Supplement,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Mark Time Non Pen,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS DDaT Specialist Allowance,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement IT Intelligent Customer Function,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Enhancement KIM,Basic Pay,Y +51114001,Allowances - Permanent UK Staff,BIS Arrears Enhancement Communications,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Special Allowance,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Personal Pay Addition,Basic Pay,Y +51111001,Salaries - Permanent UK Staff,BIS Non Consolidated Perf Non Pens,Do Not Use In Forecast,N +51114001,Allowances - Permanent UK Staff,BIS Basic Pay Pen,Basic Pay,Y \ No newline at end of file diff --git a/payroll/models.py b/payroll/models.py index f6c32cacc..3a2663e38 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -2,11 +2,20 @@ import boto3 from io import StringIO from django.db import models -# from app_layer.log import LogService +class ForcastPayroll(models.Model): + name = models.CharField(max_length=100) + nac = models.CharField(max_length=100) + nac_description = models.CharField(max_length=100) + project_code = models.CharField(max_length=100) + programme_code = models.CharField(max_length=100) + budget_type = models.CharField(max_length=100) + class Meta: + abstract = False class EmployeePayroll(models.Model): + id = models.CharField(max_length=100, unique=True, primary_key=True) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -29,11 +38,15 @@ class EmployeePayroll(models.Model): jan = models.IntegerField(editable=True, verbose_name='january') feb = models.IntegerField(editable=True, verbose_name='february') mar = models.IntegerField(editable=True, verbose_name='march') + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) class Meta: abstract = False class NonEmployeePayroll(models.Model): + id = models.CharField(max_length=100, unique=True, primary_key=True) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -57,50 +70,42 @@ class NonEmployeePayroll(models.Model): jan = models.IntegerField(editable=True, verbose_name='january') feb = models.IntegerField(editable=True, verbose_name='february') mar = models.IntegerField(editable=True, verbose_name='march') + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) class Meta: abstract = False -class ForcastPayroll(models.Model): - name = models.CharField(max_length=100) - nac = models.CharField(max_length=100) - nac_description = models.CharField(max_length=100) - project_code = models.CharField(max_length=100) - programme_code = models.CharField(max_length=100) - budget_type = models.CharField(max_length=100) - - class Meta: - abstract = False - - -class Payroll(models.Model): - payroll_id = models.CharField(max_length=100, unique=True, primary_key=True) - created_at = models.DateTimeField(auto_now=True) - business_unit_number = models.CharField(max_length=100) - business_unit_name = models.CharField(max_length=100) - cost_center_number = models.CharField(max_length=100) - cost_center_name = models.CharField(max_length=100) - employee_name = models.CharField(max_length=100) - employee_number = models.CharField(max_length=100) - assignment_number = models.CharField(max_length=100) - payroll_name = models.CharField(max_length=100) - employee_organization = models.CharField(max_length=100) - employee_location = models.CharField(max_length=100) - person_type = models.CharField(max_length=100) - employee_category = models.CharField(max_length=100) - assignment_type = models.CharField(max_length=100) - position = models.CharField(max_length=100) - grade = models.CharField(max_length=100) - account_code = models.CharField(max_length=100) - account_name = models.CharField(max_length=100) - pay_element_name = models.CharField(max_length=100) - effective_date = models.DateField() - debit_amount = models.DecimalField(max_digits=10, decimal_places=2) - credit_amount = models.DecimalField(max_digits=10, decimal_places=2) - +class PayrollModel: + def __init__(self, business_unit_number, business_unit_name, cost_center_number, cost_center_name, employee_name, employee_number, assignment_number, payroll_name, employee_organization, employee_location, person_type, employee_category, assignment_type, position, grade, account_code, account_name, pay_element_name, effective_date, debit_amount, credit_amount, basic_pay, superannuation, ernic): + self.business_unit_number = business_unit_number + self.business_unit_name = business_unit_name + self.cost_center_number = cost_center_number + self.cost_center_name = cost_center_name + self.employee_name = employee_name + self.employee_number = employee_number + self.assignment_number = assignment_number + self.payroll_name = payroll_name + self.employee_organization = employee_organization + self.employee_location = employee_location + self.person_type = person_type + self.employee_category = employee_category + self.assignment_type = assignment_type + self.position = position + self.grade = grade + self.account_code = account_code + self.account_name = account_name + self.pay_element_name = pay_element_name + self.effective_date = effective_date + self.debit_amount = debit_amount + self.credit_amount = credit_amount + self.basic_pay = basic_pay + self.superannuation = superannuation + self.ernic = ernic - class Meta: - abstract = False +class Payroll(): + payroll_list = [] def parse_csv(self, bucket_name: str, file_path: str): try: @@ -118,7 +123,7 @@ def parse_csv(self, bucket_name: str, file_path: str): reader = csv.DictReader(file) for row in reader: - Payroll.objects.create( + payroll = PayrollModel( business_unit_number=row['business_unit_number'], business_unit_name=row['business_unit_name'], cost_center_number=row['cost_center_number'], @@ -140,13 +145,166 @@ def parse_csv(self, bucket_name: str, file_path: str): effective_date=row['effective_date'], debit_amount=row['debit_amount'], credit_amount=row['credit_amount'], + basic_pay=row['basic_pay'], + superannuation=row['superannuation'], + ernic=row['ernic'] ) + self.payroll_list.append(payroll) except Exception as e: - # log.exc('an error occurred while parsing the payroll CSV file', e) raise e + def get_data_list(self): + return self.payroll_list + +class HR(models.Model): + group = models.CharField(max_length=255) + directorate = models.CharField(max_length=255) + cc = models.CharField(max_length=255) + cc_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + first_name = models.CharField(max_length=255) + se_no = models.CharField(max_length=50) + salary = models.DecimalField(max_digits=10, decimal_places=2) + grade = models.CharField(max_length=10) + employee_location_city_name = models.CharField(max_length=255) + person_type = models.CharField(max_length=255) + assignment_status = models.CharField(max_length=255) + appointment_status = models.CharField(max_length=255) + working_hours = models.DecimalField(max_digits=5, decimal_places=2) + fte = models.DecimalField(max_digits=4, decimal_places=2) # Full-Time Equivalent + wmi_person = models.CharField(max_length=255, blank=True, null=True) + wmi = models.CharField(max_length=255, blank=True, null=True) + actual_group = models.CharField(max_length=255) + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) # Employer's National Insurance Contribution + total = models.DecimalField(max_digits=10, decimal_places=2) + costing_cc = models.CharField(max_length=255) + return_field = models.CharField(max_length=255) # Assuming 'Return' is a field name; rename if necessary + programme_code = models.CharField(max_length=255, blank=True, null=True) + + def __str__(self): + return f"{self.first_name} {self.last_name} ({self.se_no})" + + def parse_csv(self, bucket_name: str, file_path: str): + try: + # Initialize S3 client + s3 = boto3.client('s3') + + # Get the file from S3 + s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) + + # Read the file content + file_content = s3_object['Body'].read().decode('utf-8-sig') + + # Use StringIO to read the content as a CSV + file = StringIO(file_content) + reader = csv.DictReader(file) + + for row in reader: + HR.objects.create( + group=row['group'], + directorate=row['directorate'], + cc=row['cc'], + cc_name=row['cc_name'], + last_name=row['last_name'], + first_name=row['first_name'], + se_no=row['se_no'], + salary=row['salary'], + grade=row['grade'], + employee_location_city_name=row['employee_location_city_name'], + person_type=row['person_type'], + assignment_status=row['assignment_status'], + appointment_status=row['appointment_status'], + working_hours=row['working_hours'], + fte=row['fte'], + wmi_person=row.get('wmi_person', ''), + wmi=row.get('wmi', ''), + actual_group=row['actual_group'], + basic_pay=row['basic_pay'], + superannuation=row['superannuation'], + ernic=row['ernic'], + total=row['total'], + costing_cc=row['costing_cc'], + return_field=row['return_field'], + programme_code=row['306162 Code'] + ) + except Exception as e: + # log.exc('an error occurred while parsing the HR CSV file', e) + raise e + + def update_records_with_basic_pay_superannuation_ernic_values(self, payroll_records: list): + hr_records = HR.objects.all() + lookup = PayrollLookup() + + # For each HR record get the total debit amount (debit - credit) and then + # get pay_element_name from the record and use it to get the tool type payment + # using lookup.get_tool_type_payment(pay_element_name) + # If the tool type payment is 'Basic Pay' then add the total debit amount to the basic_pay field of the record + # If the tool type payment is 'Superannuation' then add the total debit amount to the superannuation field of the record + # If the tool type payment is 'ERNIC' then add the total debit amount to the ernic field of the record + # If the tool type payment is not found then log the pay_element_name as not found + for hr_record in hr_records: + + # Iterate through the payroll records and find employee_number that matches staff_employee_number + current_payroll_record = None + for payroll_record in payroll_records: + if (payroll_record.employee_number == hr_record.se_no and + payroll_record.cost_center_number == hr_record.cc): + current_payroll_record = payroll_record + break + + if current_payroll_record is None: + continue + + total_debit = current_payroll_record.debit_amount - current_payroll_record.credit_amount + tool_type_payment = lookup.get_tool_type_payment(current_payroll_record.pay_element_name).lower() + + if tool_type_payment == 'basic pay': + hr_record.basic_pay += total_debit + elif tool_type_payment == 'superannuation': + hr_record.superannuation += total_debit + elif tool_type_payment == 'ernic': + hr_record.ernic += total_debit + else: + # log the pay_element_name as not found + pass + + hr_record.save() + + # For each HR record get the record.basic_pay and set record.wmi_person to 'Yes' if basic_pay is greater than 0 + # and set record.wmi_person to 'No' if basic_pay is less than or equal to 0 + for hr_record in hr_records: + hr_record.wmi_person = 'payroll' if hr_record.basic_pay > 0 else 'nonpayroll' + hr_record.save() + + class Meta: + verbose_name = "HR" + verbose_name_plural = "HR Records" + +class PayrollLookup(): + def __init__(self): + self.lookup_table = {} + self.load_lookup_table() + + + def load_lookup_table(self): + try: + # Read the file content of PayrollLookup.csv located at the current directory + file_content = open('PayrollLookup.csv', 'r').read() + + # Use StringIO to read the content as a CSV + file = StringIO(file_content) + reader = csv.DictReader(file) + + for row in reader: + self.lookup_table[row['PayElementName']] = row['ToolTypePayment'] + + except Exception as e: + # log.exc('an error occurred while loading the payroll lookup table', e) + raise e + + + def get_tool_type_payment(self, pay_element_name: str) -> str: + return self.lookup_table.get(pay_element_name, "Not found") - def get_unique_rows_by_cost_center(self, cost_center_number: str): - rows = Payroll.objects.filter(cost_center_number=cost_center_number) - unique_rows = rows.distinct('employee_number') - return unique_rows diff --git a/payroll/services.py b/payroll/services.py new file mode 100644 index 000000000..5f75f347e --- /dev/null +++ b/payroll/services.py @@ -0,0 +1,104 @@ +import datetime +from collections import defaultdict + +from payroll.models import EmployeePayroll, NonEmployeePayroll, HR + + +def update_employee_tables(): + # Get all objects from HR table + hr_objects = HR.objects.all() + + # Loop through all objects and update the employee table based on the HR record.wmi_person + # If wmi_person is payroll, then put it in the employee table (EmployeePayroll) + # If wmi_person is not payroll, then put it in the non-employee table (NonEmployeePayroll) + # If the record does not exist, create it + + for hr_record in hr_objects: + name = hr_record.first_name + " " + hr_record.last_name + grade = hr_record.grade + se_no = hr_record.se_no + fte = hr_record.fte + programme_code = hr_record.programme_code + assignment_status = hr_record.assignment_status + current_month = datetime.datetime.now().strftime('%B').lower() + current_year = datetime.datetime.now().year + + if hr_record.wmi_person == "payroll": + employee_record = EmployeePayroll() + employee_record.name = name + employee_record.grade = grade + employee_record.staff_number = se_no + employee_record.fte = fte + employee_record.programme_code = programme_code + employee_record.eu_non_eu = "unknown" + employee_record.budget_type = "unknown" + employee_record.assignment_status = assignment_status + employee_record.current_month = current_month + employee_record.current_year = current_year + employee_record.save() + elif hr_record.wmi_person == "nonpayroll": + non_employee_record = NonEmployeePayroll() + non_employee_record.name = name + non_employee_record.grade = grade + non_employee_record.staff_number = se_no + non_employee_record.fte = fte + non_employee_record.programme_code = programme_code + non_employee_record.eu_non_eu = "unknown" + non_employee_record.budget_type = "unknown" + non_employee_record.assignment_status = assignment_status + non_employee_record.person_type = hr_record.person_type + non_employee_record.current_month = current_month + non_employee_record.current_year = current_year + non_employee_record.save() + +def get_forecast_basic_pay_for_employee_non_employee_payroll() -> dict: + # Get all objects from EmployeePayroll table + employee_objects = EmployeePayroll.objects.all() + + # Get all objects from NonEmployeePayroll table + non_employee_objects = NonEmployeePayroll.objects.all() + + # Loop through all objects and create a list of dictionaries for each month (april to march) + # Each dictionary will have the following keys: + # - month + # - total (sum all basic_pay values for that month) + + # Initialize lists + employee_list = [] + non_employee_list = [] + + # Define the months + months = [ + 'april', 'may', 'june', 'july', 'august', + 'september', 'october', 'november', 'december', + 'january', 'february', 'march' + ] + + # Use defaultdict to group employee and non-employee records by month + employee_monthly_totals = defaultdict(float) + non_employee_monthly_totals = defaultdict(float) + + # Sum the basic pay for each month for employees and non-employees + for employee_record in employee_objects: + employee_monthly_totals[employee_record.current_month.lower()] \ + += employee_record.basic_pay + + for non_employee_record in non_employee_objects: + non_employee_monthly_totals[non_employee_record.current_month.lower()] \ + += non_employee_record.basic_pay + + # Build the monthly totals lists + for month in months: + employee_list.append({ + "month": month, + "total": employee_monthly_totals[month] + }) + non_employee_list.append({ + "month": month, + "total": non_employee_monthly_totals[month] + }) + + return { + "employee": employee_list, + "non_employee": non_employee_list + } diff --git a/transaction/__init__.py b/transaction/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/transaction/admin.py b/transaction/admin.py deleted file mode 100644 index de4554a2c..000000000 --- a/transaction/admin.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.contrib import admin -from .models import Transaction - -admin.site.register(Transaction) \ No newline at end of file diff --git a/transaction/apps.py b/transaction/apps.py deleted file mode 100644 index 26d1c705d..000000000 --- a/transaction/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class TransactionsConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'transaction' diff --git a/transaction/migrations/0001_initial.py b/transaction/migrations/0001_initial.py deleted file mode 100644 index f2bd540d7..000000000 --- a/transaction/migrations/0001_initial.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 5.1 on 2024-08-29 10:22 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Transaction', - fields=[ - ('transaction_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), - ('source', models.CharField(max_length=100)), - ('entity', models.CharField(max_length=100)), - ('cost_centre', models.CharField(max_length=100)), - ('group', models.CharField(max_length=100)), - ('account', models.CharField(max_length=100)), - ('programme', models.CharField(max_length=100)), - ('line_description', models.TextField()), - ('net', models.DecimalField(decimal_places=2, max_digits=10)), - ('fiscal_period', models.CharField(max_length=100)), - ('date_of_journal', models.DateField()), - ('purchase_order_number', models.CharField(max_length=100)), - ('supplier_name', models.CharField(max_length=100)), - ('level4_code', models.CharField(max_length=100)), - ], - ), - ] diff --git a/transaction/migrations/__init__.py b/transaction/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/transaction/models.py b/transaction/models.py deleted file mode 100644 index 29cb96443..000000000 --- a/transaction/models.py +++ /dev/null @@ -1,61 +0,0 @@ -import csv -from io import StringIO - -import boto3 -from django.db import models - -class Transaction(models.Model): - transaction_id = models.CharField(max_length=100, unique=True, primary_key=True) - created_at = models.DateTimeField(auto_now=True) - source = models.CharField(max_length=100) - entity = models.CharField(max_length=100) - cost_centre = models.CharField(max_length=100) - group = models.CharField(max_length=100) - account = models.CharField(max_length=100) - programme = models.CharField(max_length=100) - line_description = models.TextField() - net = models.DecimalField(max_digits=10, decimal_places=2) - fiscal_period = models.CharField(max_length=100) - date_of_journal = models.DateField() - purchase_order_number = models.CharField(max_length=100) - supplier_name = models.CharField(max_length=100) - level4_code = models.CharField(max_length=100) - - def parse_csv(self, bucket_name: str, file_path: str): - try: - # Initialize S3 client - s3 = boto3.client('s3') - - # Get the file from S3 - s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) - - # Read the file content - file_content = s3_object['Body'].read().decode('utf-8-sig') - - # Use StringIO to read the content as a CSV - file = StringIO(file_content) - reader = csv.DictReader(file) - - for row in reader: - Transaction.objects.create( - transaction_id=row['transaction_id'], - source=row['source'], - entity=row['entity'], - cost_centre=row['cost_centre'], - group=row['group'], - account=row['account'], - programme=row['programme'], - line_description=row['line_description'], - net=row['net'], - fiscal_period=row['fiscal_period'], - date_of_journal=row['date_of_journal'], - purchase_order_number=row['purchase_order_number'], - supplier_name=row['supplier_name'], - level4_code=row['level4_code'], - ) - except Exception as e: - # log.exc('an error occurred while parsing the CSV file', e) - raise e - - def __str__(self): - return self.transaction_id \ No newline at end of file diff --git a/transaction/serializers.py b/transaction/serializers.py deleted file mode 100644 index fc5d41f4e..000000000 --- a/transaction/serializers.py +++ /dev/null @@ -1,9 +0,0 @@ -# transaction/serializers.py -from rest_framework import serializers -from .models import Transaction - - -class TransactionSerializer(serializers.ModelSerializer): - class Meta: - model = Transaction - fields = ['transaction_id', 'amount', 'timestamp', 'description'] \ No newline at end of file diff --git a/transaction/tests.py b/transaction/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/transaction/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/transaction/views.py b/transaction/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/transaction/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/zero_transaction/__init__.py b/zero_transaction/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/zero_transaction/admin.py b/zero_transaction/admin.py deleted file mode 100644 index 8c38f3f3d..000000000 --- a/zero_transaction/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/zero_transaction/apps.py b/zero_transaction/apps.py deleted file mode 100644 index 7d960518a..000000000 --- a/zero_transaction/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class ZeroTransactionsConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'zero_transaction' diff --git a/zero_transaction/migrations/0001_initial.py b/zero_transaction/migrations/0001_initial.py deleted file mode 100644 index ea786c996..000000000 --- a/zero_transaction/migrations/0001_initial.py +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by Django 5.1 on 2024-09-03 15:58 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='ZeroTransaction', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('entity', models.CharField(max_length=100)), - ('cost_centre', models.CharField(max_length=100)), - ('account', models.CharField(max_length=100)), - ('programme', models.CharField(max_length=100)), - ('analysis_1', models.CharField(max_length=100)), - ('spare_1', models.CharField(blank=True, max_length=100, null=True)), - ('spare_2', models.CharField(blank=True, max_length=100, null=True)), - ('apr', models.FloatField()), - ('may', models.FloatField()), - ('jun', models.FloatField()), - ('jul', models.FloatField()), - ('aug', models.FloatField()), - ('sep', models.FloatField()), - ('oct', models.FloatField()), - ('nov', models.FloatField()), - ('dec', models.FloatField()), - ('jan', models.FloatField()), - ('feb', models.FloatField()), - ('mar', models.FloatField()), - ('adj1', models.FloatField(blank=True, null=True)), - ('adj2', models.FloatField(blank=True, null=True)), - ('adj3', models.FloatField(blank=True, null=True)), - ('total', models.FloatField()), - ], - options={ - 'verbose_name_plural': 'Zero Transaction Entries', - }, - ), - ] diff --git a/zero_transaction/migrations/__init__.py b/zero_transaction/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/zero_transaction/models.py b/zero_transaction/models.py deleted file mode 100644 index 7d196e5b7..000000000 --- a/zero_transaction/models.py +++ /dev/null @@ -1,86 +0,0 @@ -import csv -import boto3 -from io import StringIO -from django.db import models -# from app_layer.log import LogService - - -class ZeroTransaction(models.Model): - entity = models.CharField(max_length=100) - cost_centre = models.CharField(max_length=100) - account = models.CharField(max_length=100) - programme = models.CharField(max_length=100) - analysis_1 = models.CharField(max_length=100) - spare_1 = models.CharField(max_length=100, blank=True, null=True) - spare_2 = models.CharField(max_length=100, blank=True, null=True) - - apr = models.FloatField() - may = models.FloatField() - jun = models.FloatField() - jul = models.FloatField() - aug = models.FloatField() - sep = models.FloatField() - oct = models.FloatField() - nov = models.FloatField() - dec = models.FloatField() - jan = models.FloatField() - feb = models.FloatField() - mar = models.FloatField() - - adj1 = models.FloatField(blank=True, null=True) - adj2 = models.FloatField(blank=True, null=True) - adj3 = models.FloatField(blank=True, null=True) - - total = models.FloatField() - - def parse_csv(self, bucket_name: str, file_path: str): - try: - # Initialize S3 client - s3 = boto3.client('s3') - - # Get the file from S3 - s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) - - # Read the file content - file_content = s3_object['Body'].read().decode('utf-8-sig') - - # Use StringIO to read the content as a CSV - file = StringIO(file_content) - reader = csv.DictReader(file) - - for row in reader: - ZeroTransaction.objects.create( - entity=row['entity'], - cost_centre=row['cost_centre'], - account=row['account'], - programme=row['programme'], - analysis_1=row['analysis_1'], - spare_1=row.get('spare_1', ''), - spare_2=row.get('spare_2', ''), - apr=row['apr'], - may=row['may'], - jun=row['jun'], - jul=row['jul'], - aug=row['aug'], - sep=row['sep'], - oct=row['oct'], - nov=row['nov'], - dec=row['dec'], - jan=row['jan'], - feb=row['feb'], - mar=row['mar'], - adj1=row.get('adj1', ''), - adj2=row.get('adj2', ''), - adj3=row.get('adj3', ''), - total=row['total'], - ) - except Exception as e: - # log.exc('an error occurred while parsing the CSV file', e) - raise e - - class Meta: - verbose_name_plural = "Zero Transaction Entries" - - def __str__(self): - return f"{self.entity} - {self.cost_centre} - {self.account}" - diff --git a/zero_transaction/tests.py b/zero_transaction/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/zero_transaction/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/zero_transaction/views.py b/zero_transaction/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/zero_transaction/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. From 5f45bffdd51a02ae5e6bfcc225f72ef5ecc3a982 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Fri, 13 Sep 2024 16:12:05 +0100 Subject: [PATCH 13/23] chore:remove unused log service and redundant payroll models Deleted the log service module and initial migration script from the payroll directory. Removed obsolete HR model from payroll.models and refactored CsvFileProcessor and related modules to use the revised Payroll model. --- app_layer/adapters/csv_file_processor.py | 41 ++- app_layer/adapters/excel_file_processor.py | 2 +- app_layer/adapters/s3_output_bucket.py | 2 +- app_layer/layer/service.py | 29 +- app_layer/log.py | 335 --------------------- app_layer/ports/file_processor.py | 2 +- app_layer/ports/output_service.py | 4 +- app_layer/views.py | 26 +- forecast/views/edit_forecast.py | 2 + forecast/views/upload_file.py | 4 + payroll/migrations/0001_initial.py | 123 -------- payroll/models.py | 208 +++---------- payroll/services.py | 15 +- payroll/tests.py | 3 - payroll/views.py | 2 - 15 files changed, 95 insertions(+), 703 deletions(-) delete mode 100644 app_layer/log.py delete mode 100644 payroll/migrations/0001_initial.py delete mode 100644 payroll/tests.py diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py index 48d52e931..873f677fc 100644 --- a/app_layer/adapters/csv_file_processor.py +++ b/app_layer/adapters/csv_file_processor.py @@ -1,10 +1,11 @@ import os from typing import Optional, Any, List - -from app_layer.log import LogService from app_layer.ports.file_processor import FileProcessor +from hr.models import HRModel from payroll import models as payroll_models +from payroll.models import PayrollModel + class CsvFileProcessor(FileProcessor): """ @@ -17,32 +18,40 @@ class CsvFileProcessor(FileProcessor): - send_to_output(log: LogService, output_adapter, file_path: str, content: str) """ - def process_file(self, bucket_name: str, file_path: str, results: List[Optional[Any]] = None): + def process_file(self, bucket_name: str, file_path: str): # log.deb(f'processing csv file: {file_path}...') is_valid = os.path.basename(file_path.lower()).startswith(('hrauto', 'payrollauto')) - if not is_valid: - # log.err(f'invalid csv file: {file_path}. will not process.') return False + results = [] + file_type_mapping = { + 'hrauto': 'hr', + 'payrollauto': 'payroll' + } + + # Determine file type based on prefix + for prefix, filetype in file_type_mapping.items(): + if file_path.lower().startswith(prefix): + results.append(dict(file_type=filetype)) + break + + # Get file type (without popping from list) from results list (value type dict) + filetype = results[-1].get('file_type') + # Extract file_name from file_path - file_name = os.path.basename(file_path) - if file_name.lower().startswith('hrauto'): - hr_model = payroll_models.HR() + if filetype == 'hr': + hr_model = HRModel() hr_model.parse_csv(bucket_name, file_path) - elif file_name.lower().startswith('payrollauto'): - payroll_model = payroll_models.Payroll() - payroll_data = payroll_model.parse_csv(bucket_name, file_path) - if results is not None: - results.append(payroll_data) + elif filetype == 'payroll': + payroll_model = PayrollModel() + payroll_model.parse_csv(bucket_name, file_path) else: - # log.deb(f'unknown file: {file_name}. will not continue processing.') return False - # log.deb(f'processed file: {file_name}') return True - def send_to_output(self, log: LogService, output_adapter, file_path: str, content: str): + def send_to_output(self, output_adapter, file_path: str, content: str): output_adapter.send(file_path, content) diff --git a/app_layer/adapters/excel_file_processor.py b/app_layer/adapters/excel_file_processor.py index 661a3719a..b04b3d75b 100644 --- a/app_layer/adapters/excel_file_processor.py +++ b/app_layer/adapters/excel_file_processor.py @@ -13,5 +13,5 @@ def process_file(self, bucket_name, file_path: str, results: Optional[Any] = Non # log.deb(f'processing excel file: {file_path} - valid[{is_valid}]') return is_valid - def send_to_output(self, log, output_adapter, file_path: str, content: str): + def send_to_output(self, output_adapter, file_path: str, content: str): output_adapter.send(file_path, content) diff --git a/app_layer/adapters/s3_output_bucket.py b/app_layer/adapters/s3_output_bucket.py index 2babd093d..d45107b41 100644 --- a/app_layer/adapters/s3_output_bucket.py +++ b/app_layer/adapters/s3_output_bucket.py @@ -2,6 +2,6 @@ class S3OutputBucket(OutputService): - def send(self, log, output_adapter, file_path: str, content: str): + def send(self, output_adapter, file_path: str, content: str): # Email sending logic goes here print(f'sending email with content: {content} for file: {file_path}') diff --git a/app_layer/layer/service.py b/app_layer/layer/service.py index 422f6c43c..92587ed56 100644 --- a/app_layer/layer/service.py +++ b/app_layer/layer/service.py @@ -1,15 +1,11 @@ -from typing import Any, Optional, List +from typing import Any from app_layer.adapters.csv_file_processor import CsvFileProcessor from app_layer.adapters.excel_file_processor import ExcelFileProcessor -from app_layer.adapters.s3_output_bucket import S3OutputBucket -from app_layer.log import LogService -from payroll import models +from hr.models import HRModel -def process_data_input_source_files(log: LogService, file_paths: [str]) -> dict[str, Any]: - # Basic validation check +def process_data_input_source_files(file_paths: [str]) -> dict[str, Any]: if not file_paths: - log.war('no files to process. exiting...') return {'status_code': 204, 'message': 'no files to process'} buckets = {} @@ -31,24 +27,19 @@ def process_data_input_source_files(log: LogService, file_paths: [str]) -> dict[ valid_file_processors = {} # Process each file and store the valid data file processors into database - log.deb('processing data input source files...') - results: List[Optional[Any]] = [] # only contains payroll data (we do not store this object into the database) for bucket_name, files in buckets.items(): for file_name, file_type in files.items(): - file_path = f"{bucket_name}/{file_name}" + file_path = f'{bucket_name}/{file_name}' processor = processors.get(file_type) - if processor: - valid_file_processors[file_path] = processor if processor.process_file(bucket_name, file_path, results) else None + valid_file_processors[file_path] = processor \ + if processor.process_file(bucket_name, file_path) \ + else None else: - log.err(f'no processor found for file: {file_name}') + return {'status_code': 400, 'message': 'invalid file type'} # HR service - hr_model = models.HR() - hr_model.update_records_with_basic_pay_superannuation_ernic_values(results.pop()) - - # When ETL service is done, send the processed content to the output adapter - s3_output_bucket = S3OutputBucket() - + hr_model = HRModel() + hr_model.update_records_with_basic_pay_superannuation_ernic_values() return {'status_code': 200, 'message': 'all input processor files processed'} diff --git a/app_layer/log.py b/app_layer/log.py deleted file mode 100644 index 3e7bcea05..000000000 --- a/app_layer/log.py +++ /dev/null @@ -1,335 +0,0 @@ -# Pixel Code -# Email: haresh@pixelcode.uk -# -# Copyright (c) 2024. -# -# All rights reserved. -# -# No part of this code may be used or reproduced in any manner whatsoever without the prior written consent of the author. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -# SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# For permission requests, write to the author, at the email address above. - -""" -log.py is a utility module that provides logging functionality for the runtime. - -The module provides a LogService class that can be used to log messages at different log levels. -The LogService class uses the aws_lambda_powertools library to provide logging functionality. - -The LogLevel enumeration defines the different log levels that can be used in the logger. - -The LogService class provides methods to log messages at different log levels, such as INFO, DEBUG, WARNING, ERROR, and - CRITICAL. -The class also provides a method to set the log level of the logger. - -The LogService class uses the inspect and traceback modules to build log messages with additional information such as - the function name, line number, and exception details. - -The LogService class is designed to be used as a utility class to provide logging functionality in the runtime code. - -Example usage: - - log_service = LogService() - log_service.build("MyService", LogLevel.INFO) - log_service.inf("This is an info message") - log_service.deb("This is a debug message") - log_service.war("This is a warning message") - log_service.err("This is an error message") - log_service.cri("This is a critical message") - log_service.set_log_level(LogLevel.DEBUG) - log_service.deb("This is a debug message with log level DEBUG") -""" - -import inspect -import traceback -from enum import Enum - -from aws_lambda_powertools import Logger - -from .exception import EmptyOrNoneValueException - - -class LogLevel(Enum): - """ - LogLevel is an enumeration of the different log levels that can be used in the logger. - """ - DEBUG = 10 - INFO = 20 - WARNING = 30 - ERROR = 40 - CRITICAL = 50 - - -def _log_message_builder(log_level: LogLevel, frame, message: str, exception: Exception = None, - add_trace: bool = False) -> dict: - """ - Builds a log message with the given log level, frame, and message. - - Parameters - ---------- - log_level : LogLevel - The log level of the message (unused but kept for future use). - frame : frame - The frame object of the calling function. - message : str - The message to log. - exception : Exception - The exception to log. - add_trace : bool - Flag to add the traceback to the message. - - Returns - ------- - dict - The log message as a dictionary. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - - # create a dictionary to map, frame.f_code.co_name, frame.f_lineno - - stack_info = { - 'frame': frame.f_code.co_name, - 'line': frame.f_lineno - } - - if message is None or message == "": - message = "" - - stack_info['message'] = message - - - if exception is not None: - stack_info['exception'] = str(exception) - - if add_trace: - stack_info['trace'] = traceback.format_exc() - - # create a json from dictionary - return stack_info - - -class LogService(object): - """ - LogService is a utility class that provides logging functionality. - """ - - def __init__(self): - self._service_name = None - self._logger = None - - def build(self, service_name: str, log_level: LogLevel = LogLevel.INFO) -> 'LogService': - """ - Builds a logger with the given service name and log level. - - Parameters - ---------- - service_name : str - The name of the service. - log_level : LogLevel - The log level of the logger. - - Raises - ------ - EmptyOrNoneValueException - If the service_name argument is empty or None. - """ - - if service_name is None or service_name == "": - raise EmptyOrNoneValueException.append_message("service_name argument cannot be empty (or none)") - - self._logger = Logger(service=service_name, level=log_level.value, log_record_order=["timestamp", "level", "message"]) - return self - - def inject_lambda_context(self, lambda_handler): - """ - Decorator to inject the Lambda context into the logger. - - Parameters - ---------- - lambda_handler : function - The lambda handler function. - - Returns - ------- - function - The decorated lambda handler function. - - Example usage - ------------- - @log_service.inject_lambda_context - def lambda_handler(event, context): - log_service.inf("This is an info message") - return "Hello, World!" - """ - - def decorate(event, context: LambdaContext): - self._logger.structure_logs(append=True, cold_start=self._logger.cold_start, lambda_context=context) - return lambda_handler(event, context) - - return decorate - - def set_correlation_id_from_event(self, event: dict): - """ - Sets the correlation id for the logger based on the event. - - Parameters - ---------- - event : dict - The event dictionary. - """ - if event is None: - self._logger.warn("event argument cannot be None. unable to set correlation id for logger") - return - - request = APIGatewayProxyEvent(event) - self._logger.set_correlation_id(request.request_context.request_id) - - def get_logger(self) -> Logger: - """ - Returns the logger instance. - - Returns - ------- - Logger - The logger instance. - """ - - return self._logger - - def inf(self, message: str): - """ - Logs an INFO message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.INFO, - inspect.currentframe().f_back, - message) - self._logger.info(log_message) - - def deb(self, message: str): - """ - Logs a DEBUG message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.DEBUG, - inspect.currentframe().f_back, - message) - self._logger.debug(log_message) - - def war(self, message: str): - """ - Logs a WARNING message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.WARNING, - inspect.currentframe().f_back, - message) - self._logger.warning(log_message) - - def err(self, message: str): - """ - Logs an ERROR message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.ERROR, - inspect.currentframe().f_back, - message) - self._logger.error(log_message) - - def exc(self, message: str, exception: Exception): - """ - Logs an ERROR message with an exception. - - Parameters - ---------- - message : str - The message to log. - exception : Exception - The exception to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.ERROR, - inspect.currentframe().f_back, - message, - exception) - self._logger.exception(log_message) - - def cri(self, message: str): - """ - Logs a CRITICAL message. - - Parameters - ---------- - message : str - The message to log. - - Raises - ------ - EmptyOrNoneValueException - If the message argument is empty or None. - """ - log_message = _log_message_builder(LogLevel.CRITICAL, - inspect.currentframe().f_back, - message) - self._logger.critical(log_message) - - def set_log_level(self, log_level: LogLevel): - """ - Sets the log level of the logger. - - Parameters - ---------- - log_level : LogLevel - The log level to set. - """ - self._logger.setLevel(log_level.__str__()) diff --git a/app_layer/ports/file_processor.py b/app_layer/ports/file_processor.py index 275b939a5..d95bd9ed0 100644 --- a/app_layer/ports/file_processor.py +++ b/app_layer/ports/file_processor.py @@ -41,7 +41,7 @@ def process_file(self, bucket_name: str, file_path: str, results: List[Optional[ pass @abstractmethod - def send_to_output(self, log, output_adapter, file_path: str, content: str): + def send_to_output(self, output_adapter, file_path: str, content: str): """ send_to_output(self, log: LogService, output_adapter, file_path: str, content: str) diff --git a/app_layer/ports/output_service.py b/app_layer/ports/output_service.py index b08ab5f5b..ef77f4915 100644 --- a/app_layer/ports/output_service.py +++ b/app_layer/ports/output_service.py @@ -2,8 +2,6 @@ from abc import ABCMeta, abstractmethod -from app_layer.log import LogService - class OutputService(metaclass=ABCMeta): """ @@ -41,5 +39,5 @@ def send(self, log: LogService, ssm_client: BaseClient, file_path: str, content: ``` """ @abstractmethod - def send(self, log: LogService, ssm_client, file_path: str, content: str): + def send(self, ssm_client, file_path: str, content: str): pass diff --git a/app_layer/views.py b/app_layer/views.py index 67ba5b678..f20095273 100644 --- a/app_layer/views.py +++ b/app_layer/views.py @@ -2,17 +2,10 @@ import requests from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt -from rest_framework import status -from rest_framework.decorators import api_view -from rest_framework.response import Response from layer.service import process_data_input_source_files -from log import LogService, LogLevel -from transaction.serializers import TransactionSerializer -from django.db import transaction as db_transaction app_name = "fat" region_name = 'us-west-1' -log = LogService().build(app_name, LogLevel.DEBUG) @csrf_exempt @@ -46,26 +39,9 @@ def sns_notification(request): buckets[bucket_name][file_name] = file_type # Process the data input source files - result = process_data_input_source_files(log, buckets) + result = process_data_input_source_files(buckets) # Return the result return JsonResponse(result) return JsonResponse({'status': 'not allowed'}, status=405) - - -# External API endpoint -# --------------------- -# To allow data workspace to contact fft for data - -@api_view(['GET']) -@db_transaction.atomic -def get_latest(request): - if request.method == 'GET': - # Below is an example of how to use the TransactionSerializer, not the actual implementation - # of the get_latest function. - serializer = TransactionSerializer(data=request.data) - if serializer.is_valid(): - serializer.save() - return Response(serializer.data, status=status.HTTP_201_CREATED) - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) \ No newline at end of file diff --git a/forecast/views/edit_forecast.py b/forecast/views/edit_forecast.py index 96b84e9b6..2c3d91c2d 100644 --- a/forecast/views/edit_forecast.py +++ b/forecast/views/edit_forecast.py @@ -97,9 +97,11 @@ def get_financial_code_serialiser(cost_centre_code, financial_year): ) .order_by(*edit_forecast_order()) ) + financial_code_serialiser = FinancialCodeSerializer( financial_codes, many=True, context={"financial_year": financial_year} ) + return financial_code_serialiser diff --git a/forecast/views/upload_file.py b/forecast/views/upload_file.py index 6b22845ce..d2c058e7d 100644 --- a/forecast/views/upload_file.py +++ b/forecast/views/upload_file.py @@ -1,10 +1,14 @@ import logging +from cffi.cffi_opcode import CLASS_NAME from django.conf import settings from django.contrib.auth.mixins import UserPassesTestMixin +from django.core.files.storage import FileSystemStorage, default_storage +from django.http import JsonResponse from django.urls import reverse_lazy from django.views.generic.edit import FormView +from app_layer.layer.service import process_data_input_source_files from forecast.forms import UploadActualsForm, UploadBudgetsForm from forecast.tasks import process_uploaded_file from upload_file.models import FileUpload diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py deleted file mode 100644 index d6cebb34e..000000000 --- a/payroll/migrations/0001_initial.py +++ /dev/null @@ -1,123 +0,0 @@ -# Generated by Django 5.1 on 2024-09-08 17:51 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='EmployeePayroll', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100)), - ('grade', models.CharField(max_length=100)), - ('staff_number', models.CharField(max_length=100)), - ('fte', models.DecimalField(decimal_places=2, max_digits=5)), - ('programme_code', models.CharField(max_length=100)), - ('budget_type', models.CharField(max_length=100)), - ('eu_non_eu', models.CharField(max_length=100)), - ('assignment_status', models.CharField(max_length=100)), - ('current_month', models.CharField(max_length=100)), - ('current_year', models.CharField(max_length=100)), - ('apr', models.IntegerField(verbose_name='april')), - ('may', models.IntegerField(verbose_name='may')), - ('jun', models.IntegerField(verbose_name='june')), - ('jul', models.IntegerField(verbose_name='july')), - ('aug', models.IntegerField(verbose_name='august')), - ('sep', models.IntegerField(verbose_name='september')), - ('oct', models.IntegerField(verbose_name='october')), - ('nov', models.IntegerField(verbose_name='november')), - ('dec', models.IntegerField(verbose_name='december')), - ('jan', models.IntegerField(verbose_name='january')), - ('feb', models.IntegerField(verbose_name='february')), - ('mar', models.IntegerField(verbose_name='march')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='ForcastPayroll', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100)), - ('nac', models.CharField(max_length=100)), - ('nac_description', models.CharField(max_length=100)), - ('project_code', models.CharField(max_length=100)), - ('programme_code', models.CharField(max_length=100)), - ('budget_type', models.CharField(max_length=100)), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='NonEmployeePayroll', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=100)), - ('grade', models.CharField(max_length=100)), - ('staff_number', models.CharField(max_length=100)), - ('fte', models.DecimalField(decimal_places=2, max_digits=5)), - ('programme_code', models.CharField(max_length=100)), - ('budget_type', models.CharField(max_length=100)), - ('eu_non_eu', models.CharField(max_length=100)), - ('assignment_status', models.CharField(max_length=100)), - ('person_type', models.CharField(max_length=100)), - ('current_month', models.CharField(max_length=100)), - ('current_year', models.CharField(max_length=100)), - ('apr', models.IntegerField(verbose_name='april')), - ('may', models.IntegerField(verbose_name='may')), - ('jun', models.IntegerField(verbose_name='june')), - ('jul', models.IntegerField(verbose_name='july')), - ('aug', models.IntegerField(verbose_name='august')), - ('sep', models.IntegerField(verbose_name='september')), - ('oct', models.IntegerField(verbose_name='october')), - ('nov', models.IntegerField(verbose_name='november')), - ('dec', models.IntegerField(verbose_name='december')), - ('jan', models.IntegerField(verbose_name='january')), - ('feb', models.IntegerField(verbose_name='february')), - ('mar', models.IntegerField(verbose_name='march')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='Payroll', - fields=[ - ('payroll_id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), - ('created_at', models.DateTimeField(auto_now=True)), - ('business_unit_number', models.CharField(max_length=100)), - ('business_unit_name', models.CharField(max_length=100)), - ('cost_center_number', models.CharField(max_length=100)), - ('cost_center_name', models.CharField(max_length=100)), - ('employee_name', models.CharField(max_length=100)), - ('employee_number', models.CharField(max_length=100)), - ('assignment_number', models.CharField(max_length=100)), - ('payroll_name', models.CharField(max_length=100)), - ('employee_organization', models.CharField(max_length=100)), - ('employee_location', models.CharField(max_length=100)), - ('person_type', models.CharField(max_length=100)), - ('employee_category', models.CharField(max_length=100)), - ('assignment_type', models.CharField(max_length=100)), - ('position', models.CharField(max_length=100)), - ('grade', models.CharField(max_length=100)), - ('account_code', models.CharField(max_length=100)), - ('account_name', models.CharField(max_length=100)), - ('pay_element_name', models.CharField(max_length=100)), - ('effective_date', models.DateField()), - ('debit_amount', models.DecimalField(decimal_places=2, max_digits=10)), - ('credit_amount', models.DecimalField(decimal_places=2, max_digits=10)), - ], - options={ - 'abstract': False, - }, - ), - ] diff --git a/payroll/models.py b/payroll/models.py index 3a2663e38..a403d2f4d 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -77,53 +77,56 @@ class NonEmployeePayroll(models.Model): class Meta: abstract = False -class PayrollModel: - def __init__(self, business_unit_number, business_unit_name, cost_center_number, cost_center_name, employee_name, employee_number, assignment_number, payroll_name, employee_organization, employee_location, person_type, employee_category, assignment_type, position, grade, account_code, account_name, pay_element_name, effective_date, debit_amount, credit_amount, basic_pay, superannuation, ernic): - self.business_unit_number = business_unit_number - self.business_unit_name = business_unit_name - self.cost_center_number = cost_center_number - self.cost_center_name = cost_center_name - self.employee_name = employee_name - self.employee_number = employee_number - self.assignment_number = assignment_number - self.payroll_name = payroll_name - self.employee_organization = employee_organization - self.employee_location = employee_location - self.person_type = person_type - self.employee_category = employee_category - self.assignment_type = assignment_type - self.position = position - self.grade = grade - self.account_code = account_code - self.account_name = account_name - self.pay_element_name = pay_element_name - self.effective_date = effective_date - self.debit_amount = debit_amount - self.credit_amount = credit_amount - self.basic_pay = basic_pay - self.superannuation = superannuation - self.ernic = ernic +class PayrollModel(models.Model): + business_unit_number = models.CharField(max_length=100) + business_unit_name = models.CharField(max_length=100) + cost_center_number = models.CharField(max_length=100) + cost_center_name = models.CharField(max_length=100) + employee_name = models.CharField(max_length=100) + employee_number = models.CharField(max_length=100) + assignment_number = models.CharField(max_length=100) + payroll_name = models.CharField(max_length=100) + employee_organization = models.CharField(max_length=100) + employee_location = models.CharField(max_length=100) + person_type = models.CharField(max_length=100) + employee_category = models.CharField(max_length=100) + assignment_type = models.CharField(max_length=100) + position = models.CharField(max_length=100) + grade = models.CharField(max_length=100) + account_code = models.CharField(max_length=100) + account_name = models.CharField(max_length=100) + pay_element_name = models.CharField(max_length=100) + effective_date = models.CharField(max_length=100) + debit_amount = models.CharField(max_length=100) + credit_amount = models.CharField(max_length=100) + basic_pay = models.CharField(max_length=100) + superannuation = models.CharField(max_length=100) + ernic = models.CharField(max_length=100) -class Payroll(): - payroll_list = [] + class Meta: + abstract = False def parse_csv(self, bucket_name: str, file_path: str): try: - # Initialize S3 client - s3 = boto3.client('s3') + if bucket_name is not None: + # Initialize S3 client + s3 = boto3.client('s3') - # Get the file from S3 - s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) + # Get the file from S3 + s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) - # Read the file content - file_content = s3_object['Body'].read().decode('utf-8-sig') + # Read the file content + file_content = s3_object['Body'].read().decode('utf-8-sig') + else: + with open(file_path, 'r') as file: + file_content = file.read() # Use StringIO to read the content as a CSV file = StringIO(file_content) reader = csv.DictReader(file) for row in reader: - payroll = PayrollModel( + payroll_entry = PayrollModel( business_unit_number=row['business_unit_number'], business_unit_name=row['business_unit_name'], cost_center_number=row['cost_center_number'], @@ -149,138 +152,14 @@ def parse_csv(self, bucket_name: str, file_path: str): superannuation=row['superannuation'], ernic=row['ernic'] ) - self.payroll_list.append(payroll) - except Exception as e: - raise e - - def get_data_list(self): - return self.payroll_list - -class HR(models.Model): - group = models.CharField(max_length=255) - directorate = models.CharField(max_length=255) - cc = models.CharField(max_length=255) - cc_name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255) - first_name = models.CharField(max_length=255) - se_no = models.CharField(max_length=50) - salary = models.DecimalField(max_digits=10, decimal_places=2) - grade = models.CharField(max_length=10) - employee_location_city_name = models.CharField(max_length=255) - person_type = models.CharField(max_length=255) - assignment_status = models.CharField(max_length=255) - appointment_status = models.CharField(max_length=255) - working_hours = models.DecimalField(max_digits=5, decimal_places=2) - fte = models.DecimalField(max_digits=4, decimal_places=2) # Full-Time Equivalent - wmi_person = models.CharField(max_length=255, blank=True, null=True) - wmi = models.CharField(max_length=255, blank=True, null=True) - actual_group = models.CharField(max_length=255) - basic_pay = models.DecimalField(max_digits=10, decimal_places=2) - superannuation = models.DecimalField(max_digits=10, decimal_places=2) - ernic = models.DecimalField(max_digits=10, decimal_places=2) # Employer's National Insurance Contribution - total = models.DecimalField(max_digits=10, decimal_places=2) - costing_cc = models.CharField(max_length=255) - return_field = models.CharField(max_length=255) # Assuming 'Return' is a field name; rename if necessary - programme_code = models.CharField(max_length=255, blank=True, null=True) - def __str__(self): - return f"{self.first_name} {self.last_name} ({self.se_no})" - - def parse_csv(self, bucket_name: str, file_path: str): - try: - # Initialize S3 client - s3 = boto3.client('s3') - - # Get the file from S3 - s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) - - # Read the file content - file_content = s3_object['Body'].read().decode('utf-8-sig') - - # Use StringIO to read the content as a CSV - file = StringIO(file_content) - reader = csv.DictReader(file) - - for row in reader: - HR.objects.create( - group=row['group'], - directorate=row['directorate'], - cc=row['cc'], - cc_name=row['cc_name'], - last_name=row['last_name'], - first_name=row['first_name'], - se_no=row['se_no'], - salary=row['salary'], - grade=row['grade'], - employee_location_city_name=row['employee_location_city_name'], - person_type=row['person_type'], - assignment_status=row['assignment_status'], - appointment_status=row['appointment_status'], - working_hours=row['working_hours'], - fte=row['fte'], - wmi_person=row.get('wmi_person', ''), - wmi=row.get('wmi', ''), - actual_group=row['actual_group'], - basic_pay=row['basic_pay'], - superannuation=row['superannuation'], - ernic=row['ernic'], - total=row['total'], - costing_cc=row['costing_cc'], - return_field=row['return_field'], - programme_code=row['306162 Code'] - ) + payroll_entry.save() except Exception as e: - # log.exc('an error occurred while parsing the HR CSV file', e) raise e - def update_records_with_basic_pay_superannuation_ernic_values(self, payroll_records: list): - hr_records = HR.objects.all() - lookup = PayrollLookup() - - # For each HR record get the total debit amount (debit - credit) and then - # get pay_element_name from the record and use it to get the tool type payment - # using lookup.get_tool_type_payment(pay_element_name) - # If the tool type payment is 'Basic Pay' then add the total debit amount to the basic_pay field of the record - # If the tool type payment is 'Superannuation' then add the total debit amount to the superannuation field of the record - # If the tool type payment is 'ERNIC' then add the total debit amount to the ernic field of the record - # If the tool type payment is not found then log the pay_element_name as not found - for hr_record in hr_records: - - # Iterate through the payroll records and find employee_number that matches staff_employee_number - current_payroll_record = None - for payroll_record in payroll_records: - if (payroll_record.employee_number == hr_record.se_no and - payroll_record.cost_center_number == hr_record.cc): - current_payroll_record = payroll_record - break - - if current_payroll_record is None: - continue - - total_debit = current_payroll_record.debit_amount - current_payroll_record.credit_amount - tool_type_payment = lookup.get_tool_type_payment(current_payroll_record.pay_element_name).lower() - - if tool_type_payment == 'basic pay': - hr_record.basic_pay += total_debit - elif tool_type_payment == 'superannuation': - hr_record.superannuation += total_debit - elif tool_type_payment == 'ernic': - hr_record.ernic += total_debit - else: - # log the pay_element_name as not found - pass - - hr_record.save() - - # For each HR record get the record.basic_pay and set record.wmi_person to 'Yes' if basic_pay is greater than 0 - # and set record.wmi_person to 'No' if basic_pay is less than or equal to 0 - for hr_record in hr_records: - hr_record.wmi_person = 'payroll' if hr_record.basic_pay > 0 else 'nonpayroll' - hr_record.save() - - class Meta: - verbose_name = "HR" - verbose_name_plural = "HR Records" + @staticmethod + def delete_all_records(): + PayrollModel.objects.all().delete() class PayrollLookup(): def __init__(self): @@ -301,10 +180,9 @@ def load_lookup_table(self): self.lookup_table[row['PayElementName']] = row['ToolTypePayment'] except Exception as e: - # log.exc('an error occurred while loading the payroll lookup table', e) raise e def get_tool_type_payment(self, pay_element_name: str) -> str: - return self.lookup_table.get(pay_element_name, "Not found") + return self.lookup_table.get(pay_element_name, "unknown") diff --git a/payroll/services.py b/payroll/services.py index 5f75f347e..02c82301b 100644 --- a/payroll/services.py +++ b/payroll/services.py @@ -1,12 +1,11 @@ import datetime from collections import defaultdict - -from payroll.models import EmployeePayroll, NonEmployeePayroll, HR +from hr.models import HRModel +from payroll.models import EmployeePayroll, NonEmployeePayroll def update_employee_tables(): - # Get all objects from HR table - hr_objects = HR.objects.all() + hr_objects = HRModel.objects.all() # Loop through all objects and update the employee table based on the HR record.wmi_person # If wmi_person is payroll, then put it in the employee table (EmployeePayroll) @@ -14,7 +13,7 @@ def update_employee_tables(): # If the record does not exist, create it for hr_record in hr_objects: - name = hr_record.first_name + " " + hr_record.last_name + name = f'{hr_record.first_name} {hr_record.last_name}' grade = hr_record.grade se_no = hr_record.se_no fte = hr_record.fte @@ -80,12 +79,10 @@ def get_forecast_basic_pay_for_employee_non_employee_payroll() -> dict: # Sum the basic pay for each month for employees and non-employees for employee_record in employee_objects: - employee_monthly_totals[employee_record.current_month.lower()] \ - += employee_record.basic_pay + employee_monthly_totals[employee_record.current_month.lower()] += employee_record.basic_pay for non_employee_record in non_employee_objects: - non_employee_monthly_totals[non_employee_record.current_month.lower()] \ - += non_employee_record.basic_pay + non_employee_monthly_totals[non_employee_record.current_month.lower()] += non_employee_record.basic_pay # Build the monthly totals lists for month in months: diff --git a/payroll/tests.py b/payroll/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/payroll/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/payroll/views.py b/payroll/views.py index 13c0a8c27..1ef74d4a5 100644 --- a/payroll/views.py +++ b/payroll/views.py @@ -1,6 +1,4 @@ from django.shortcuts import render -from django.views.generic import TemplateView -from .models import Payroll def payroll_list(request): return render(request, 'payroll/list/list.html') From 84f81ac811ca5323f037f7bf6864fc7df1f40961 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 14 Sep 2024 17:19:37 +0100 Subject: [PATCH 14/23] feat: add payroll editing and non-employee payroll processing Implemented functionality to post payroll edits and handle non-employee payroll data. This includes new views, forms, serializers, and URL patterns to support these features, as well as refactoring existing models and functions. --- app_layer/adapters/csv_file_processor.py | 4 +- config/settings/base.py | 1 + .../src/Components/PayrollTableCell/index.jsx | 20 +++--- locustfile.py | 11 ++++ payroll/edit_payroll.py | 62 ++++++++++++++++--- payroll/forms.py | 28 +++++++++ payroll/models.py | 6 +- payroll/serialisers.py | 38 +++++++++++- payroll/templates/payroll/edit/edit.html | 22 ++++--- payroll/urls.py | 19 ++++-- 10 files changed, 171 insertions(+), 40 deletions(-) create mode 100644 payroll/forms.py diff --git a/app_layer/adapters/csv_file_processor.py b/app_layer/adapters/csv_file_processor.py index 873f677fc..4d4507592 100644 --- a/app_layer/adapters/csv_file_processor.py +++ b/app_layer/adapters/csv_file_processor.py @@ -4,7 +4,7 @@ from hr.models import HRModel from payroll import models as payroll_models -from payroll.models import PayrollModel +from payroll.models import PayrollEntry class CsvFileProcessor(FileProcessor): @@ -46,7 +46,7 @@ def process_file(self, bucket_name: str, file_path: str): hr_model = HRModel() hr_model.parse_csv(bucket_name, file_path) elif filetype == 'payroll': - payroll_model = PayrollModel() + payroll_model = PayrollEntry() payroll_model.parse_csv(bucket_name, file_path) else: return False diff --git a/config/settings/base.py b/config/settings/base.py index ff8384ff2..363ed4d71 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -76,6 +76,7 @@ "axes", "django_chunk_upload_handlers", "payroll", + "hr", "app_layer", ] diff --git a/front_end/src/Components/PayrollTableCell/index.jsx b/front_end/src/Components/PayrollTableCell/index.jsx index 4e810c1c3..83ea08067 100644 --- a/front_end/src/Components/PayrollTableCell/index.jsx +++ b/front_end/src/Components/PayrollTableCell/index.jsx @@ -2,9 +2,10 @@ import React, {Fragment, useState, useEffect, memo } from 'react' import { useSelector, useDispatch } from 'react-redux' import { SET_EDITING_CELL } from '../../Reducers/Edit' import { + formatValue, postData, - processForecastData, - formatValue + processForecastData, processPayrollData + } from '../../Util' import { SET_ERROR } from '../../Reducers/Error' import { SET_CELLS } from '../../Reducers/Cells' @@ -115,11 +116,10 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) const updateValue = () => { // setValue(value) + console.log('staff number:', cells[rowIndex]["staff_number"].value) console.log('cell value:', value) console.log('cell key:', cellKey) - - return; let newValue = 0 if (value > 1) { @@ -141,22 +141,18 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value let payload = new FormData() - payload.append("natural_account_code", cells[rowIndex]["natural_account_code"].value) - payload.append("programme_code", cells[rowIndex]["programme"].value) - payload.append("project_code", cells[rowIndex]["project_code"].value) - payload.append("analysis1_code", cells[rowIndex]["analysis1_code"].value) - payload.append("analysis2_code", cells[rowIndex]["analysis2_code"].value) + payload.append("staff_number", cells[rowIndex]["staff_number"].value) payload.append("csrfmiddlewaretoken", crsfToken) payload.append("month", cellKey) - payload.append("amount", intAmount) + payload.append("amount", getValue()) postData( - `/forecast/update-forecast/${window.cost_centre}/${window.financial_year}`, + `/payroll/paste-payroll/${window.cost_centre}/${window.financial_year}`, payload ).then((response) => { setIsUpdating(false) if (response.status === 200) { - let rows = processForecastData(response.data) + let rows = processPayrollData(response.data) dispatch({ type: SET_CELLS, cells: rows diff --git a/locustfile.py b/locustfile.py index eece4b116..25f5b0895 100644 --- a/locustfile.py +++ b/locustfile.py @@ -27,6 +27,17 @@ def post_to_edit_forecast(self): }, ) + @task(3) + def post_to_edit_payroll(self): + self.client.post( + f"/payroll/edit/{os.environ['TEST_COST_CENTRE_CODE']}/", + data={ + "all_selected": True, + "paste_content": paste_content, + "csrfmiddlewaretoken": os.environ["CRSF_TOKEN"], + "headers": sso_headers, + }, + ) class FFTUser(HttpLocust): task_set = UserBehaviour diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 2d87c2333..2e71fd0a2 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -1,17 +1,47 @@ import json import logging +from django.db import transaction +from django.http import JsonResponse +from django.views.generic import FormView from django.views.generic.base import TemplateView from costcentre.models import CostCentre from forecast.views.base import ( - CostCentrePermissionTest, + CostCentrePermissionTest, NoCostCentreCodeInURLError, ) -from payroll.models import EmployeePayroll -from payroll.serialisers import EmployeePayrollSerializer, EmployeeMonthlyPayrollSerializer +from payroll.forms import PasteHRForm +from payroll.models import EmployeePayroll, NonEmployeePayroll +from payroll.serialisers import EmployeePayrollSerializer, EmployeeMonthlyPayrollSerializer, \ + NonEmployeePayrollSerializer, NonEmployeeMonthlyPayrollSerializer logger = logging.getLogger(__name__) +class EditPayrollUpdatesView( + CostCentrePermissionTest, + FormView, +): + form_class = PasteHRForm + + @transaction.atomic + def form_valid(self, form): # noqa: C901 + if "cost_centre_code" not in self.kwargs: + raise NoCostCentreCodeInURLError("no cost centre code provided in URL") + + try: + cost_centre_code = self.kwargs["cost_centre_code"] + paste_content = form.cleaned_data["paste_content"] + pasted_at_row = form.cleaned_data.get("pasted_at_row", None) + all_selected = form.cleaned_data.get("all_selected", False) + + logger.info(f"cost_centre_code: {cost_centre_code}") + logger.info(f"paste_content: {paste_content}") + logger.info(f"pasted_at_row: {pasted_at_row}") + logger.info(f"all_selected: {all_selected}") + return JsonResponse({"status": "success"}) + except Exception as e: + logger.error(f"error parsing form data: {e}") + return self.form_invalid(form) class EditPayrollView( CostCentrePermissionTest, @@ -40,12 +70,21 @@ def get_employee_payroll_serialiser(self): payroll_serialiser = EmployeePayrollSerializer(get_all_employee_data, many=True) return payroll_serialiser - def get_employee_payroll_monthly_serialiser(self): get_all_employee_data = EmployeePayroll.objects.all() payroll_monthly_serialiser = EmployeeMonthlyPayrollSerializer(get_all_employee_data, many=True) return payroll_monthly_serialiser + def get_non_employee_payroll_serialiser(self): + get_all_non_employee_data = NonEmployeePayroll.objects.all() + non_payroll_serialiser = NonEmployeePayrollSerializer(get_all_non_employee_data, many=True) + return non_payroll_serialiser + + def get_non_employee_payroll_monthly_serialiser(self): + get_all_non_employee_data = NonEmployeePayroll.objects.all() + non_payroll_monthly_serialiser = NonEmployeeMonthlyPayrollSerializer(get_all_non_employee_data, many=True) + return non_payroll_monthly_serialiser + def get_context_data(self, **kwargs): employee_payroll_serialiser = self.get_employee_payroll_serialiser() employee_payroll_serialiser_data = employee_payroll_serialiser.data @@ -55,14 +94,23 @@ def get_context_data(self, **kwargs): employee_payroll_monthly_serialiser_data = employee_payroll_monthly_serialiser.data employee_payroll_monthly_data = json.dumps(employee_payroll_monthly_serialiser_data) + non_employee_payroll_serialiser = self.get_non_employee_payroll_serialiser() + non_employee_payroll_serialiser_data = non_employee_payroll_serialiser.data + non_employee_payroll_data = json.dumps(non_employee_payroll_serialiser_data) + + non_employee_payroll_monthly_serialiser = self.get_non_employee_payroll_monthly_serialiser() + non_employee_payroll_monthly_serialiser_data = non_employee_payroll_monthly_serialiser.data + non_employee_payroll_monthly_data = json.dumps(non_employee_payroll_monthly_serialiser_data) self.title = "Edit payroll forecast" + paste_form = PasteHRForm() + context = super().get_context_data(**kwargs) + context["paste_form"] = paste_form context['payroll_employee_data'] = employee_payroll_data context['payroll_employee_monthly_data'] = employee_payroll_monthly_data - - context['payroll_non_employee_data'] = employee_payroll_data - context['payroll_non_employee_monthly_data'] = employee_payroll_monthly_data + context['payroll_non_employee_data'] = non_employee_payroll_data + context['payroll_non_employee_monthly_data'] = non_employee_payroll_monthly_data return context diff --git a/payroll/forms.py b/payroll/forms.py new file mode 100644 index 000000000..8492b95d1 --- /dev/null +++ b/payroll/forms.py @@ -0,0 +1,28 @@ +import json + +from django import forms + +# Form for pasting HR data into the payroll table +class PasteHRForm(forms.Form): + all_selected = forms.BooleanField(widget=forms.HiddenInput(), required=False) + pasted_at_row = forms.CharField( + widget=forms.HiddenInput(), + required=False, + ) + paste_content = forms.CharField( + widget=forms.HiddenInput(), + required=False, + ) + + def clean_pasted_at_row(self): + data = self.cleaned_data["pasted_at_row"] + + if not data: + return None + + try: + json_data = json.loads(data) + except json.JSONDecodeError: + raise forms.ValidationError("invalid row data supplied") + + return json_data \ No newline at end of file diff --git a/payroll/models.py b/payroll/models.py index a403d2f4d..23b3d4f3a 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -77,7 +77,7 @@ class NonEmployeePayroll(models.Model): class Meta: abstract = False -class PayrollModel(models.Model): +class PayrollEntry(models.Model): business_unit_number = models.CharField(max_length=100) business_unit_name = models.CharField(max_length=100) cost_center_number = models.CharField(max_length=100) @@ -126,7 +126,7 @@ def parse_csv(self, bucket_name: str, file_path: str): reader = csv.DictReader(file) for row in reader: - payroll_entry = PayrollModel( + payroll_entry = PayrollEntry( business_unit_number=row['business_unit_number'], business_unit_name=row['business_unit_name'], cost_center_number=row['cost_center_number'], @@ -159,7 +159,7 @@ def parse_csv(self, bucket_name: str, file_path: str): @staticmethod def delete_all_records(): - PayrollModel.objects.all().delete() + PayrollEntry.objects.all().delete() class PayrollLookup(): def __init__(self): diff --git a/payroll/serialisers.py b/payroll/serialisers.py index 281a1e0c4..284dfb777 100644 --- a/payroll/serialisers.py +++ b/payroll/serialisers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from payroll.models import EmployeePayroll +from payroll.models import EmployeePayroll, NonEmployeePayroll class EmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): @@ -36,4 +36,40 @@ class Meta: "eu_non_eu", "assignment_status", ] + read_only_fields = fields + + +class NonEmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): + class Meta: + model = NonEmployeePayroll + fields = [ + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", + "jan", + "feb", + "mar", + ] + read_only_fields = fields + + +class NonEmployeePayrollSerializer(serializers.ModelSerializer): + class Meta: + model = NonEmployeePayroll + fields = [ + "name", + "grade", + "staff_number", + "fte", + "programme_code", + "budget_type", + "eu_non_eu", + "assignment_status", + ] read_only_fields = fields \ No newline at end of file diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index b900e3f38..fa9fa11e8 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -18,17 +18,17 @@
-
- -
-
-
+
+
+ {% csrf_token %} + {{ paste_form }} +
-
- -
-
-
+
+
+ {% csrf_token %} + {{ paste_form }} +
{% endblock %} {% block scripts %} {% vite_dev_client %} {% vite_js 'src/index.jsx' %} diff --git a/payroll/urls.py b/payroll/urls.py index e01ec9a33..8cb7ded43 100644 --- a/payroll/urls.py +++ b/payroll/urls.py @@ -1,11 +1,20 @@ from django.urls import path from . import views -from .edit_payroll import EditPayrollView +from .edit_payroll import EditPayrollView, EditPayrollUpdatesView from .edit_select_cost_centre import SelectCostCentreView urlpatterns = [ - path('list/', views.payroll_list, name='payroll_list'), - path("edit/select-cost-centre/", SelectCostCentreView.as_view(), name="select_cost_centre"), - path("edit///",EditPayrollView.as_view(), name="edit_payroll"), - + path('list/', + views.payroll_list, + name='payroll_list'), + path("edit/select-cost-centre/", + SelectCostCentreView.as_view(), + name="select_cost_centre"), + path("edit///", + EditPayrollView.as_view(), + name="edit_payroll"), + # Path that takes the updated payroll data with cost centre code and financial year as parameters + path("paste-payroll//", + EditPayrollUpdatesView.as_view(), + name="paste_payroll"), ] \ No newline at end of file From 933549ecf57c508e35ee44cb259bf2dff2ba8214 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 14 Sep 2024 18:29:32 +0100 Subject: [PATCH 15/23] chore:separate employee and non-employee payroll data Refactored state management to distinguish between employee and non-employee payroll cells. Introduced new reducers, components, and utility functions to handle payroll and non-payroll data independently. Updated UI and data processing logic for better organization and maintainability. --- .../Components/EditPayrollEmployee/index.jsx | 6 +- .../EditPayrollNonEmployee/index.jsx | 10 +- .../src/Components/PayrollCellValue/index.jsx | 30 ++ .../Components/PayrollNonCellValue/index.jsx | 30 ++ .../src/Components/PayrollNonTable/index.jsx | 25 +- .../Components/PayrollNonTableCell/index.jsx | 271 ++++++++++++++++++ .../src/Components/PayrollTable/index.jsx | 20 +- .../src/Components/PayrollTableCell/index.jsx | 10 +- front_end/src/Reducers/Cells.js | 17 +- front_end/src/Store.js | 2 +- front_end/src/Util.js | 42 ++- payroll/templates/payroll/edit/edit.html | 20 +- 12 files changed, 436 insertions(+), 47 deletions(-) create mode 100644 front_end/src/Components/PayrollCellValue/index.jsx create mode 100644 front_end/src/Components/PayrollNonCellValue/index.jsx create mode 100644 front_end/src/Components/PayrollNonTableCell/index.jsx diff --git a/front_end/src/Components/EditPayrollEmployee/index.jsx b/front_end/src/Components/EditPayrollEmployee/index.jsx index 106e96b93..454c89709 100644 --- a/front_end/src/Components/EditPayrollEmployee/index.jsx +++ b/front_end/src/Components/EditPayrollEmployee/index.jsx @@ -6,7 +6,7 @@ import { store } from '../../Store'; import EditActionBar from '../../Components/EditActionBar/index' import { SET_ERROR } from '../../Reducers/Error' -import { SET_CELLS } from '../../Reducers/Cells' +import { SET_EMPLOYEE_CELLS } from '../../Reducers/Cells' import { OPEN_FILTER_IF_CLOSED } from '../../Reducers/Filter' import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' import { @@ -23,7 +23,7 @@ function EditPayrollEmployee() { const selectedRow = useSelector(state => state.selected.selectedRow) const allSelected = useSelector(state => state.selected.all) - const cells = useSelector(state => state.allCells.cells); + const cells = useSelector(state => state.allCells.employeeCells); const editCellId = useSelector(state => state.edit.cellId); const [sheetUpdating, setSheetUpdating] = useState(false) @@ -49,7 +49,7 @@ function EditPayrollEmployee() { if (window.payroll_employee_data) { let rows = processPayrollData(window.payroll_employee_data) dispatch({ - type: SET_CELLS, + type: SET_EMPLOYEE_CELLS, cells: rows }) } else { diff --git a/front_end/src/Components/EditPayrollNonEmployee/index.jsx b/front_end/src/Components/EditPayrollNonEmployee/index.jsx index 1e2aef64e..b8b57e2a3 100644 --- a/front_end/src/Components/EditPayrollNonEmployee/index.jsx +++ b/front_end/src/Components/EditPayrollNonEmployee/index.jsx @@ -6,12 +6,12 @@ import { store } from '../../Store'; import EditActionBar from '../../Components/EditActionBar/index' import { SET_ERROR } from '../../Reducers/Error' -import { SET_CELLS } from '../../Reducers/Cells' +import {SET_NON_EMPLOYEE_CELLS} from '../../Reducers/Cells' import { OPEN_FILTER_IF_CLOSED } from '../../Reducers/Filter' import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' import { getCellId, - processPayrollData, + processNonPayrollData, } from '../../Util' @@ -23,7 +23,7 @@ function EditPayrollNonEmployee() { const selectedRow = useSelector(state => state.selected.selectedRow) const allSelected = useSelector(state => state.selected.all) - const cells = useSelector(state => state.allCells.cells); + const cells = useSelector(state => state.allCells.nonEmployeeCells); const editCellId = useSelector(state => state.edit.cellId); const [sheetUpdating, setSheetUpdating] = useState(false) @@ -47,9 +47,9 @@ function EditPayrollNonEmployee() { const timer = () => { setTimeout(() => { if (window.payroll_non_employee_data) { - let rows = processPayrollData(window.payroll_non_employee_data) + let rows = processNonPayrollData(window.payroll_non_employee_data) dispatch({ - type: SET_CELLS, + type: SET_NON_EMPLOYEE_CELLS, cells: rows }) } else { diff --git a/front_end/src/Components/PayrollCellValue/index.jsx b/front_end/src/Components/PayrollCellValue/index.jsx new file mode 100644 index 000000000..a1940dde5 --- /dev/null +++ b/front_end/src/Components/PayrollCellValue/index.jsx @@ -0,0 +1,30 @@ +import React, {Fragment} from 'react' +import { useSelector } from 'react-redux' + +import { + formatValue +} from '../../Util' + +const PayrollCellValue = ({rowIndex, cellKey, format}) => { + console.log("PayrollCellValue component has been rendered"); + console.log("rowIndex: ", rowIndex); + console.log("cellKey: ", cellKey); + console.log("format: ", format); + const cell = useSelector(state => state.allCells.employeeCells[rowIndex][cellKey]); + + const getValue = (value) => { + if (format) { + return formatValue(parseInt(value, 10)/100) + } + + return value + } + + return ( + + {getValue(cell.value)} + + ) +} + +export default PayrollCellValue diff --git a/front_end/src/Components/PayrollNonCellValue/index.jsx b/front_end/src/Components/PayrollNonCellValue/index.jsx new file mode 100644 index 000000000..ad6f0f337 --- /dev/null +++ b/front_end/src/Components/PayrollNonCellValue/index.jsx @@ -0,0 +1,30 @@ +import React, {Fragment} from 'react' +import { useSelector } from 'react-redux' + +import { + formatValue +} from '../../Util' + +const PayrollNonCellValue = ({rowIndex, cellKey, format}) => { + console.log("PayrollNonCellValue component has been rendered"); + console.log("rowIndex: ", rowIndex); + console.log("cellKey: ", cellKey); + console.log("format: ", format); + const cell = useSelector(state => state.allCells.nonEmployeeCells[rowIndex][cellKey]); + + const getValue = (value) => { + if (format) { + return formatValue(parseInt(value, 10)/100) + } + + return value + } + + return ( + + {getValue(cell.value)} + + ) +} + +export default PayrollNonCellValue diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx index 1204ef774..e1546d984 100644 --- a/front_end/src/Components/PayrollNonTable/index.jsx +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -1,7 +1,7 @@ import React, {Fragment, memo } from 'react' import { useSelector, useDispatch } from 'react-redux' import { nanoid } from 'nanoid' -import PayrollTableCell from '../../Components/PayrollTableCell/index' +import PayrollNonTableCell from '../../Components/PayrollNonTableCell/index' import InfoCell from '../../Components/InfoCell/index' import CellValue from '../../Components/CellValue/index' import TableHeader from '../../Components/TableHeader/index' @@ -13,11 +13,12 @@ import { import { SET_EDITING_CELL } from '../../Reducers/Edit' import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' +import PayrollNonCellValue from "../PayrollNonCellValue/index.jsx"; function Table({rowData, sheetUpdating}) { const dispatch = useDispatch(); - const rows = useSelector(state => state.allCells.cells); + const rows = useSelector(state => state.allCells.nonEmployeeCells); const selectedRow = useSelector(state => state.selected.selectedRow); const allSelected = useSelector(state => state.selected.all); @@ -115,28 +116,28 @@ function Table({rowData, sheetUpdating}) { - + - + - + - + - + - + - + - + {Object.keys(window.payroll_non_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_non_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) @@ -145,7 +146,7 @@ function Table({rowData, sheetUpdating}) { Object.keys(monthValues).map((monthKey) => { const monthValue = monthValues[monthKey]; // Access the value for each month return ( - - + ); }) ); diff --git a/front_end/src/Components/PayrollNonTableCell/index.jsx b/front_end/src/Components/PayrollNonTableCell/index.jsx new file mode 100644 index 000000000..ab3f554af --- /dev/null +++ b/front_end/src/Components/PayrollNonTableCell/index.jsx @@ -0,0 +1,271 @@ +import React, {Fragment, useState, useEffect, memo } from 'react' +import { useSelector, useDispatch } from 'react-redux' +import { SET_EDITING_CELL } from '../../Reducers/Edit' +import { + formatValue, + postData, + processNonPayrollData + +} from '../../Util' +import { SET_ERROR } from '../../Reducers/Error' +import { SET_NON_EMPLOYEE_CELLS } from '../../Reducers/Cells' + +const NonPayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) => { + let editing = false + let isEditable = true + + const checkValue = (val) => { + if (cellId === val) { + editing = true + return false + } else if (editing) { + // Turn off editing + editing = false + return false + } + + return true + } + + let selectChanged = false + + const checkSelectRow = (selectedRow) => { + if (selectedRow === rowIndex) { + selectChanged = true + return false + } else if (selectChanged) { + selectChanged = false + return false + } + + return true + } + + const dispatch = useDispatch(); + + const cells = useSelector(state => state.allCells.nonEmployeeCells); + const cell = useSelector(state => state.allCells.nonEmployeeCells[rowIndex][cellKey]); + const editCellId = useSelector(state => state.edit.cellId, checkValue); + + const [isUpdating, setIsUpdating] = useState(false) + + const selectedRow = useSelector(state => state.selected.selectedRow, checkSelectRow); + const allSelected = useSelector(state => state.selected.all); + + + + // Check for actual + // if (window.payroll_monthly_data.indexOf(cellKey) > -1) { + // isEditable = false + // } + + const getValue = () => { + return cellValue + } + + const [value, setValue] = useState(getValue()) + + useEffect(() => { + if (cell) { + setValue(cell) + } + }, [cell]); + + const isSelected = () => { + if (allSelected) { + return true + } + + return selectedRow === rowIndex + } + + const wasEdited = () => { + return isEditable; + } + + const getClasses = () => { + let editable = '' + + if (!isEditable) { + editable = ' not-editable' + } + + if (!cell) + return "govuk-table__cell non-payroll-month-cell figure-cell " + (isSelected() ? 'selected' : '') + editable + + // let negative = '' + + // if (cell.amount < 0) { + // negative = " negative" + // } + + return "govuk-table__cell non-payroll-month-cell figure-cell " + (wasEdited() ? 'edited ' : '') + (isSelected() ? 'selected' : '') + editable + negative + } + + const setContentState = (value) => { + var re = /^-?\d*\.?\d{0,12}$/; + var isValid = (value.match(re) !== null); + + if (!isValid) { + return + } + setValue(value) + } + + + const updateValue = () => { + // setValue(value) + console.log('staff number:', cells[rowIndex]["staff_number"].value) + console.log('cell value:', value) + console.log('cell key:', cellKey) + + let newValue = 0 + + if (value > 1) { + newValue = 1 + } + + if (value < 0) { + newValue = 0 + } + + let intNewValue = parseInt(newValue, 10) + + if (getValue() === intNewValue) { + return + } + + setIsUpdating(true) + + let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value + + let payload = new FormData() + payload.append("staff_number", cells[rowIndex]["staff_number"].value) + payload.append("csrfmiddlewaretoken", crsfToken) + payload.append("month", cellKey) + payload.append("amount", getValue()) + + postData( + `/payroll/paste-payroll/${window.cost_centre}/${window.financial_year}`, + payload + ).then((response) => { + setIsUpdating(false) + if (response.status === 200) { + let rows = processNonPayrollData(response.data) + dispatch({ + type: SET_NON_EMPLOYEE_CELLS, + cells: rows + }) + } else { + dispatch( + SET_ERROR({ + errorMessage: response.data.error + }) + ); + } + }) + } + + const handleBlur = (event) => { + updateValue() + dispatch( + SET_EDITING_CELL({ + "cellId": null + }) + ) + } + + const handleKeyDown = (event) => { + if (event.key === "Tab") { + updateValue() + } + } + + const handleKeyPress = (event) => { + if(event.key === 'Enter') { + updateValue() + event.preventDefault() + } + } + + const getId = () => { + if (!cell) + return + + if (isUpdating) { + return cellId + "_updating" + } + + return cellId + } + + const isCellUpdating = () => { + if (cell && !isEditable) + return false + + if (isUpdating) + return true + + if (sheetUpdating && isSelected()) { + return true + } + + return false + } + + const getCellContent = () => { + if (isCellUpdating()) { + return ( + + UPDATING... + + ) + } else { + if (editCellId === cellId) { + return ( + input && input.focus() } + id={cellId + "_input"} + className="cell-input" + type="text" + value={value} + onChange={e => setContentState(e.target.value)} + onKeyPress={handleKeyPress} + onKeyDown={handleKeyDown} + onBlur={handleBlur} + /> + ) + } else { + return {formatValue(getValue())} + } + } + } + + return ( + + { + if (isSelected()) { + dispatch( + SET_EDITING_CELL({ + "cellId": cellId + }) + ); + } + }} + > + {getCellContent()} + + + ); +} + +const comparisonFn = function(prevProps, nextProps) { + return ( + prevProps.sheetUpdating === nextProps.sheetUpdating + ) +}; + +export default memo(NonPayrollTableCell, comparisonFn); diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index 8ca10ce52..c581f8c14 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -3,7 +3,6 @@ import { useSelector, useDispatch } from 'react-redux' import { nanoid } from 'nanoid' import PayrollTableCell from '../../Components/PayrollTableCell/index' import InfoCell from '../../Components/InfoCell/index' -import CellValue from '../../Components/CellValue/index' import TableHeader from '../../Components/TableHeader/index' import ToggleCell from '../../Components/ToggleCell/index' import ActualsHeaderRow from '../../Components/ActualsHeaderRow/index' @@ -13,11 +12,12 @@ import { import { SET_EDITING_CELL } from '../../Reducers/Edit' import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' +import PayrollCellValue from "../PayrollCellValue/index.jsx"; function Table({rowData, sheetUpdating}) { const dispatch = useDispatch(); - const rows = useSelector(state => state.allCells.cells); + const rows = useSelector(state => state.allCells.employeeCells); const selectedRow = useSelector(state => state.selected.selectedRow); const allSelected = useSelector(state => state.selected.all); @@ -115,28 +115,28 @@ function Table({rowData, sheetUpdating}) { - + - + - + - + - + - + - + - + {Object.keys(window.payroll_employee_monthly_data).map((dataKey, index) => { const monthValues = window.payroll_employee_monthly_data[dataKey]; // Access the month object (e.g., { "apr": 1, "may": 1, ... }) diff --git a/front_end/src/Components/PayrollTableCell/index.jsx b/front_end/src/Components/PayrollTableCell/index.jsx index 83ea08067..9a23fd8ec 100644 --- a/front_end/src/Components/PayrollTableCell/index.jsx +++ b/front_end/src/Components/PayrollTableCell/index.jsx @@ -4,11 +4,11 @@ import { SET_EDITING_CELL } from '../../Reducers/Edit' import { formatValue, postData, - processForecastData, processPayrollData + processPayrollData } from '../../Util' import { SET_ERROR } from '../../Reducers/Error' -import { SET_CELLS } from '../../Reducers/Cells' +import { SET_EMPLOYEE_CELLS } from '../../Reducers/Cells' const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) => { @@ -44,8 +44,8 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) const dispatch = useDispatch(); - const cells = useSelector(state => state.allCells.cells); - const cell = useSelector(state => state.allCells.cells[rowIndex][cellKey]); + const cells = useSelector(state => state.allCells.employeeCells); + const cell = useSelector(state => state.allCells.employeeCells[rowIndex][cellKey]); const editCellId = useSelector(state => state.edit.cellId, checkValue); const [isUpdating, setIsUpdating] = useState(false) @@ -154,7 +154,7 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) if (response.status === 200) { let rows = processPayrollData(response.data) dispatch({ - type: SET_CELLS, + type: SET_EMPLOYEE_CELLS, cells: rows }) } else { diff --git a/front_end/src/Reducers/Cells.js b/front_end/src/Reducers/Cells.js index 5660f6a35..ccf86b750 100644 --- a/front_end/src/Reducers/Cells.js +++ b/front_end/src/Reducers/Cells.js @@ -1,16 +1,29 @@ export const SET_CELLS = 'SET_CELLS'; +export const SET_EMPLOYEE_CELLS = 'SET_EMPLOYEE_CELLS'; +export const SET_NON_EMPLOYEE_CELLS = 'SET_NON_EMPLOYEE_CELLS'; const cellsInitial = { - cells: [] + cells: [], + employeeCells: [], + nonEmployeeCells: [] }; + export const allCells = (state = cellsInitial, action) => { switch (action.type) { case SET_CELLS: return Object.assign({}, state, { cells: action.cells }); + case SET_EMPLOYEE_CELLS: + return Object.assign({}, state, { + employeeCells: action.cells + }); + case SET_NON_EMPLOYEE_CELLS: + return Object.assign({}, state, { + nonEmployeeCells: action.cells + }); default: return state; } -} \ No newline at end of file +} diff --git a/front_end/src/Store.js b/front_end/src/Store.js index aa6dec222..7bdfeef40 100644 --- a/front_end/src/Store.js +++ b/front_end/src/Store.js @@ -2,7 +2,7 @@ import { createStore, combineReducers } from 'redux'; // import { persistReducer } from 'redux-persist'; // import storage from 'redux-persist/lib/storage'; -import { allCells } from './Reducers/Cells'; +import {allCells} from './Reducers/Cells'; import selected from './Reducers/Selected'; import edit from './Reducers/Edit'; import error from './Reducers/Error'; diff --git a/front_end/src/Util.js b/front_end/src/Util.js index fe1079370..1e2f26d42 100644 --- a/front_end/src/Util.js +++ b/front_end/src/Util.js @@ -64,6 +64,44 @@ export async function postData(url = '', data = {}) { } +export const processNonPayrollData = (payrollData) => { + console.log('payrollNonData data', payrollData) + let nonEmployeePayrollCols = [ + "name", + "grade", + "staff_number", + "fte", + "programme_code", + "budget_type", + "eu_non_eu", + "assignment_status", + ] + + let rows = []; + let colIndex = 0 + + payrollData.forEach(function (rowData, rowIndex) { + let cells = {} + + // eslint-disable-next-line + for (const nonEmployeePayrollCol of nonEmployeePayrollCols) { + cells[nonEmployeePayrollCol] = { + rowIndex: rowIndex, + colIndex: colIndex, + key: nonEmployeePayrollCol, + value: rowData[nonEmployeePayrollCol], + isEditable: false + } + colIndex++ + } + + rows.push(cells) + }); + + console.log('processed payroll non employee rows data', rows) + return rows; +} + export const processPayrollData = (payrollData) => { console.log('payrollData data', payrollData) let employeePayrollCols = [ @@ -77,7 +115,6 @@ export const processPayrollData = (payrollData) => { "assignment_status", ] - let rows = []; let colIndex = 0 @@ -100,8 +137,7 @@ export const processPayrollData = (payrollData) => { rows.push(cells) }); - - console.log('rows data', rows) + console.log('processed payroll employee rows data', rows) return rows; } diff --git a/payroll/templates/payroll/edit/edit.html b/payroll/templates/payroll/edit/edit.html index fa9fa11e8..fdda608cc 100644 --- a/payroll/templates/payroll/edit/edit.html +++ b/payroll/templates/payroll/edit/edit.html @@ -18,24 +18,32 @@ -
-
- {% csrf_token %} - {{ paste_form }} -
+
+ +
+
+
+ +
+ +
+
+
-
{% csrf_token %} {{ paste_form }}
+ {% endblock %} {% block scripts %} From 97ec28524061c651b9a9279cae432bdaa6b6d97c Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 14 Sep 2024 22:49:51 +0100 Subject: [PATCH 16/23] chore:add separate selected row states for employee and non-employee tables This update introduces distinct states for selected rows in employee and non-employee tables. The reducers and components have been updated to handle these new states, ensuring that selecting a row in one table type does not affect the selection in the other. This change improves the data handling and user interface clarity. --- .../src/Components/PayrollNonTable/index.jsx | 13 ++++++------- .../src/Components/PayrollNonTableCell/index.jsx | 2 +- front_end/src/Components/PayrollTable/index.jsx | 12 ++++++------ .../src/Components/PayrollTableCell/index.jsx | 6 +----- front_end/src/Reducers/Selected.js | 16 +++++++++++++++- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx index e1546d984..41e7b3f16 100644 --- a/front_end/src/Components/PayrollNonTable/index.jsx +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -3,7 +3,6 @@ import { useSelector, useDispatch } from 'react-redux' import { nanoid } from 'nanoid' import PayrollNonTableCell from '../../Components/PayrollNonTableCell/index' import InfoCell from '../../Components/InfoCell/index' -import CellValue from '../../Components/CellValue/index' import TableHeader from '../../Components/TableHeader/index' import ToggleCell from '../../Components/ToggleCell/index' import ActualsHeaderRow from '../../Components/ActualsHeaderRow/index' @@ -12,7 +11,7 @@ import { } from '../../Util' import { SET_EDITING_CELL } from '../../Reducers/Edit' -import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' +import { SELECT_ALL, UNSELECT_ALL, SET_NON_EMPLOYEE_SELECTED_ROW } from '../../Reducers/Selected' import PayrollNonCellValue from "../PayrollNonCellValue/index.jsx"; @@ -20,7 +19,7 @@ function Table({rowData, sheetUpdating}) { const dispatch = useDispatch(); const rows = useSelector(state => state.allCells.nonEmployeeCells); - const selectedRow = useSelector(state => state.selected.selectedRow); + const selectedRow = useSelector(state => state.selected.nonEmployeeSelectedRow); const allSelected = useSelector(state => state.selected.all); return ( @@ -95,14 +94,14 @@ function Table({rowData, sheetUpdating}) { ) if (selectedRow === rowIndex) { dispatch( - SET_SELECTED_ROW({ - selectedRow: null + SET_NON_EMPLOYEE_SELECTED_ROW({ + nonEmployeeSelectedRow: null }) ) } else { dispatch( - SET_SELECTED_ROW({ - selectedRow: rowIndex + SET_NON_EMPLOYEE_SELECTED_ROW({ + nonEmployeeSelectedRow: rowIndex }) ) } diff --git a/front_end/src/Components/PayrollNonTableCell/index.jsx b/front_end/src/Components/PayrollNonTableCell/index.jsx index ab3f554af..2b0faaa38 100644 --- a/front_end/src/Components/PayrollNonTableCell/index.jsx +++ b/front_end/src/Components/PayrollNonTableCell/index.jsx @@ -49,7 +49,7 @@ const NonPayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValu const [isUpdating, setIsUpdating] = useState(false) - const selectedRow = useSelector(state => state.selected.selectedRow, checkSelectRow); + const selectedRow = useSelector(state => state.selected.nonEmployeeSelectedRow, checkSelectRow); const allSelected = useSelector(state => state.selected.all); diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index c581f8c14..d396b2677 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -11,7 +11,7 @@ import { } from '../../Util' import { SET_EDITING_CELL } from '../../Reducers/Edit' -import { SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL } from '../../Reducers/Selected' +import {SET_SELECTED_ROW, SELECT_ALL, UNSELECT_ALL, SET_EMPLOYEE_SELECTED_ROW} from '../../Reducers/Selected' import PayrollCellValue from "../PayrollCellValue/index.jsx"; @@ -19,7 +19,7 @@ function Table({rowData, sheetUpdating}) { const dispatch = useDispatch(); const rows = useSelector(state => state.allCells.employeeCells); - const selectedRow = useSelector(state => state.selected.selectedRow); + const selectedRow = useSelector(state => state.selected.employeeSelectedRow); const allSelected = useSelector(state => state.selected.all); return ( @@ -94,14 +94,14 @@ function Table({rowData, sheetUpdating}) { ) if (selectedRow === rowIndex) { dispatch( - SET_SELECTED_ROW({ - selectedRow: null + SET_EMPLOYEE_SELECTED_ROW({ + employeeSelectedRow: null }) ) } else { dispatch( - SET_SELECTED_ROW({ - selectedRow: rowIndex + SET_EMPLOYEE_SELECTED_ROW({ + employeeSelectedRow: rowIndex }) ) } diff --git a/front_end/src/Components/PayrollTableCell/index.jsx b/front_end/src/Components/PayrollTableCell/index.jsx index 9a23fd8ec..fbe78a8a1 100644 --- a/front_end/src/Components/PayrollTableCell/index.jsx +++ b/front_end/src/Components/PayrollTableCell/index.jsx @@ -50,7 +50,7 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) const [isUpdating, setIsUpdating] = useState(false) - const selectedRow = useSelector(state => state.selected.selectedRow, checkSelectRow); + const selectedRow = useSelector(state => state.selected.employeeSelectedRow, checkSelectRow); const allSelected = useSelector(state => state.selected.all); @@ -116,10 +116,6 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) const updateValue = () => { // setValue(value) - console.log('staff number:', cells[rowIndex]["staff_number"].value) - console.log('cell value:', value) - console.log('cell key:', cellKey) - let newValue = 0 if (value > 1) { diff --git a/front_end/src/Reducers/Selected.js b/front_end/src/Reducers/Selected.js index a4aec3c31..417152312 100644 --- a/front_end/src/Reducers/Selected.js +++ b/front_end/src/Reducers/Selected.js @@ -6,6 +6,8 @@ const selected = createSlice({ slice: 'select', initialState: { selectedRow: -1, + employeeSelectedRow: -1, + nonEmployeeSelectedRow: -1, all: false }, reducers: { @@ -21,13 +23,25 @@ const selected = createSlice({ state.all = false state.selectedRow = -1 }, + SET_EMPLOYEE_SELECTED_ROW: (state, action) => { + state.all = false + state.nonEmployeeSelectedRow = -1 + state.employeeSelectedRow = action.payload.employeeSelectedRow + }, + SET_NON_EMPLOYEE_SELECTED_ROW: (state, action) => { + state.all = false + state.nonEmployeeSelectedRow = action.payload.nonEmployeeSelectedRow + state.employeeSelectedRow = -1 + } } }); export const { SET_SELECTED_ROW, SELECT_ALL, - UNSELECT_ALL + UNSELECT_ALL, + SET_EMPLOYEE_SELECTED_ROW, + SET_NON_EMPLOYEE_SELECTED_ROW, } = selected.actions; export default selected.reducer; From f54a498c34b3158547ddbd706bc328cac09b8f77 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 14 Sep 2024 22:55:44 +0100 Subject: [PATCH 17/23] update:selected reducer to handle specific row selections Added `nonEmployeeSelectedRow` and `employeeSelectedRow` resets in `SELECT_ALL` and `UNSELECT_ALL` actions to ensure proper state management for different row types. This change enhances the clarity of the selected row handling mechanism. --- front_end/src/Reducers/Selected.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/front_end/src/Reducers/Selected.js b/front_end/src/Reducers/Selected.js index 417152312..9674d9702 100644 --- a/front_end/src/Reducers/Selected.js +++ b/front_end/src/Reducers/Selected.js @@ -18,10 +18,14 @@ const selected = createSlice({ SELECT_ALL: (state, action) => { state.all = true state.selectedRow = -1 + state.nonEmployeeSelectedRow = -1 + state.employeeSelectedRow = -1 }, UNSELECT_ALL: (state, action) => { state.all = false state.selectedRow = -1 + state.nonEmployeeSelectedRow = -1 + state.employeeSelectedRow = -1 }, SET_EMPLOYEE_SELECTED_ROW: (state, action) => { state.all = false From 2e8e24dbfe25ad7734dd4dac1992b88d20ab59be Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sat, 14 Sep 2024 22:56:14 +0100 Subject: [PATCH 18/23] chore:add initial HR and Payroll models Introduce initial migrations and models for HR and Payroll apps. This includes creating models for EmployeePayroll, ForecastPayroll, HR, and NonEmployeePayroll, along with functionalities for parsing CSV data from S3 and updating records. --- hr/__init__.py | 0 hr/admin.py | 3 + hr/apps.py | 6 ++ hr/migrations/__init__.py | 0 hr/models.py | 137 +++++++++++++++++++++++++++++ hr/tests.py | 3 + hr/views.py | 3 + payroll/migrations/0001_initial.py | 133 ++++++++++++++++++++++++++++ 8 files changed, 285 insertions(+) create mode 100644 hr/__init__.py create mode 100644 hr/admin.py create mode 100644 hr/apps.py create mode 100644 hr/migrations/__init__.py create mode 100644 hr/models.py create mode 100644 hr/tests.py create mode 100644 hr/views.py create mode 100644 payroll/migrations/0001_initial.py diff --git a/hr/__init__.py b/hr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hr/admin.py b/hr/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/hr/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/hr/apps.py b/hr/apps.py new file mode 100644 index 000000000..00228a6a0 --- /dev/null +++ b/hr/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class HrConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'hr' diff --git a/hr/migrations/__init__.py b/hr/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hr/models.py b/hr/models.py new file mode 100644 index 000000000..fc4318d40 --- /dev/null +++ b/hr/models.py @@ -0,0 +1,137 @@ +import csv +from io import StringIO + +import boto3 +from django.db import models +from payroll.models import PayrollLookup, PayrollEntry + + +class HRModel(models.Model): + group = models.CharField(max_length=255) + directorate = models.CharField(max_length=255) + cc = models.CharField(max_length=255) + cc_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + first_name = models.CharField(max_length=255) + se_no = models.CharField(max_length=50) + salary = models.DecimalField(max_digits=10, decimal_places=2) + grade = models.CharField(max_length=10) + employee_location_city_name = models.CharField(max_length=255) + person_type = models.CharField(max_length=255) + assignment_status = models.CharField(max_length=255) + appointment_status = models.CharField(max_length=255) + working_hours = models.DecimalField(max_digits=5, decimal_places=2) + fte = models.DecimalField(max_digits=4, decimal_places=2) # Full-Time Equivalent + wmi_person = models.CharField(max_length=255, blank=True, null=True) + wmi = models.CharField(max_length=255, blank=True, null=True) + actual_group = models.CharField(max_length=255) + basic_pay = models.DecimalField(max_digits=10, decimal_places=2) + superannuation = models.DecimalField(max_digits=10, decimal_places=2) + ernic = models.DecimalField(max_digits=10, decimal_places=2) # Employer's National Insurance Contribution + total = models.DecimalField(max_digits=10, decimal_places=2) + costing_cc = models.CharField(max_length=255) + return_field = models.CharField(max_length=255) # Assuming 'Return' is a field name; rename if necessary + programme_code = models.CharField(max_length=255, blank=True, null=True) + + def __str__(self): + return f"{self.first_name} {self.last_name} ({self.se_no})" + + def parse_csv(self, bucket_name: str, file_path: str): + try: + # Initialize S3 client + s3 = boto3.client('s3') + + # Get the file from S3 + s3_object = s3.get_object(Bucket=bucket_name, Key=file_path) + + # Read the file content + file_content = s3_object['Body'].read().decode('utf-8-sig') + + # Use StringIO to read the content as a CSV + file = StringIO(file_content) + reader = csv.DictReader(file) + + for row in reader: + HRModel.objects.create( + group=row['group'], + directorate=row['directorate'], + cc=row['cc'], + cc_name=row['cc_name'], + last_name=row['last_name'], + first_name=row['first_name'], + se_no=row['se_no'], + salary=row['salary'], + grade=row['grade'], + employee_location_city_name=row['employee_location_city_name'], + person_type=row['person_type'], + assignment_status=row['assignment_status'], + appointment_status=row['appointment_status'], + working_hours=row['working_hours'], + fte=row['fte'], + wmi_person=row.get('wmi_person', ''), + wmi=row.get('wmi', ''), + actual_group=row['actual_group'], + basic_pay=row['basic_pay'], + superannuation=row['superannuation'], + ernic=row['ernic'], + total=row['total'], + costing_cc=row['costing_cc'], + return_field=row['return_field'], + programme_code=row['306162 Code'] + ) + except Exception as e: + raise e + + def update_records_with_basic_pay_superannuation_ernic_values(self): + payroll_records = PayrollEntry.objects.all() + + # If payroll_records is empty, return + if not payroll_records: + return + + hr_records = HRModel.objects.all() + lookup = PayrollLookup() + + # For each HR record get the total debit amount (debit - credit) and then + # get pay_element_name from the record and use it to get the tool type payment + # using lookup.get_tool_type_payment(pay_element_name) + # If the tool type payment is 'Basic Pay' then add the total debit amount to the basic_pay field of the record + # If the tool type payment is 'Superannuation' then add the total debit amount to the superannuation field of the record + # If the tool type payment is 'ERNIC' then add the total debit amount to the ernic field of the record + # If the tool type payment is not found then log the pay_element_name as not found + for hr_record in hr_records: + # Iterate through the payroll records and find employee_number that matches staff_employee_number + current_payroll_record = None + for payroll_record in payroll_records: + if (payroll_record.employee_number == hr_record.se_no and + payroll_record.cost_center_number == hr_record.cc): + current_payroll_record = payroll_record + break + + if current_payroll_record is None: + continue + + total_debit = current_payroll_record.debit_amount - current_payroll_record.credit_amount + tool_type_payment = lookup.get_tool_type_payment(current_payroll_record.pay_element_name).lower() + + if tool_type_payment == 'basic pay': + hr_record.basic_pay += total_debit + elif tool_type_payment == 'superannuation': + hr_record.superannuation += total_debit + elif tool_type_payment == 'ernic': + hr_record.ernic += total_debit + else: + # log the pay_element_name as not found + pass + + hr_record.save() + + # For each HR record get the record.basic_pay and set record.wmi_person to 'Yes' if basic_pay is greater than 0 + # and set record.wmi_person to 'No' if basic_pay is less than or equal to 0 + for hr_record in hr_records: + hr_record.wmi_person = 'payroll' if hr_record.basic_pay > 0 else 'nonpayroll' + hr_record.save() + + class Meta: + verbose_name = "HR" + verbose_name_plural = "HR Records" \ No newline at end of file diff --git a/hr/tests.py b/hr/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/hr/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/hr/views.py b/hr/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/hr/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py new file mode 100644 index 000000000..dd4ca610e --- /dev/null +++ b/payroll/migrations/0001_initial.py @@ -0,0 +1,133 @@ +# Generated by Django 5.1 on 2024-09-12 22:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='EmployeePayroll', + fields=[ + ('id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('name', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('staff_number', models.CharField(max_length=100)), + ('fte', models.DecimalField(decimal_places=2, max_digits=5)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ('eu_non_eu', models.CharField(max_length=100)), + ('assignment_status', models.CharField(max_length=100)), + ('current_month', models.CharField(max_length=100)), + ('current_year', models.CharField(max_length=100)), + ('apr', models.IntegerField(verbose_name='april')), + ('may', models.IntegerField(verbose_name='may')), + ('jun', models.IntegerField(verbose_name='june')), + ('jul', models.IntegerField(verbose_name='july')), + ('aug', models.IntegerField(verbose_name='august')), + ('sep', models.IntegerField(verbose_name='september')), + ('oct', models.IntegerField(verbose_name='october')), + ('nov', models.IntegerField(verbose_name='november')), + ('dec', models.IntegerField(verbose_name='december')), + ('jan', models.IntegerField(verbose_name='january')), + ('feb', models.IntegerField(verbose_name='february')), + ('mar', models.IntegerField(verbose_name='march')), + ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), + ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), + ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ForcastPayroll', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('nac', models.CharField(max_length=100)), + ('nac_description', models.CharField(max_length=100)), + ('project_code', models.CharField(max_length=100)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='HR', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('group', models.CharField(max_length=255)), + ('directorate', models.CharField(max_length=255)), + ('cc', models.CharField(max_length=255)), + ('cc_name', models.CharField(max_length=255)), + ('last_name', models.CharField(max_length=255)), + ('first_name', models.CharField(max_length=255)), + ('se_no', models.CharField(max_length=50)), + ('salary', models.DecimalField(decimal_places=2, max_digits=10)), + ('grade', models.CharField(max_length=10)), + ('employee_location_city_name', models.CharField(max_length=255)), + ('person_type', models.CharField(max_length=255)), + ('assignment_status', models.CharField(max_length=255)), + ('appointment_status', models.CharField(max_length=255)), + ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), + ('fte', models.DecimalField(decimal_places=2, max_digits=4)), + ('wmi_person', models.CharField(blank=True, max_length=255, null=True)), + ('wmi', models.CharField(blank=True, max_length=255, null=True)), + ('actual_group', models.CharField(max_length=255)), + ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), + ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), + ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ('total', models.DecimalField(decimal_places=2, max_digits=10)), + ('costing_cc', models.CharField(max_length=255)), + ('return_field', models.CharField(max_length=255)), + ('programme_code', models.CharField(blank=True, max_length=255, null=True)), + ], + options={ + 'verbose_name': 'HR', + 'verbose_name_plural': 'HR Records', + }, + ), + migrations.CreateModel( + name='NonEmployeePayroll', + fields=[ + ('id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('name', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('staff_number', models.CharField(max_length=100)), + ('fte', models.DecimalField(decimal_places=2, max_digits=5)), + ('programme_code', models.CharField(max_length=100)), + ('budget_type', models.CharField(max_length=100)), + ('eu_non_eu', models.CharField(max_length=100)), + ('assignment_status', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('current_month', models.CharField(max_length=100)), + ('current_year', models.CharField(max_length=100)), + ('apr', models.IntegerField(verbose_name='april')), + ('may', models.IntegerField(verbose_name='may')), + ('jun', models.IntegerField(verbose_name='june')), + ('jul', models.IntegerField(verbose_name='july')), + ('aug', models.IntegerField(verbose_name='august')), + ('sep', models.IntegerField(verbose_name='september')), + ('oct', models.IntegerField(verbose_name='october')), + ('nov', models.IntegerField(verbose_name='november')), + ('dec', models.IntegerField(verbose_name='december')), + ('jan', models.IntegerField(verbose_name='january')), + ('feb', models.IntegerField(verbose_name='february')), + ('mar', models.IntegerField(verbose_name='march')), + ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), + ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), + ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ], + options={ + 'abstract': False, + }, + ), + ] From 4f131ac5ee973887b9ef5b69e396881c6fee07fe Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sun, 15 Sep 2024 01:09:13 +0100 Subject: [PATCH 19/23] refactor:payroll code to improve data processing Updated payroll table and cell components to streamline ID generation and data validation logic. Improved processing of payroll data in backend and removed unnecessary console logs. Refactored form handling in Django to simplify data cleaning. --- .../src/Components/PayrollCellValue/index.jsx | 4 - .../Components/PayrollNonCellValue/index.jsx | 4 - .../src/Components/PayrollNonTable/index.jsx | 2 +- .../Components/PayrollNonTableCell/index.jsx | 56 +++++------ .../src/Components/PayrollTable/index.jsx | 2 +- .../src/Components/PayrollTableCell/index.jsx | 53 +++++----- payroll/edit_payroll.py | 99 +++++++++++++++++-- payroll/forms.py | 11 ++- 8 files changed, 153 insertions(+), 78 deletions(-) diff --git a/front_end/src/Components/PayrollCellValue/index.jsx b/front_end/src/Components/PayrollCellValue/index.jsx index a1940dde5..23f5f08e0 100644 --- a/front_end/src/Components/PayrollCellValue/index.jsx +++ b/front_end/src/Components/PayrollCellValue/index.jsx @@ -6,10 +6,6 @@ import { } from '../../Util' const PayrollCellValue = ({rowIndex, cellKey, format}) => { - console.log("PayrollCellValue component has been rendered"); - console.log("rowIndex: ", rowIndex); - console.log("cellKey: ", cellKey); - console.log("format: ", format); const cell = useSelector(state => state.allCells.employeeCells[rowIndex][cellKey]); const getValue = (value) => { diff --git a/front_end/src/Components/PayrollNonCellValue/index.jsx b/front_end/src/Components/PayrollNonCellValue/index.jsx index ad6f0f337..be1b77dfc 100644 --- a/front_end/src/Components/PayrollNonCellValue/index.jsx +++ b/front_end/src/Components/PayrollNonCellValue/index.jsx @@ -6,10 +6,6 @@ import { } from '../../Util' const PayrollNonCellValue = ({rowIndex, cellKey, format}) => { - console.log("PayrollNonCellValue component has been rendered"); - console.log("rowIndex: ", rowIndex); - console.log("cellKey: ", cellKey); - console.log("format: ", format); const cell = useSelector(state => state.allCells.nonEmployeeCells[rowIndex][cellKey]); const getValue = (value) => { diff --git a/front_end/src/Components/PayrollNonTable/index.jsx b/front_end/src/Components/PayrollNonTable/index.jsx index 41e7b3f16..50fc4a3da 100644 --- a/front_end/src/Components/PayrollNonTable/index.jsx +++ b/front_end/src/Components/PayrollNonTable/index.jsx @@ -148,7 +148,7 @@ function Table({rowData, sheetUpdating}) { { let editing = false @@ -103,33 +100,28 @@ const NonPayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValu } const setContentState = (value) => { - var re = /^-?\d*\.?\d{0,12}$/; - var isValid = (value.match(re) !== null); - - if (!isValid) { - return - } + // var re = /^-?\d*\.?\d{0,12}$/; + // var isValid = (value.match(re) !== null); + // + // if (!isValid) { + // return + // } setValue(value) } const updateValue = () => { - // setValue(value) - console.log('staff number:', cells[rowIndex]["staff_number"].value) - console.log('cell value:', value) - console.log('cell key:', cellKey) - - let newValue = 0 - - if (value > 1) { - newValue = 1 - } - - if (value < 0) { - newValue = 0 - } + setValue(value) + // let newValue = 0 + // if (value > 1) { + // newValue = 1 + // } + // + // if (value < 0) { + // newValue = 0 + // } - let intNewValue = parseInt(newValue, 10) + let intNewValue = parseInt(value, 10) if (getValue() === intNewValue) { return @@ -140,10 +132,11 @@ const NonPayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValu let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value let payload = new FormData() + payload.append("table", "non_payroll") payload.append("staff_number", cells[rowIndex]["staff_number"].value) payload.append("csrfmiddlewaretoken", crsfToken) payload.append("month", cellKey) - payload.append("amount", getValue()) + payload.append("amount", value) postData( `/payroll/paste-payroll/${window.cost_centre}/${window.financial_year}`, @@ -151,11 +144,14 @@ const NonPayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValu ).then((response) => { setIsUpdating(false) if (response.status === 200) { - let rows = processNonPayrollData(response.data) - dispatch({ - type: SET_NON_EMPLOYEE_CELLS, - cells: rows - }) + const month = response.data.month; + // const payroll = response.data.payroll; + window.payroll_non_employee_monthly_data = month; + // let rows = processNonPayrollData(payroll) + // dispatch({ + // type: SET_NON_EMPLOYEE_CELLS, + // cells: rows + // }) } else { dispatch( SET_ERROR({ diff --git a/front_end/src/Components/PayrollTable/index.jsx b/front_end/src/Components/PayrollTable/index.jsx index d396b2677..b5c335124 100644 --- a/front_end/src/Components/PayrollTable/index.jsx +++ b/front_end/src/Components/PayrollTable/index.jsx @@ -148,7 +148,7 @@ function Table({rowData, sheetUpdating}) { { - let editing = false let isEditable = true @@ -104,29 +100,28 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) } const setContentState = (value) => { - var re = /^-?\d*\.?\d{0,12}$/; - var isValid = (value.match(re) !== null); - - if (!isValid) { - return - } + // var re = /^-?\d*\.?\d{0,12}$/; + // var isValid = (value.match(re) !== null); + // + // if (!isValid) { + // return + // } setValue(value) } const updateValue = () => { - // setValue(value) - let newValue = 0 - - if (value > 1) { - newValue = 1 - } - - if (value < 0) { - newValue = 0 - } + setValue(value) + // let newValue = 0 + // if (value > 1) { + // newValue = 1 + // } + // + // if (value < 0) { + // newValue = 0 + // } - let intNewValue = parseInt(newValue, 10) + let intNewValue = parseInt(value, 10) if (getValue() === intNewValue) { return @@ -137,10 +132,11 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) let crsfToken = document.getElementsByName("csrfmiddlewaretoken")[0].value let payload = new FormData() + payload.append("table", "payroll") payload.append("staff_number", cells[rowIndex]["staff_number"].value) payload.append("csrfmiddlewaretoken", crsfToken) payload.append("month", cellKey) - payload.append("amount", getValue()) + payload.append("amount", value) postData( `/payroll/paste-payroll/${window.cost_centre}/${window.financial_year}`, @@ -148,11 +144,14 @@ const PayrollTableCell = ({rowIndex, cellId, cellKey, sheetUpdating, cellValue}) ).then((response) => { setIsUpdating(false) if (response.status === 200) { - let rows = processPayrollData(response.data) - dispatch({ - type: SET_EMPLOYEE_CELLS, - cells: rows - }) + const month = response.data.month; + // const payroll = response.data.payroll; + window.payroll_employee_monthly_data = month + // let rows = processPayrollData(response.data) + // dispatch({ + // type: SET_EMPLOYEE_CELLS, + // cells: rows + // }) } else { dispatch( SET_ERROR({ diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 2e71fd0a2..6c63332f8 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -30,15 +30,102 @@ def form_valid(self, form): # noqa: C901 try: cost_centre_code = self.kwargs["cost_centre_code"] - paste_content = form.cleaned_data["paste_content"] - pasted_at_row = form.cleaned_data.get("pasted_at_row", None) - all_selected = form.cleaned_data.get("all_selected", False) + paste_content = form.clean_pasted_at_row() + staff_employee = paste_content["staff_number"][0] + month_name = paste_content["month"][0] + amount_value = paste_content["amount"][0] + table = paste_content["table"][0] logger.info(f"cost_centre_code: {cost_centre_code}") logger.info(f"paste_content: {paste_content}") - logger.info(f"pasted_at_row: {pasted_at_row}") - logger.info(f"all_selected: {all_selected}") - return JsonResponse({"status": "success"}) + logger.info(f"month name: {month_name}") + logger.info(f"amount value: {amount_value}") + logger.info(f"staff_employee: {staff_employee}") + logger.info(f"table: {table}") + + # get the first item from the list from table data + if table == "non_payroll": + non_employee_payroll = NonEmployeePayroll.objects.get( + staff_number=staff_employee, + # cost_centre_code=cost_centre_code, + ) + if month_name == "jan": + non_employee_payroll.jan = int(amount_value) + if month_name == "feb": + non_employee_payroll.feb = int(amount_value) + if month_name == "mar": + non_employee_payroll.mar = int(amount_value) + if month_name == "apr": + non_employee_payroll.apr = int(amount_value) + if month_name == "may": + non_employee_payroll.may = int(amount_value) + if month_name == "jun": + non_employee_payroll.jun = int(amount_value) + if month_name == "jul": + non_employee_payroll.jul = int(amount_value) + if month_name == "aug": + non_employee_payroll.aug = int(amount_value) + if month_name == "sep": + non_employee_payroll.sep = int(amount_value) + if month_name == "oct": + non_employee_payroll.oct = int(amount_value) + if month_name == "nov": + non_employee_payroll.nov = int(amount_value) + if month_name == "dec": + non_employee_payroll.dec = int(amount_value) + + non_employee_payroll.save() + else: + employee_payroll = EmployeePayroll.objects.get( + staff_number=staff_employee, + # cost_centre_code=cost_centre_code, + ) + if month_name == "jan": + employee_payroll.jan = int(amount_value) + if month_name == "feb": + employee_payroll.feb = int(amount_value) + if month_name == "mar": + employee_payroll.mar = int(amount_value) + if month_name == "apr": + employee_payroll.apr = int(amount_value) + if month_name == "may": + employee_payroll.may = int(amount_value) + if month_name == "jun": + employee_payroll.jun = int(amount_value) + if month_name == "jul": + employee_payroll.jul = int(amount_value) + if month_name == "aug": + employee_payroll.aug = int(amount_value) + if month_name == "sep": + employee_payroll.sep = int(amount_value) + if month_name == "oct": + employee_payroll.oct = int(amount_value) + if month_name == "nov": + employee_payroll.nov = int(amount_value) + if month_name == "dec": + employee_payroll.dec = int(amount_value) + + employee_payroll.save() + + if table == "non_payroll": + get_all_non_employee_data = NonEmployeePayroll.objects.all() + non_payroll_monthly_serialiser = NonEmployeeMonthlyPayrollSerializer(get_all_non_employee_data, many=True) + non_payroll_serialiser = NonEmployeePayrollSerializer(get_all_non_employee_data, many=True) + + return JsonResponse({ + "month": non_payroll_monthly_serialiser.data, + "payroll": non_payroll_serialiser.data + }) + else: + get_all_employee_data = EmployeePayroll.objects.all() + payroll_monthly_serialiser = EmployeeMonthlyPayrollSerializer(get_all_employee_data, many=True) + payroll_serialiser = EmployeePayrollSerializer(get_all_employee_data, many=True) + + return JsonResponse({ + "month": payroll_monthly_serialiser.data, + "payroll": payroll_serialiser.data + }) + except Exception as e: logger.error(f"error parsing form data: {e}") return self.form_invalid(form) diff --git a/payroll/forms.py b/payroll/forms.py index 8492b95d1..c7b2c7dca 100644 --- a/payroll/forms.py +++ b/payroll/forms.py @@ -1,7 +1,10 @@ import json +import logging from django import forms +logger = logging.getLogger(__name__) + # Form for pasting HR data into the payroll table class PasteHRForm(forms.Form): all_selected = forms.BooleanField(widget=forms.HiddenInput(), required=False) @@ -15,14 +18,12 @@ class PasteHRForm(forms.Form): ) def clean_pasted_at_row(self): - data = self.cleaned_data["pasted_at_row"] + data = dict(self.data.lists()) if not data: return None try: - json_data = json.loads(data) + return data except json.JSONDecodeError: - raise forms.ValidationError("invalid row data supplied") - - return json_data \ No newline at end of file + raise forms.ValidationError("invalid row data supplied") \ No newline at end of file From 311968fcfa35b9d4ad69c4c136cff826b4b7ed93 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sun, 15 Sep 2024 23:50:46 +0100 Subject: [PATCH 20/23] chore:add detailed docstrings to payroll module Detailed docstrings have been added to various files within the payroll module including models, views, serializers, and forms. These docstrings provide comprehensive explanations of class attributes, methods, and overall functionality to improve code readability and maintainability. --- forecast/views/edit_forecast.py | 10 +- payroll/edit_payroll.py | 59 +++++++++-- payroll/forms.py | 11 ++ payroll/models.py | 169 +++++++++++++++++++++++++++++++ payroll/serialisers.py | 61 +++++++++++ payroll/services.py | 172 ++++++++++++++++++++++++++------ 6 files changed, 443 insertions(+), 39 deletions(-) diff --git a/forecast/views/edit_forecast.py b/forecast/views/edit_forecast.py index 2c3d91c2d..33c8c27cf 100644 --- a/forecast/views/edit_forecast.py +++ b/forecast/views/edit_forecast.py @@ -51,7 +51,7 @@ NoCostCentreCodeInURLError, NoFinancialYearInURLError, ) - +from payroll.services import get_forecast_basic_pay_for_employee UNAVAILABLE_FORECAST_EDIT_TITLE = "Forecast editing is locked" UNAVAILABLE_FUTURE_FORECAST_EDIT_TITLE = "Future forecast editing is locked" @@ -481,7 +481,15 @@ def get_context_data(self, **kwargs): ) serialiser_data = financial_code_serialiser.data + + data = get_forecast_basic_pay_for_employee(self.cost_centre_code, self.financial_year) + + # Add each item from data list to the top of the serialiser_data list + for item in data: + serialiser_data.insert(0, item) + forecast_dump = json.dumps(serialiser_data) + logger.info(f"forecast_dump data: {forecast_dump}") if self.financial_year == get_current_financial_year(): self.title = "Edit forecast" actual_data = ( diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 6c63332f8..7b3dc1b2b 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -21,6 +21,21 @@ class EditPayrollUpdatesView( CostCentrePermissionTest, FormView, ): + """ + EditPayrollUpdatesView is a class-based view in Django for handling the editing + of payroll updates through a form. It validates the form data and updates the + respective payroll records in the database. + + Attributes: + form_class: The form class associated with this view. + + Methods: + form_valid(form): + Validates the form data. If the form is valid, it updates the payroll + records based on the provided data. Handles both 'employee' and 'non_employee' + payroll updates. Returns a JsonResponse with serialized payroll data or + calls form_invalid if an error occurs. + """ form_class = PasteHRForm @transaction.atomic @@ -36,13 +51,6 @@ def form_valid(self, form): # noqa: C901 amount_value = paste_content["amount"][0] table = paste_content["table"][0] - logger.info(f"cost_centre_code: {cost_centre_code}") - logger.info(f"paste_content: {paste_content}") - logger.info(f"month name: {month_name}") - logger.info(f"amount value: {amount_value}") - logger.info(f"staff_employee: {staff_employee}") - logger.info(f"table: {table}") - # get the first item from the list from table data if table == "non_payroll": non_employee_payroll = NonEmployeePayroll.objects.get( @@ -134,6 +142,43 @@ class EditPayrollView( CostCentrePermissionTest, TemplateView, ): + """ + EditPayrollView class is a Django view that deals with rendering the payroll editing template + and providing payroll data for employees and non-employees. + + Attributes: + template_name: A string representing the template name used to render the view. + + Methods: + class_name: + Returns a string representing additional CSS class to be used in the template. + + cost_centre_details: + Retrieves details about the cost centre based on the cost centre code. + Returns a dictionary with cost centre details including group, directorate, and cost centre names and codes. + + get_employee_payroll_serialiser: + Retrieves all employee payroll data and serializes it. + Returns an instance of EmployeePayrollSerializer containing all employee payroll data. + + get_employee_payroll_monthly_serialiser: + Retrieves all employee monthly payroll data and serializes it. + Returns an instance of EmployeeMonthlyPayrollSerializer containing all employee monthly payroll data. + + get_non_employee_payroll_serialiser: + Retrieves all non-employee payroll data and serializes it. + Returns an instance of NonEmployeePayrollSerializer containing all non-employee payroll data. + + get_non_employee_payroll_monthly_serialiser: + Retrieves all non-employee monthly payroll data and serializes it. + Returns an instance of NonEmployeeMonthlyPayrollSerializer containing all non-employee monthly payroll data. + + get_context_data: + Assembles the context data for rendering the template. + Includes serialized payroll data for employees and non-employees, both regular and monthly data, + and a form instance for pasting HR data. + Returns a dictionary containing the context data to be passed to the template. + """ template_name = "payroll/edit/edit.html" def class_name(self): diff --git a/payroll/forms.py b/payroll/forms.py index c7b2c7dca..07054c366 100644 --- a/payroll/forms.py +++ b/payroll/forms.py @@ -7,6 +7,17 @@ # Form for pasting HR data into the payroll table class PasteHRForm(forms.Form): + """ + A form for handling pasted data in an HR system. + + Attributes: + all_selected: A hidden boolean field that indicates whether all rows are selected. + pasted_at_row: A hidden char field that indicates the row where the data is pasted. + paste_content: A hidden char field that holds the pasted content. + + Methods: + clean_pasted_at_row: Cleans and validates the pasted_at_row field's data. + """ all_selected = forms.BooleanField(widget=forms.HiddenInput(), required=False) pasted_at_row = forms.CharField( widget=forms.HiddenInput(), diff --git a/payroll/models.py b/payroll/models.py index 23b3d4f3a..6f743d181 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -4,6 +4,20 @@ from django.db import models class ForcastPayroll(models.Model): + """ + ForcastPayroll is a Django model representing the payroll forecast details. + + Attributes: + name: A CharField representing the name, with a maximum length of 100 characters. + nac: A CharField representing the NAC, with a maximum length of 100 characters. + nac_description: A CharField representing the description of the NAC, with a maximum length of 100 characters. + project_code: A CharField representing the project code, with a maximum length of 100 characters. + programme_code: A CharField representing the programme code, with a maximum length of 100 characters. + budget_type: A CharField representing the type of budget, with a maximum length of 100 characters. + + Meta: + abstract: Boolean flag indicating whether the model is abstract. Set to False. + """ name = models.CharField(max_length=100) nac = models.CharField(max_length=100) nac_description = models.CharField(max_length=100) @@ -15,7 +29,69 @@ class Meta: abstract = False class EmployeePayroll(models.Model): + """ + EmployeePayroll model stores payroll information for employees. + + Attributes: + id : str + Unique identifier for each employee. + name : str + Name of the employee. + grade : str + Grade of the employee. + staff_number : str + Staff number assigned to the employee. + fte : Decimal + Full-time equivalent for the employee. + programme_code : str + Code of the programme the employee is part of. + budget_type : str + Type of budget under which the employee is categorized. + eu_non_eu : str + Indicates whether the employee is from the EU or non-EU. + assignment_status : str + Current assignment status of the employee. + current_month : str + The current month. + current_year : str + The current year. + apr : int + Payroll data for April. + may : int + Payroll data for May. + jun : int + Payroll data for June. + jul : int + Payroll data for July. + aug : int + Payroll data for August. + sep : int + Payroll data for September. + oct : int + Payroll data for October. + nov : int + Payroll data for November. + dec : int + Payroll data for December. + jan : int + Payroll data for January. + feb : int + Payroll data for February. + mar : int + Payroll data for March. + basic_pay : Decimal + Basic pay of the employee. + superannuation : Decimal + Superannuation amount for the employee. + ernic : Decimal + Employer's National Insurance Contribution. + + Meta: + abstract : bool + Indicates whether the model is abstract. In this case, set to False. + """ id = models.CharField(max_length=100, unique=True, primary_key=True) + # cost_centre_code = models.CharField(max_length=100) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -46,7 +122,43 @@ class Meta: abstract = False class NonEmployeePayroll(models.Model): + """ + Class representing a non-employee payroll model. + + Attributes: + id (CharField): Unique identifier for the payroll entry, serving as the primary key. + name (CharField): Name of the individual. + grade (CharField): Grade classification of the individual. + staff_number (CharField): Unique staff number of the individual. + fte (DecimalField): Full-Time Equivalent value, with a maximum of 5 digits and 2 decimal places. + programme_code (CharField): Code indicating the associated programme. + budget_type (CharField): Type of budget associated with the payroll entry. + eu_non_eu (CharField): Indicator if the individual is from EU or non-EU. + assignment_status (CharField): Status of the individual's assignment. + person_type (CharField): Type of individual, e.g., contractor, consultant. + current_month (CharField): The current month of the payroll entry. + current_year (CharField): The current year of the payroll entry. + apr (IntegerField): Payroll data for the month of April. + may (IntegerField): Payroll data for the month of May. + jun (IntegerField): Payroll data for the month of June. + jul (IntegerField): Payroll data for the month of July. + aug (IntegerField): Payroll data for the month of August. + sep (IntegerField): Payroll data for the month of September. + oct (IntegerField): Payroll data for the month of October. + nov (IntegerField): Payroll data for the month of November. + dec (IntegerField): Payroll data for the month of December. + jan (IntegerField): Payroll data for the month of January. + feb (IntegerField): Payroll data for the month of February. + mar (IntegerField): Payroll data for the month of March. + basic_pay (DecimalField): Basic pay amount, with a maximum of 10 digits and 2 decimal places. + superannuation (DecimalField): Superannuation contribution amount, with a maximum of 10 digits and 2 decimal places. + ernic (DecimalField): Employer's National Insurance contribution amount, with a maximum of 10 digits and 2 decimal places. + + Meta: + abstract (bool): Indicates that this model is not abstract. + """ id = models.CharField(max_length=100, unique=True, primary_key=True) + # cost_centre_code = models.CharField(max_length=100) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -78,6 +190,45 @@ class Meta: abstract = False class PayrollEntry(models.Model): + """ + Represents a payroll entry in a Django model. + + Attributes: + - business_unit_number: Character field to store the business unit number. + - business_unit_name: Character field to store the business unit name. + - cost_center_number: Character field to store the cost center number. + - cost_center_name: Character field to store the cost center name. + - employee_name: Character field to store the employee name. + - employee_number: Character field to store the employee number. + - assignment_number: Character field to store the assignment number. + - payroll_name: Character field to store the payroll name. + - employee_organization: Character field to store the employee's organization. + - employee_location: Character field to store the employee's location. + - person_type: Character field to store the type of person (e.g., employee, contractor). + - employee_category: Character field to store the employee's category. + - assignment_type: Character field to store the type of assignment. + - position: Character field to store the position of the employee. + - grade: Character field to store the grade of the employee. + - account_code: Character field to store the account code. + - account_name: Character field to store the account name. + - pay_element_name: Character field to store the pay element name. + - effective_date: Character field to store the effective date. + - debit_amount: Character field to store the debit amount. + - credit_amount: Character field to store the credit amount. + - basic_pay: Character field to store the basic pay. + - superannuation: Character field to store the superannuation information. + - ernic: Character field to store the ERNIC information. + + Methods: + - parse_csv: Parses a CSV file from an S3 bucket or a local file path and creates PayrollEntry objects from its rows. + Parameters: + - bucket_name: S3 bucket name as a string. + - file_path: Path to the file in the S3 bucket or local file system. + Raises: + - Exception: If there is an error during parsing or saving the data. + + - delete_all_records: Deletes all PayrollEntry records from the database. + """ business_unit_number = models.CharField(max_length=100) business_unit_name = models.CharField(max_length=100) cost_center_number = models.CharField(max_length=100) @@ -162,6 +313,24 @@ def delete_all_records(): PayrollEntry.objects.all().delete() class PayrollLookup(): + """ + class PayrollLookup(): + + __init__(): + Initializes a new instance of the PayrollLookup class. + Loads the lookup table using the load_lookup_table method. + + load_lookup_table(): + Loads the lookup table from a CSV file named 'PayrollLookup.csv'. + Reads the file content and populates the lookup_table dictionary + with PayElementName as the key and ToolTypePayment as the value. + Raises an exception if there is an issue reading the file. + + get_tool_type_payment(pay_element_name: str) -> str: + Retrieves the ToolTypePayment corresponding to the given + pay_element_name from the lookup_table dictionary. + Returns "unknown" if the pay_element_name is not found. + """ def __init__(self): self.lookup_table = {} self.load_lookup_table() diff --git a/payroll/serialisers.py b/payroll/serialisers.py index 284dfb777..89c72afaf 100644 --- a/payroll/serialisers.py +++ b/payroll/serialisers.py @@ -4,6 +4,19 @@ class EmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): + """ + Serializer for EmployeeMonthlyPayroll to serialize and deserialize + EmployeePayroll model data for each month. + + class EmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): + class Meta: + model = EmployeePayroll + fields = [ + "apr", "may", "jun", "jul", "aug", "sep", + "oct", "nov", "dec", "jan", "feb", "mar" + ] + read_only_fields = fields + """ class Meta: model = EmployeePayroll fields = [ @@ -24,6 +37,15 @@ class Meta: class EmployeePayrollSerializer(serializers.ModelSerializer): + """ + Serializer for EmployeePayroll model. + + Attributes: + Meta (class): Meta options for the EmployeePayrollSerializer. + model (EmployeePayroll): Specifies the model to be serialized. + fields (list): List of fields to be included in the serialization. + read_only_fields (list): List of fields that should be read-only. + """ class Meta: model = EmployeePayroll fields = [ @@ -40,6 +62,29 @@ class Meta: class NonEmployeeMonthlyPayrollSerializer(serializers.ModelSerializer): + """ + NonEmployeeMonthlyPayrollSerializer is a ModelSerializer for the NonEmployeePayroll model. + + Meta class: + + - model: Specifies the model to be serialized, which is NonEmployeePayroll. + + - fields: Lists the fields to be included in the serialization. These fields represent the months of the fiscal year: + - "apr" + - "may" + - "jun" + - "jul" + - "aug" + - "sep" + - "oct" + - "nov" + - "dec" + - "jan" + - "feb" + - "mar" + + - read_only_fields: Specifies the fields that are read-only. In this case, all fields listed in 'fields' are set to read-only. + """ class Meta: model = NonEmployeePayroll fields = [ @@ -60,6 +105,22 @@ class Meta: class NonEmployeePayrollSerializer(serializers.ModelSerializer): + """ + NonEmployeePayrollSerializer is a ModelSerializer for the NonEmployeePayroll model. + + class Meta: + - model: Specifies the model to be serialized, which is NonEmployeePayroll. + - fields: Lists the fields to be included in the serialization. These are: + - name + - grade + - staff_number + - fte + - programme_code + - budget_type + - eu_non_eu + - assignment_status + - read_only_fields: Specifies the fields to be treated as read-only. In this case, it is set to the same fields listed in 'fields'. + """ class Meta: model = NonEmployeePayroll fields = [ diff --git a/payroll/services.py b/payroll/services.py index 02c82301b..1c740ab21 100644 --- a/payroll/services.py +++ b/payroll/services.py @@ -5,6 +5,16 @@ def update_employee_tables(): + """ + Fetches all records from the HRModel and updates the employee tables based on the `wmi_person` field. + + - If `wmi_person` equals 'payroll', a corresponding record is created in the `EmployeePayroll` table. + - If `wmi_person` equals 'nonpayroll', a corresponding record is created in the `NonEmployeePayroll` table. + - If a record does not exist in the target table, a new record is created. + + Attributes of each HR record such as `name`, `grade`, `staff_number`, `fte`, `programme_code`, assignment_status, + current_month, and current_year are used to populate the respective fields in the destination table. + """ hr_objects = HRModel.objects.all() # Loop through all objects and update the employee table based on the HR record.wmi_person @@ -50,8 +60,35 @@ def update_employee_tables(): non_employee_record.current_year = current_year non_employee_record.save() -def get_forecast_basic_pay_for_employee_non_employee_payroll() -> dict: - # Get all objects from EmployeePayroll table +def get_forecast_basic_pay_for_employee(cost_center_code, financial_year): + """ + Generates a forecast of basic pay, ERNIC, and superannuation for employees in a given cost center and financial year. + + Parameters: + cost_center_code (str): The code of the cost center to filter the employees. + financial_year (str): The financial year to filter the employee records. + + Returns: + list: A list of dictionaries, each representing a monthly forecast. Each dictionary contains: + - programme_description: The description of the programme. + - nac_description: The description of the natural account code. + - natural_account_code: The natural account code. + - programme: The programme type. + - cost_centre: The cost center code. + - analysis1_code: Analysis code 1. + - analysis2_code: Analysis code 2. + - project_code: The project code. + - monthly_figures: A list containing a dictionary with fields: + - actual: A boolean indicating if the figure is actual or forecasted. + - month: The month number. + - amount: The total amount for the month. + - starting_amount: The starting amount for the month. + - archived_status: Archive status (can be None). + - budget: The budget amount (set to 0 by default). + """ + # Get all objects from EmployeePayroll table where cost_center_code is equal to the cost_center_code + + # employee_objects = EmployeePayroll.objects.filter(cost_center_code=cost_center_code, current_year=financial_year) employee_objects = EmployeePayroll.objects.all() # Get all objects from NonEmployeePayroll table @@ -63,39 +100,112 @@ def get_forecast_basic_pay_for_employee_non_employee_payroll() -> dict: # - total (sum all basic_pay values for that month) # Initialize lists - employee_list = [] - non_employee_list = [] - - # Define the months - months = [ - 'april', 'may', 'june', 'july', 'august', - 'september', 'october', 'november', 'december', - 'january', 'february', 'march' - ] + forecast_months = [] # Use defaultdict to group employee and non-employee records by month employee_monthly_totals = defaultdict(float) - non_employee_monthly_totals = defaultdict(float) + employee_ernic_totals = defaultdict(float) + employee_superannuation_totals = defaultdict(float) + + months = [ + "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", "jan", "feb", "mar" + ] # Sum the basic pay for each month for employees and non-employees for employee_record in employee_objects: - employee_monthly_totals[employee_record.current_month.lower()] += employee_record.basic_pay - - for non_employee_record in non_employee_objects: - non_employee_monthly_totals[non_employee_record.current_month.lower()] += non_employee_record.basic_pay - - # Build the monthly totals lists - for month in months: - employee_list.append({ - "month": month, - "total": employee_monthly_totals[month] - }) - non_employee_list.append({ - "month": month, - "total": non_employee_monthly_totals[month] - }) - - return { - "employee": employee_list, - "non_employee": non_employee_list + for month in months: + if getattr(employee_record, month) > 0: + employee_monthly_totals[month] += int(employee_record.basic_pay) + + if getattr(employee_record, month) > 0: + employee_ernic_totals[month] += employee_record.ernic + + if getattr(employee_record, month) > 0: + employee_superannuation_totals[month] += employee_record.superannuation + + month_map = { + "apr": 1, "may": 2, "jun": 3, "jul": 4, "aug": 5, + "sep": 6, "oct": 7, "nov": 8, "dec": 9, + "jan": 10, "feb": 11, "mar": 12 } + + # Loop through employee_monthly_totals and create a forecast_row for each month + for month, total in employee_monthly_totals.items(): + month_number = month_map[month] + forecast_row = { + { + "programme_description": "Basic Pay", + "nac_description": "Basic Pay", + "natural_account_code": "0000", + "programme": "Basic Pay", + "cost_centre": cost_center_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": [ + { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, + } + ], + "budget": 0 + } + } + forecast_months.append(forecast_row) + + for month, total in employee_ernic_totals.items(): + month_number = month_map[month] + forecast_row = { + { + "programme_description": "ERNIC", + "nac_description": "ERNIC", + "natural_account_code": "0000", + "programme": "ERNIC", + "cost_centre": cost_center_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": [ + { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, + } + ], + "budget": 0 + } + } + forecast_months.append(forecast_row) + + for month, total in employee_superannuation_totals.items(): + month_number = month_map[month] + forecast_row = { + { + "programme_description": "Superannuation", + "nac_description": "Superannuation", + "natural_account_code": "0000", + "programme": "Superannuation", + "cost_centre": cost_center_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": [ + { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, + } + ], + "budget": 0 + } + } + forecast_months.append(forecast_row) + + return forecast_months From 1f3e83f0f4526db29c5f179e2858c9f6869ba286 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Mon, 16 Sep 2024 01:23:51 +0100 Subject: [PATCH 21/23] Add cost_centre_code field and refactor payroll models Added the cost_centre_code field to the EmployeePayroll and NonEmployeePayroll models and updated the related functions to use this field. Changed DecimalField to FloatField for the basic_pay, superannuation, and ernic fields across the models and relevant methods. --- payroll/edit_payroll.py | 64 +++++++------ payroll/migrations/0001_initial.py | 84 ++++++++-------- payroll/models.py | 16 ++-- payroll/services.py | 148 ++++++++++++++--------------- 4 files changed, 158 insertions(+), 154 deletions(-) diff --git a/payroll/edit_payroll.py b/payroll/edit_payroll.py index 7b3dc1b2b..dbf6082a2 100644 --- a/payroll/edit_payroll.py +++ b/payroll/edit_payroll.py @@ -50,68 +50,70 @@ def form_valid(self, form): # noqa: C901 month_name = paste_content["month"][0] amount_value = paste_content["amount"][0] table = paste_content["table"][0] + amount = float(amount_value) # get the first item from the list from table data if table == "non_payroll": non_employee_payroll = NonEmployeePayroll.objects.get( staff_number=staff_employee, - # cost_centre_code=cost_centre_code, + cost_centre_code=cost_centre_code, ) + if month_name == "jan": - non_employee_payroll.jan = int(amount_value) + non_employee_payroll.jan = amount if month_name == "feb": - non_employee_payroll.feb = int(amount_value) + non_employee_payroll.feb = amount if month_name == "mar": - non_employee_payroll.mar = int(amount_value) + non_employee_payroll.mar = amount if month_name == "apr": - non_employee_payroll.apr = int(amount_value) + non_employee_payroll.apr = amount if month_name == "may": - non_employee_payroll.may = int(amount_value) + non_employee_payroll.may = amount if month_name == "jun": - non_employee_payroll.jun = int(amount_value) + non_employee_payroll.jun = amount if month_name == "jul": - non_employee_payroll.jul = int(amount_value) + non_employee_payroll.jul = amount if month_name == "aug": - non_employee_payroll.aug = int(amount_value) + non_employee_payroll.aug = amount if month_name == "sep": - non_employee_payroll.sep = int(amount_value) + non_employee_payroll.sep = amount if month_name == "oct": - non_employee_payroll.oct = int(amount_value) + non_employee_payroll.oct = amount if month_name == "nov": - non_employee_payroll.nov = int(amount_value) + non_employee_payroll.nov = amount if month_name == "dec": - non_employee_payroll.dec = int(amount_value) + non_employee_payroll.dec = amount non_employee_payroll.save() else: employee_payroll = EmployeePayroll.objects.get( staff_number=staff_employee, - # cost_centre_code=cost_centre_code, + cost_centre_code=cost_centre_code, ) if month_name == "jan": - employee_payroll.jan = int(amount_value) + employee_payroll.jan = amount if month_name == "feb": - employee_payroll.feb = int(amount_value) + employee_payroll.feb = amount if month_name == "mar": - employee_payroll.mar = int(amount_value) + employee_payroll.mar = amount if month_name == "apr": - employee_payroll.apr = int(amount_value) + employee_payroll.apr = amount if month_name == "may": - employee_payroll.may = int(amount_value) + employee_payroll.may = amount if month_name == "jun": - employee_payroll.jun = int(amount_value) + employee_payroll.jun = amount if month_name == "jul": - employee_payroll.jul = int(amount_value) + employee_payroll.jul = amount if month_name == "aug": - employee_payroll.aug = int(amount_value) + employee_payroll.aug = amount if month_name == "sep": - employee_payroll.sep = int(amount_value) + employee_payroll.sep = amount if month_name == "oct": - employee_payroll.oct = int(amount_value) + employee_payroll.oct = amount if month_name == "nov": - employee_payroll.nov = int(amount_value) + employee_payroll.nov = amount if month_name == "dec": - employee_payroll.dec = int(amount_value) + employee_payroll.dec = amount employee_payroll.save() @@ -198,22 +200,24 @@ def cost_centre_details(self): } def get_employee_payroll_serialiser(self): - get_all_employee_data = EmployeePayroll.objects.all() + logger.info(f"employee payroll serialiser - cost_centre_code: {self.cost_centre_code}") + get_all_employee_data = EmployeePayroll.objects.filter(cost_centre_code=self.cost_centre_code) payroll_serialiser = EmployeePayrollSerializer(get_all_employee_data, many=True) return payroll_serialiser def get_employee_payroll_monthly_serialiser(self): - get_all_employee_data = EmployeePayroll.objects.all() + get_all_employee_data = EmployeePayroll.objects.filter(cost_centre_code=self.cost_centre_code) payroll_monthly_serialiser = EmployeeMonthlyPayrollSerializer(get_all_employee_data, many=True) return payroll_monthly_serialiser def get_non_employee_payroll_serialiser(self): - get_all_non_employee_data = NonEmployeePayroll.objects.all() + logger.info(f"non employee payroll serialiser - cost_centre_code: {self.cost_centre_code}") + get_all_non_employee_data = NonEmployeePayroll.objects.filter(cost_centre_code=self.cost_centre_code) non_payroll_serialiser = NonEmployeePayrollSerializer(get_all_non_employee_data, many=True) return non_payroll_serialiser def get_non_employee_payroll_monthly_serialiser(self): - get_all_non_employee_data = NonEmployeePayroll.objects.all() + get_all_non_employee_data = NonEmployeePayroll.objects.filter(cost_centre_code=self.cost_centre_code) non_payroll_monthly_serialiser = NonEmployeeMonthlyPayrollSerializer(get_all_non_employee_data, many=True) return non_payroll_monthly_serialiser diff --git a/payroll/migrations/0001_initial.py b/payroll/migrations/0001_initial.py index dd4ca610e..d8e510aac 100644 --- a/payroll/migrations/0001_initial.py +++ b/payroll/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-09-12 22:42 +# Generated by Django 5.1 on 2024-09-15 23:56 from django.db import migrations, models @@ -15,6 +15,7 @@ class Migration(migrations.Migration): name='EmployeePayroll', fields=[ ('id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('cost_centre_code', models.CharField(max_length=100)), ('name', models.CharField(max_length=100)), ('grade', models.CharField(max_length=100)), ('staff_number', models.CharField(max_length=100)), @@ -37,9 +38,9 @@ class Migration(migrations.Migration): ('jan', models.IntegerField(verbose_name='january')), ('feb', models.IntegerField(verbose_name='february')), ('mar', models.IntegerField(verbose_name='march')), - ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), - ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), - ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ('basic_pay', models.FloatField(verbose_name='basic pay')), + ('superannuation', models.FloatField(verbose_name='superannuation')), + ('ernic', models.FloatField(verbose_name='ernic')), ], options={ 'abstract': False, @@ -60,45 +61,11 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='HR', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('group', models.CharField(max_length=255)), - ('directorate', models.CharField(max_length=255)), - ('cc', models.CharField(max_length=255)), - ('cc_name', models.CharField(max_length=255)), - ('last_name', models.CharField(max_length=255)), - ('first_name', models.CharField(max_length=255)), - ('se_no', models.CharField(max_length=50)), - ('salary', models.DecimalField(decimal_places=2, max_digits=10)), - ('grade', models.CharField(max_length=10)), - ('employee_location_city_name', models.CharField(max_length=255)), - ('person_type', models.CharField(max_length=255)), - ('assignment_status', models.CharField(max_length=255)), - ('appointment_status', models.CharField(max_length=255)), - ('working_hours', models.DecimalField(decimal_places=2, max_digits=5)), - ('fte', models.DecimalField(decimal_places=2, max_digits=4)), - ('wmi_person', models.CharField(blank=True, max_length=255, null=True)), - ('wmi', models.CharField(blank=True, max_length=255, null=True)), - ('actual_group', models.CharField(max_length=255)), - ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), - ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), - ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), - ('total', models.DecimalField(decimal_places=2, max_digits=10)), - ('costing_cc', models.CharField(max_length=255)), - ('return_field', models.CharField(max_length=255)), - ('programme_code', models.CharField(blank=True, max_length=255, null=True)), - ], - options={ - 'verbose_name': 'HR', - 'verbose_name_plural': 'HR Records', - }, - ), migrations.CreateModel( name='NonEmployeePayroll', fields=[ ('id', models.CharField(max_length=100, primary_key=True, serialize=False, unique=True)), + ('cost_centre_code', models.CharField(max_length=100)), ('name', models.CharField(max_length=100)), ('grade', models.CharField(max_length=100)), ('staff_number', models.CharField(max_length=100)), @@ -122,9 +89,42 @@ class Migration(migrations.Migration): ('jan', models.IntegerField(verbose_name='january')), ('feb', models.IntegerField(verbose_name='february')), ('mar', models.IntegerField(verbose_name='march')), - ('basic_pay', models.DecimalField(decimal_places=2, max_digits=10)), - ('superannuation', models.DecimalField(decimal_places=2, max_digits=10)), - ('ernic', models.DecimalField(decimal_places=2, max_digits=10)), + ('basic_pay', models.FloatField(verbose_name='basic pay')), + ('superannuation', models.FloatField(verbose_name='superannuation')), + ('ernic', models.FloatField(verbose_name='ernic')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='PayrollEntry', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('business_unit_number', models.CharField(max_length=100)), + ('business_unit_name', models.CharField(max_length=100)), + ('cost_center_number', models.CharField(max_length=100)), + ('cost_center_name', models.CharField(max_length=100)), + ('employee_name', models.CharField(max_length=100)), + ('employee_number', models.CharField(max_length=100)), + ('assignment_number', models.CharField(max_length=100)), + ('payroll_name', models.CharField(max_length=100)), + ('employee_organization', models.CharField(max_length=100)), + ('employee_location', models.CharField(max_length=100)), + ('person_type', models.CharField(max_length=100)), + ('employee_category', models.CharField(max_length=100)), + ('assignment_type', models.CharField(max_length=100)), + ('position', models.CharField(max_length=100)), + ('grade', models.CharField(max_length=100)), + ('account_code', models.CharField(max_length=100)), + ('account_name', models.CharField(max_length=100)), + ('pay_element_name', models.CharField(max_length=100)), + ('effective_date', models.CharField(max_length=100)), + ('debit_amount', models.CharField(max_length=100)), + ('credit_amount', models.CharField(max_length=100)), + ('basic_pay', models.CharField(max_length=100)), + ('superannuation', models.CharField(max_length=100)), + ('ernic', models.CharField(max_length=100)), ], options={ 'abstract': False, diff --git a/payroll/models.py b/payroll/models.py index 6f743d181..b7d7d96f4 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -91,7 +91,7 @@ class EmployeePayroll(models.Model): Indicates whether the model is abstract. In this case, set to False. """ id = models.CharField(max_length=100, unique=True, primary_key=True) - # cost_centre_code = models.CharField(max_length=100) + cost_centre_code = models.CharField(max_length=100) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -114,9 +114,9 @@ class EmployeePayroll(models.Model): jan = models.IntegerField(editable=True, verbose_name='january') feb = models.IntegerField(editable=True, verbose_name='february') mar = models.IntegerField(editable=True, verbose_name='march') - basic_pay = models.DecimalField(max_digits=10, decimal_places=2) - superannuation = models.DecimalField(max_digits=10, decimal_places=2) - ernic = models.DecimalField(max_digits=10, decimal_places=2) + basic_pay = models.FloatField(editable=True, verbose_name='basic pay') + superannuation = models.FloatField(editable=True, verbose_name='superannuation') + ernic = models.FloatField(editable=True, verbose_name='ernic') class Meta: abstract = False @@ -158,7 +158,7 @@ class NonEmployeePayroll(models.Model): abstract (bool): Indicates that this model is not abstract. """ id = models.CharField(max_length=100, unique=True, primary_key=True) - # cost_centre_code = models.CharField(max_length=100) + cost_centre_code = models.CharField(max_length=100) name = models.CharField(max_length=100) grade = models.CharField(max_length=100) staff_number = models.CharField(max_length=100) @@ -182,9 +182,9 @@ class NonEmployeePayroll(models.Model): jan = models.IntegerField(editable=True, verbose_name='january') feb = models.IntegerField(editable=True, verbose_name='february') mar = models.IntegerField(editable=True, verbose_name='march') - basic_pay = models.DecimalField(max_digits=10, decimal_places=2) - superannuation = models.DecimalField(max_digits=10, decimal_places=2) - ernic = models.DecimalField(max_digits=10, decimal_places=2) + basic_pay = models.FloatField(editable=True, verbose_name='basic pay') + superannuation = models.FloatField(editable=True, verbose_name='superannuation') + ernic = models.FloatField(editable=True, verbose_name='ernic') class Meta: abstract = False diff --git a/payroll/services.py b/payroll/services.py index 1c740ab21..57ab742ea 100644 --- a/payroll/services.py +++ b/payroll/services.py @@ -1,4 +1,5 @@ import datetime +from calendar import month from collections import defaultdict from hr.models import HRModel from payroll.models import EmployeePayroll, NonEmployeePayroll @@ -60,7 +61,7 @@ def update_employee_tables(): non_employee_record.current_year = current_year non_employee_record.save() -def get_forecast_basic_pay_for_employee(cost_center_code, financial_year): +def get_forecast_basic_pay_for_employee(cost_centre_code, financial_year): """ Generates a forecast of basic pay, ERNIC, and superannuation for employees in a given cost center and financial year. @@ -89,10 +90,7 @@ def get_forecast_basic_pay_for_employee(cost_center_code, financial_year): # Get all objects from EmployeePayroll table where cost_center_code is equal to the cost_center_code # employee_objects = EmployeePayroll.objects.filter(cost_center_code=cost_center_code, current_year=financial_year) - employee_objects = EmployeePayroll.objects.all() - - # Get all objects from NonEmployeePayroll table - non_employee_objects = NonEmployeePayroll.objects.all() + employee_objects = EmployeePayroll.objects.filter(cost_centre_code=cost_centre_code) # Loop through all objects and create a list of dictionaries for each month (april to march) # Each dictionary will have the following keys: @@ -130,82 +128,84 @@ def get_forecast_basic_pay_for_employee(cost_center_code, financial_year): } # Loop through employee_monthly_totals and create a forecast_row for each month - for month, total in employee_monthly_totals.items(): + ernic_totals = [] + for month, total in employee_ernic_totals.items(): month_number = month_map[month] - forecast_row = { - { - "programme_description": "Basic Pay", - "nac_description": "Basic Pay", - "natural_account_code": "0000", - "programme": "Basic Pay", - "cost_centre": cost_center_code, - "analysis1_code": "0000", - "analysis2_code": "0000", - "project_code": "0000", - "monthly_figures": [ - { - "actual": False, - "month": month_number, - "amount": total, - "starting_amount": total, - "archived_status": None, - } - ], - "budget": 0 - } + + ernic_month = { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, } - forecast_months.append(forecast_row) + ernic_totals.append(ernic_month) + + forecast_row = { + "programme_description": "ERNIC", + "nac_description": "ERNIC", + "natural_account_code": "0000", + "programme": "ERNIC", + "cost_centre": cost_centre_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": ernic_totals, + "budget": 0 + } + forecast_months.append(forecast_row) - for month, total in employee_ernic_totals.items(): + superannuation_totals = [] + for month, total in employee_superannuation_totals.items(): month_number = month_map[month] - forecast_row = { - { - "programme_description": "ERNIC", - "nac_description": "ERNIC", - "natural_account_code": "0000", - "programme": "ERNIC", - "cost_centre": cost_center_code, - "analysis1_code": "0000", - "analysis2_code": "0000", - "project_code": "0000", - "monthly_figures": [ - { - "actual": False, - "month": month_number, - "amount": total, - "starting_amount": total, - "archived_status": None, - } - ], - "budget": 0 - } + superannuation_month = { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, } - forecast_months.append(forecast_row) + superannuation_totals.append(superannuation_month) + + forecast_row = { + "programme_description": "Superannuation", + "nac_description": "Superannuation", + "natural_account_code": "0000", + "programme": "Superannuation", + "cost_centre": cost_centre_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": superannuation_totals, + "budget": 0 + } + forecast_months.append(forecast_row) - for month, total in employee_superannuation_totals.items(): + basic_pay_totals = [] + for month, total in employee_monthly_totals.items(): month_number = month_map[month] - forecast_row = { - { - "programme_description": "Superannuation", - "nac_description": "Superannuation", - "natural_account_code": "0000", - "programme": "Superannuation", - "cost_centre": cost_center_code, - "analysis1_code": "0000", - "analysis2_code": "0000", - "project_code": "0000", - "monthly_figures": [ - { - "actual": False, - "month": month_number, - "amount": total, - "starting_amount": total, - "archived_status": None, - } - ], - "budget": 0 - } + + basic_pay_month = { + "actual": False, + "month": month_number, + "amount": total, + "starting_amount": total, + "archived_status": None, } - forecast_months.append(forecast_row) + basic_pay_totals.append(basic_pay_month) + + forecast_row = { + "programme_description": "Basic Pay", + "nac_description": "Basic Pay", + "natural_account_code": "0000", + "programme": "Basic Pay", + "cost_centre": cost_centre_code, + "analysis1_code": "0000", + "analysis2_code": "0000", + "project_code": "0000", + "monthly_figures": basic_pay_totals, + "budget": 0 + } + forecast_months.append(forecast_row) return forecast_months From cf2db6b57b84dc10519493dfe88bdd50ab81d7bc Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Wed, 18 Sep 2024 16:38:32 +0100 Subject: [PATCH 22/23] chore:add MyHR feature and relevant updates. Added new migration for HRModel, and made corresponding changes to the models and serializers. Enhanced state management in the front end to support new MyHR functionality, added MyHREmployee components, views, and templates for displaying MyHR data. --- config/settings/base.py | 1 + config/urls.py | 1 + core/templates/base_generic.html | 6 +- front_end/src/Apps/MyHREmployee.jsx | 14 ++ .../ActualsWithForecastHeaderRow/index.jsx | 31 +++ .../src/Components/MyHREmployee/index.jsx | 200 ++++++++++++++++++ .../MyHREmployeeCellValue/index.jsx | 34 +++ .../Components/MyHREmployeeTable/index.jsx | 120 +++++++++++ .../MyHREmployeeTableCell/index.jsx | 195 +++++++++++++++++ front_end/src/Reducers/Cells.js | 14 +- front_end/src/Reducers/Selected.js | 18 ++ front_end/src/Util.js | 34 +++ front_end/src/index.jsx | 5 + hr/migrations/0001_initial.py | 49 +++++ hr/models.py | 14 +- myhr/__init__.py | 0 myhr/admin.py | 3 + myhr/apps.py | 6 + myhr/directorate.py | 59 ++++++ myhr/group.py | 58 +++++ myhr/migrations/0001_initial.py | 26 +++ myhr/migrations/__init__.py | 0 myhr/models.py | 10 + myhr/serialisers.py | 17 ++ myhr/templates/myhr/list/directorate.html | 35 +++ myhr/templates/myhr/list/list.html | 34 +++ myhr/templates/myhr/list/myhr_base.html | 14 ++ myhr/tests.py | 3 + myhr/urls.py | 9 + myhr/views.py | 4 + 30 files changed, 1001 insertions(+), 13 deletions(-) create mode 100644 front_end/src/Apps/MyHREmployee.jsx create mode 100644 front_end/src/Components/ActualsWithForecastHeaderRow/index.jsx create mode 100644 front_end/src/Components/MyHREmployee/index.jsx create mode 100644 front_end/src/Components/MyHREmployeeCellValue/index.jsx create mode 100644 front_end/src/Components/MyHREmployeeTable/index.jsx create mode 100644 front_end/src/Components/MyHREmployeeTableCell/index.jsx create mode 100644 hr/migrations/0001_initial.py create mode 100644 myhr/__init__.py create mode 100644 myhr/admin.py create mode 100644 myhr/apps.py create mode 100644 myhr/directorate.py create mode 100644 myhr/group.py create mode 100644 myhr/migrations/0001_initial.py create mode 100644 myhr/migrations/__init__.py create mode 100644 myhr/models.py create mode 100644 myhr/serialisers.py create mode 100644 myhr/templates/myhr/list/directorate.html create mode 100644 myhr/templates/myhr/list/list.html create mode 100644 myhr/templates/myhr/list/myhr_base.html create mode 100644 myhr/tests.py create mode 100644 myhr/urls.py create mode 100644 myhr/views.py diff --git a/config/settings/base.py b/config/settings/base.py index 363ed4d71..5bebafb21 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -76,6 +76,7 @@ "axes", "django_chunk_upload_handlers", "payroll", + "myhr", "hr", "app_layer", ] diff --git a/config/urls.py b/config/urls.py index 193478e7e..11d8f5303 100644 --- a/config/urls.py +++ b/config/urls.py @@ -36,6 +36,7 @@ path("oscar_return/", include("oscar_return.urls")), path("upload_split_file/", include("upload_split_file.urls")), path("payroll/", include("payroll.urls")), + path("myhr/", include("myhr.urls")), path("admin/", admin.site.urls), # TODO - split below out into develop only? path( diff --git a/core/templates/base_generic.html b/core/templates/base_generic.html index e7f7bcd6b..382ae70dc 100644 --- a/core/templates/base_generic.html +++ b/core/templates/base_generic.html @@ -115,9 +115,9 @@