diff --git a/config/settings/local.py b/config/settings/local.py index 2ad1612e8..670e51541 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -20,3 +20,8 @@ DEBUG_TOOLBAR_CONFIG = { "SHOW_TOOLBAR_CALLBACK": lambda request: DEBUG, } + +FILE_UPLOAD_HANDLERS = ( + "django.core.files.uploadhandler.MemoryFileUploadHandler", + "django.core.files.uploadhandler.TemporaryFileUploadHandler", +) diff --git a/payroll/migrations/0016_employee_has_left.py b/payroll/migrations/0016_employee_has_left.py new file mode 100644 index 000000000..287898602 --- /dev/null +++ b/payroll/migrations/0016_employee_has_left.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.4 on 2025-01-09 11:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("payroll", "0015_employee_basic_pay_employee_ernic_employee_pension"), + ] + + operations = [ + migrations.AddField( + model_name="employee", + name="has_left", + field=models.BooleanField(default=False), + ), + ] diff --git a/payroll/models.py b/payroll/models.py index 237012aa1..8afc2c85d 100644 --- a/payroll/models.py +++ b/payroll/models.py @@ -93,6 +93,7 @@ class Employee(Position): basic_pay = models.BigIntegerField(default=0, db_comment="pence") pension = models.BigIntegerField(default=0, db_comment="pence") ernic = models.BigIntegerField(default=0, db_comment="pence") + has_left = models.BooleanField(default=False) # TODO: Missing fields from Admin Tool which aren't required yet. # EU/Non-EU (from programme code model) @@ -100,7 +101,7 @@ class Employee(Position): objects = EmployeeQuerySet.as_manager() def __str__(self) -> str: - return f"{self.employee_no} - {self.first_name} {self.last_name}" + return f"{self.id} {self.employee_no} - {self.first_name} {self.last_name}" def get_full_name(self) -> str: return f"{self.first_name} {self.last_name}" diff --git a/payroll/services/ingest.py b/payroll/services/ingest.py new file mode 100644 index 000000000..433d76542 --- /dev/null +++ b/payroll/services/ingest.py @@ -0,0 +1,229 @@ +import csv +import random +from collections import namedtuple + +from django.core.files import File +from django.db import transaction + +from chartofaccountDIT.models import ProgrammeCode +from costcentre.models import CostCentre +from gifthospitality.models import Grade +from payroll.models import Employee + + +HrRow = namedtuple( + "HrRow", + ( + "group_name", + "directorate_name", + "cost_centre_id", + "cost_centre_name", + "last_name", + "first_name", + "employee_no", + "salary", + "grade_id", + "employee_location_city_name", + "person_type", + "assignment_status", + "appointment_status", + "working_hours", + "fte", + "col16", + "col17", + "col18", + "col19", + "col20", + "col21", + "col22", + "col23", + "return_date", + "col25", + "col26", + "col27", + "col28", + "col29", + "col30", + "col31", + "col32", + "col33", + "line_manager", + "programme_code_id", + "payroll_cost_centre_code", + "payroll_cost_centre_matches", + ), +) + +PayrollRow = namedtuple( + "PayrollRow", + ( + ( + "col_a", + "col_b", + "col_c", + "col_d", + "col_e", + "employee_no", + "col_g", + "col_h", + "col_i", + "col_j", + "col_k", + "col_l", + "col_m", + "col_n", + "col_o", + "col_p", + "col_q", + "pay_type", + "col_s", + "debit_amount", + "credit_amount", + ) + ), +) + + +@transaction.atomic() +def import_payroll( + hr_csv: File, + payroll_csv: File | None, + hr_csv_has_header: bool, + payroll_csv_has_header: bool, +) -> str: + # payroll_data=[] + # payrol_csv_reader= csv.reader((row.decode("utf-8") for row in payroll_csv)) + # if payroll_csv_has_header: + # next(payrol_csv_reader) + + # for payroll_row in payrol_csv_reader: + # payroll_data.append(map_payroll(PayrollRow(**payroll_row))) + + hr_csv_reader = csv.reader((row.decode("utf-8") for row in hr_csv)) + + if hr_csv_has_header: + next(hr_csv_reader) + + employees = [] + cost_centres = [] + programme_codes = [] + grades = [] + for hr_row in hr_csv_reader: + employee = hr_row_to_employee(HrRow(*hr_row)) + employees.append(employee) + cost_centres.append(employee["cost_centre"]) + programme_codes.append(employee["programme_code"]) + grades.append(employee["grade"]) + uniq_cost_centre_codes = set(cost_centres) + uniq_grades = set(grades) + uniq_programme_codes = set(programme_codes) + cost_centers = { + centre.cost_centre_code: centre + for centre in CostCentre.objects.filter( + cost_centre_code__in=uniq_cost_centre_codes + ) + } + cost_center_codes = [centre.cost_centre_code for centre in cost_centers.values()] + + programme_codes = { + code.programme_code: code + for code in ProgrammeCode.objects.filter( + programme_code__in=uniq_programme_codes + ) + } + existing_programme_codes = [ + code.programme_code for code in programme_codes.values() + ] + + grades = { + grade.grade: grade for grade in Grade.objects.filter(grade__in=uniq_grades) + } + existing_grades = [grade.grade for grade in grades.values()] + + clean_records = [] + failed_records = [] + for emp in employees: + errors = [] + if emp["cost_centre"] not in cost_center_codes: + errors.append(f"Cost centre '{emp["cost_centre"]}' doesn't exists") + if emp["programme_code"] not in existing_programme_codes: + errors.append(f"Programme code '{emp["programme_code"]}' doesn't exists") + if emp["grade"] not in existing_grades: + errors.append(f"Grade '{emp["grade"]}' doesn't exists") + if errors: + emp["errors"] = errors + failed_records.append(emp) + else: + emp["cost_centre"] = cost_centers[emp["cost_centre"]] + emp["programme_code"] = programme_codes[emp["programme_code"]] + emp["grade"] = grades[emp["grade"]] + clean_records.append(emp) + result = save_data(clean_records) + + return {"failed_records": failed_records, **result} + + +def hr_row_to_employee(hr_row) -> Employee: + employee = { + "employee_no": hr_row.employee_no, + "first_name": hr_row.first_name, + "last_name": hr_row.last_name, + "cost_centre": hr_row.cost_centre_id, + "programme_code": hr_row.programme_code_id, + "grade": hr_row.grade_id, + "assignment_status": hr_row.assignment_status, + "fte": hr_row.fte, + "basic_pay": random.randint(100000, 999999), + "ernic": random.randint(100000, 999999), + "pension": random.randint(100000, 999999), + "has_left": False, + } + return employee + + +def save_data(csv_data): + emp_nos = {employee["employee_no"] for employee in csv_data} + Employee.objects.exclude(employee_no__in=emp_nos).filter(has_left=False).update( + has_left=True + ) + return bulk_update_or_create(csv_data) + + +def bulk_update_or_create(data): + if data: + keys = list(data[0].keys()) + existing_ids = { + emp.employee_no: emp.id + for emp in Employee.objects.filter( + employee_no__in=[emp["employee_no"] for emp in data] + ) + } + to_update = [] + to_create = [] + + for item in data: + emp = Employee(**item) + for key in keys: + setattr(emp, key, item[key]) + + if item["employee_no"] in existing_ids: + emp.id = existing_ids[item["employee_no"]] + to_update.append(emp) + else: + to_create.append(emp) + + if to_create: + Employee.objects.bulk_create(to_create) + if to_update: + Employee.objects.bulk_update(to_update, keys) + + return {"created": to_create, "updated": to_update} + + +# def map_payroll(row): +# return { +# "employee_no":row.employee_no, +# "pay_type":row.pay_type, +# "debit_amount":row.debit_amount, +# "credit_amount":row.credit_amount +# } diff --git a/payroll/services/payroll.py b/payroll/services/payroll.py index 93feedb73..621b56260 100644 --- a/payroll/services/payroll.py +++ b/payroll/services/payroll.py @@ -76,8 +76,7 @@ def payroll_forecast_report( ) employee_qs = Employee.objects.filter( - cost_centre=cost_centre, - pay_periods__year=financial_year, + cost_centre=cost_centre, pay_periods__year=financial_year, has_left=False ) pay_uplift_obj = PayUplift.objects.filter(financial_year=financial_year).first() attrition_obj = get_attrition_instance(financial_year, cost_centre) @@ -152,7 +151,12 @@ def get_average_salary_for_grade(grade: Grade, cost_centre: CostCentre) -> int: salaries: list[int] = [] for filter in filters: - employee_qs = Employee.objects.payroll().filter(grade=grade).filter(filter) + employee_qs = ( + Employee.objects.payroll() + .filter(grade=grade) + .filter(filter) + .filter(has_left=False) + ) basic_pay = employee_qs.aggregate( count=Count("basic_pay"), avg=Avg("basic_pay") diff --git a/payroll/templates/payroll/page/import_payroll.html b/payroll/templates/payroll/page/import_payroll.html new file mode 100644 index 000000000..9520f2eb6 --- /dev/null +++ b/payroll/templates/payroll/page/import_payroll.html @@ -0,0 +1,71 @@ +{% extends "base_generic.html" %} +{% load breadcrumbs vite %} + +{% block title %}Import payroll{% endblock title %} + +{% block breadcrumbs %} + {{ block.super }} + {% breadcrumb "Import payroll" "payroll:import" %} +{% endblock breadcrumbs %} + +{% block content %} +

Import payroll

+
+ {% csrf_token %} +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+

+ {% if output %} + Inserted records : {{output.created|length}}
+ Updated records: {{output.updated|length}}
+ Failing records: {{output.failed_records|length}}
+ {% endif %} + + {{ error }} + {% if output.failed_records %} +

Failing records

+ + + + + + + + + + {% for item in output.failed_records %} + + + + + + {% endfor %} + +
Employee NoNameError
{{ item.employee_no }}{{ item.first_name }} {{ item.last_name }} {% for error in item.errors %} + {{error}}
+ {% endfor %} +
+ {% endif %} +
+{% endblock content %} + +{% block scripts %} + {{ block.super }} +{% endblock scripts %} diff --git a/payroll/urls.py b/payroll/urls.py index 6b9e293c0..3394a143f 100644 --- a/payroll/urls.py +++ b/payroll/urls.py @@ -48,4 +48,6 @@ views.PayModifierApiView.as_view(), name="api_pay_modifiers", ), + # TODO: Remove temporary views when ready. + path("import", views.import_payroll_page, name="import"), ] diff --git a/payroll/views.py b/payroll/views.py index d1dc253c4..68dc35b45 100644 --- a/payroll/views.py +++ b/payroll/views.py @@ -1,7 +1,8 @@ import json from django.contrib.auth.mixins import PermissionRequiredMixin, UserPassesTestMixin -from django.http import HttpResponse, JsonResponse +from django.core.exceptions import PermissionDenied +from django.http import HttpRequest, HttpResponse, JsonResponse from django.shortcuts import get_object_or_404 from django.template.response import TemplateResponse from django.urls import reverse @@ -15,6 +16,7 @@ from payroll.models import Vacancy from .services import payroll as payroll_service +from .services.ingest import import_payroll class EditPayrollBaseView(UserPassesTestMixin, View): @@ -182,3 +184,27 @@ def get_context_data(self, **kwargs): "vacancy_id": self.object.id, } return super().get_context_data(**kwargs) | context + + +def import_payroll_page(request: HttpRequest) -> HttpResponse: + if not request.user.is_superuser: + raise PermissionDenied + + output = "" + context = {} + if request.method == "POST": + if "hr_csv" not in request.FILES or "payroll_csv" not in request.FILES: + context = {"error": "Both HR and Payroll files are required"} + else: + hr_csv = request.FILES["hr_csv"] + hr_csv_has_header = request.POST.get("hr_csv_has_header", False) + payroll_csv = request.FILES["payroll_csv"] + payroll_csv_has_header = request.POST.get("hr_csv_has_header", False) + output = import_payroll( + hr_csv, payroll_csv, hr_csv_has_header, payroll_csv_has_header + ) + + context = { + "output": output, + } + return TemplateResponse(request, "payroll/page/import_payroll.html", context)