Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HR data ingest #593

Draft
wants to merge 15 commits into
base: main
Choose a base branch
from
Prev Previous commit
Next Next commit
WIP ingestion hr payroll data
DenisDDBT committed Jan 9, 2025
commit a8094ebbe10eed134a07c97165b4b0531f188f51
18 changes: 18 additions & 0 deletions payroll/migrations/0016_employee_has_left.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 5.1.4 on 2025-01-09 11:16

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("payroll", "0015_employee_basic_pay_employee_ernic_employee_pension"),
]

operations = [
migrations.AddField(
model_name="employee",
name="has_left",
field=models.BooleanField(default=False),
),
]
1 change: 1 addition & 0 deletions payroll/models.py
Original file line number Diff line number Diff line change
@@ -93,6 +93,7 @@ class Employee(Position):
basic_pay = models.BigIntegerField(default=0, db_comment="pence")
pension = models.BigIntegerField(default=0, db_comment="pence")
ernic = models.BigIntegerField(default=0, db_comment="pence")
has_left=models.BooleanField(default=False)

# TODO: Missing fields from Admin Tool which aren't required yet.
# EU/Non-EU (from programme code model)
93 changes: 82 additions & 11 deletions payroll/services/ingest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import csv
from collections import namedtuple
import traceback

from django.db import transaction
from django.db.models import Q
from django.core.files import File

from payroll.models import Employee
@@ -11,13 +14,13 @@
(
"group_name",
"directorate_name",
"cost_centre_code",
"cost_centre_id",
"cost_centre_name",
"last_name",
"first_name",
"employee_no",
"salary",
"grade",
"basic_pay",
"grade_id",
"employee_location_city_name",
"person_type",
"assignment_status",
@@ -43,7 +46,7 @@
"col32",
"col33",
"line_manager",
"programme_code",
"programme_code_id",
"payroll_cost_centre_code",
"payroll_cost_centre_matches",
),
@@ -52,9 +55,9 @@

def import_payroll(
hr_csv: File,
payroll_csv: File | None,
# payroll_csv: File | None,
hr_csv_has_header: bool,
payroll_csv_has_header: bool,
# payroll_csv_has_header: bool,
) -> str:
hr_csv_reader = csv.reader((row.decode("utf-8") for row in hr_csv))

@@ -66,14 +69,82 @@ def import_payroll(
for hr_row in hr_csv_reader:
employees.append(hr_row_to_employee(HrRow(*hr_row)))

# Employee.objects.bulk_create(employees)

return f"Success: created {len(employees)}"
return save_data(employees)


def hr_row_to_employee(hr_row: HrRow) -> Employee:
employee = Employee()
employee_fields = vars(employee)
matching_fields = [field for field in hr_row._fields if field in employee_fields]
for field in matching_fields:
setattr(employee, field, getattr(hr_row, field))
return employee

# logic here
def save_data(csv_data):
updatable_fields = [
"first_name",
"last_name",
"cost_centre_id",
"programme_code_id",
"assignment_status",
"fte",
"grade_id",
"basic_pay",
"ernic",
"pension",
"has_left"
]
csv_identifiers = {
employee.employee_no
for employee in csv_data
}
try:
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
with transaction.atomic():
existing_employees = Employee.objects.all().select_for_update()
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
existing_identifiers = {
employee.employee_no
for employee in existing_employees
}
left_identifiers = existing_identifiers - csv_identifiers
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
if left_identifiers:
left_filter = Q()
for employee_no in left_identifiers:
left_filter |= Q(
employee_no=employee_no
)
Employee.objects.filter(left_filter).update(has_left=True)
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
to_create = []
to_update = []
for employee_from_csv in csv_data:
identifier = employee_from_csv.employee_no
if identifier in existing_identifiers:
existing_employee = next(
emp for emp in existing_employees
if emp.employee_no == identifier
)
for field in updatable_fields:
if hasattr(employee_from_csv, field):
setattr(existing_employee, field, getattr(employee_from_csv, field))
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
existing_employee.has_left = False
to_update.append(existing_employee)
else:
employee_from_csv.has_left = False
to_create.append(employee_from_csv)
Employee.objects.bulk_create(to_create)
SamDudley marked this conversation as resolved.
Show resolved Hide resolved

return employee
if to_update:
Employee.objects.bulk_update( to_update, fields=updatable_fields)
return {
'created': len(to_create),
'updated': len(to_update),
'marked_left': len(left_identifiers)
}
except Exception as e:
print (traceback.format_exc())
SamDudley marked this conversation as resolved.
Show resolved Hide resolved
return {
'status': 'error',
'message': str(e),
'created': 0,
'updated': 0,
'marked_left': 0
}
27 changes: 16 additions & 11 deletions payroll/views.py
Original file line number Diff line number Diff line change
@@ -191,16 +191,21 @@ def import_payroll_page(request: HttpRequest) -> HttpResponse:
raise PermissionDenied

output = ""

context={}
if request.method == "POST":
output = import_payroll(
hr_csv=request.FILES["hr_csv"],
payroll_csv=request.FILES.get("payroll_csv"),
hr_csv_has_header=request.POST.get("hr_csv_has_header", False),
payroll_csv_has_header=request.POST.get("hr_csv_has_header", False),
)

context = {
"output": output,
}
if 'hr_csv' not in request.FILES :
context= {"error": "Please select file"}
else:
hr_csv = request.FILES['hr_csv']
hr_csv_has_header = request.POST.get('hr_csv_has_header', False)
# payroll_csv = request.FILES['payroll_csv']
# payroll_csv_has_header = request.POST.get("hr_csv_has_header", False)
output = import_payroll(
hr_csv,
hr_csv_has_header,
)

context = {
"output": output,
}
return TemplateResponse(request, "payroll/page/import_payroll.html", context)