diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f40189e..794f8ff6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: - name: Package run: qgis-plugin-ci --no-validation package 'test' - name: Lint + run: black . --diff --color + - name: Lint check run: black . --check - name: Check run: pyright . > pyright_report.txt diff --git a/comptages/comptages.py b/comptages/comptages.py index 4256c0ea..893b0a41 100644 --- a/comptages/comptages.py +++ b/comptages/comptages.py @@ -250,15 +250,7 @@ def import_file(self, file_path, count_id=None): return QgsMessageLog.logMessage( - "{} - Prepare import file {}".format( - datetime.now(), os.path.basename(file_path) - ), - "Comptages", - Qgis.Info, - ) - - QgsMessageLog.logMessage( - "{} - Import file {} started".format( + "{} - Prepare import file {} ended".format( datetime.now(), os.path.basename(file_path) ), "Comptages", @@ -369,14 +361,16 @@ def do_filter_action(self): def do_yearly_report_action(self): QgsMessageLog.logMessage( - "{} - Generate yearly report action started".format(datetime.now()), + "{} - Generate task for yearly report action started".format( + datetime.now() + ), "Comptages", Qgis.Info, ) if self.tm.countActiveTasks() > 0: push_info( - ("Veuillez patienter jusqu'à ce que l'importation " "soit terminée.") + ("Veuillez patienter jusqu'à ce que l'importation soit terminée.") ) return @@ -394,11 +388,15 @@ def do_yearly_report_action(self): section_id = selected_feature.attribute("id") section = models.Section.objects.get(id=section_id) - classes = self.layers.get_classes_of_section(section_id) + years = self.layers.get_years_of_counts_on_section(section_id) + dlg = YearlyReportDialog(self.iface) dlg.section.insert(section_id) dlg.classi.addItems(classes) + if years is not None: + years.sort() + dlg.year.setValue(years[-1]) if dlg.exec_(): year = dlg.year.value() @@ -411,7 +409,7 @@ def do_yearly_report_action(self): if not file_path: QgsMessageLog.logMessage( - "{} - Generate yearly report action ended: No file_path given".format( + "{} - Generate task for yearly report action cancelled: No file_path given".format( datetime.now() ), "Comptages", @@ -419,7 +417,7 @@ def do_yearly_report_action(self): ) return QgsMessageLog.logMessage( - "{} - Generate yearly report action can really begin now for count {} with file_path: {}".format( + "{} - Generate task for yearly report action can really begin now for count {} with file_path: {}".format( datetime.now(), selected_count, file_path ), "Comptages", @@ -435,7 +433,7 @@ def do_yearly_report_action(self): return if clazz.startswith("SPCH-MD"): - yrb = YearlyReportBike(file_path, year, section_id) + yrb = YearlyReportBike(file_path, year, section_id, clazz) yrb.run() else: self.tm.allTasksFinished.connect( @@ -454,7 +452,7 @@ def do_yearly_report_action(self): # TODO: check if there are comptages for this section and year QgsMessageLog.logMessage( - "{} - Generate yearly report action ended".format(datetime.now()), + "{} - Generate task for yearly report action ended".format(datetime.now()), "Comptages", Qgis.Info, ) @@ -505,7 +503,9 @@ def do_import_single_file_action(self, count_id): def do_generate_report_action(self, count_id): QgsMessageLog.logMessage( - "{} - Generate report action started".format(datetime.now()), + "{} - Generate report preparation started for count_id {}".format( + datetime.now(), count_id + ), "Comptages", Qgis.Info, ) @@ -514,7 +514,10 @@ def do_generate_report_action(self, count_id): if self.tm.countActiveTasks() > 0: push_info( - ("Veuillez patienter jusqu'à ce que l'importation " "soit terminée.") + ( + "Veuillez patienter jusqu'à ce que l'importation soit terminée," + " puis relancer la génération du rapport." + ) ) return @@ -525,7 +528,7 @@ def do_generate_report_action(self, count_id): "le comptage {}".format(count.id_installation.name, count.id) ) QgsMessageLog.logMessage( - "{} - Generate report action ended: No data for count {}".format( + "{} - Generate report preparation ended: No data for count {}".format( datetime.now(), count.id ), "Comptages", @@ -534,39 +537,50 @@ def do_generate_report_action(self, count_id): return file_dialog = QFileDialog() - mondays = list(report._mondays_of_count(count)) + mondays = report._mondays_of_count(count) sections_ids = ( models.Section.objects.filter(lane__id_installation__count=count) .distinct() .values_list("id", flat=True) ) report_selection_dialog = SelectSectionsToReport( - sections_ids=list(sections_ids), mondays=mondays + sections_ids=list(sections_ids), mondays=list(mondays) ) if report_selection_dialog.exec_(): selected_sections_dates: dict[str, list[date]] = ( report_selection_dialog.get_inputs() ) - title = "Exporter un rapport" + date_choosen = list() + for selsec in selected_sections_dates: + date_choosen.extend(selected_sections_dates[selsec]) + if len(date_choosen) == 0: + QgsMessageLog.logMessage( + "{} - Generate report preparation ended: Nothing choosen to report on".format( + datetime.now() + ), + "Comptages", + Qgis.Info, + ) + return + title = "Exporter un rapport" path = self.settings.value("report_export_directory") file_path = QFileDialog.getExistingDirectory(file_dialog, title, path) if not file_path: QgsMessageLog.logMessage( - "{} - Generate report action ended: No file_path given".format( + "{} - Generate report preparation ended: No file_path given".format( datetime.now() ), "Comptages", Qgis.Info, ) return + QgsMessageLog.logMessage( - f""" - {datetime.now()} - Generate report action can really begin now for count {count.id} with file_path: {file_path}. - Selected sections and dates: {selected_sections_dates} - """, + f"""{datetime.now()} - Generate report action can really begin now for count {count.id} with file_path: {file_path}. + Selected sections and dates: {selected_sections_dates}""", "Comptages", Qgis.Info, ) @@ -579,6 +593,23 @@ def do_generate_report_action(self, count_id): selected_sections_dates=selected_sections_dates, ) ) + else: + QgsMessageLog.logMessage( + "{} - Generate report preparation ended: Cancel buton clicked...".format( + datetime.now() + ), + "Comptages", + Qgis.Info, + ) + return + + QgsMessageLog.logMessage( + "{} - Generate report preparation ended for count_id {}".format( + datetime.now(), count_id + ), + "Comptages", + Qgis.Info, + ) def do_export_plan_action(self, count_id): count = models.Count.objects.get(id=count_id) diff --git a/comptages/core/importer.py b/comptages/core/importer.py index 51b15e58..d451037f 100644 --- a/comptages/core/importer.py +++ b/comptages/core/importer.py @@ -1,6 +1,6 @@ from typing import Callable, Iterator, Optional -import pytz -import os +from pytz import timezone +from os import path from datetime import datetime, timedelta from django.db.models import Q @@ -8,6 +8,8 @@ from comptages.datamodel import models from comptages.core.bulk_create_manager import BulkCreateManager +TZ = timezone("Europe/Zurich") + def simple_print_callback(progress: int): if progress % 10 == 0: @@ -51,7 +53,7 @@ def _parse_and_write( from_aggregate: bool = False, **kwargs, ): - basename = os.path.basename(file_path) + basename = path.basename(file_path) bulk_mgr = BulkCreateManager(chunk_size=1000) lanes = _populate_lane_dict(count) directions = _populate_direction_dict(count) @@ -112,10 +114,9 @@ def _parse_line_vbv1(line: str, **kwargs) -> Optional[list[dict]]: return None parsed_line = {} - tz = pytz.timezone("Europe/Zurich") try: parsed_line["numbering"] = line[0:6] - parsed_line["timestamp"] = tz.localize( + parsed_line["timestamp"] = TZ.localize( datetime.strptime("{}0000".format(line[7:24]), "%d%m%y %H%M %S %f") ) parsed_line["reserve_code"] = line[25:31] @@ -162,11 +163,10 @@ def _parse_line_mc(line: str, **kwargs) -> Optional[list[dict]]: parsed_line = {} try: - tz = pytz.timezone("Europe/Zurich") # TODO: numbering numbering = 1 parsed_line["numbering"] = numbering - parsed_line["timestamp"] = tz.localize( + parsed_line["timestamp"] = TZ.localize( datetime.strptime(line[0:19], "%Y-%m-%d %H:%M:%S") ) # On MetroCount files, the direction is 0-1 instead of 1-2 @@ -198,7 +198,6 @@ def _parse_line_int2(line, **kwargs) -> Iterator[Optional[dict]]: return None parsed_line = {} - tz = pytz.timezone("Europe/Zurich") # TODO: numbering numbering = 1 parsed_line["numbering"] = numbering @@ -206,10 +205,10 @@ def _parse_line_int2(line, **kwargs) -> Iterator[Optional[dict]]: # instead of 0000 of the next day if line[7:9] == "24": line = line[:7] + "00" + line[9:] - end = tz.localize(datetime.strptime("{}".format(line[0:11]), "%d%m%y %H%M")) + end = TZ.localize(datetime.strptime("{}".format(line[0:11]), "%d%m%y %H%M")) end += timedelta(days=1) else: - end = tz.localize(datetime.strptime("{}".format(line[0:11]), "%d%m%y %H%M")) + end = TZ.localize(datetime.strptime("{}".format(line[0:11]), "%d%m%y %H%M")) parsed_line["end"] = end parsed_line["start"] = parsed_line["end"] - timedelta(minutes=kwargs["interval"]) @@ -294,7 +293,6 @@ def _get_int_bins( def _parse_file_header(file_path: str): file_header = dict() - tz = pytz.timezone("Europe/Zurich") with open(file_path, encoding=get_file_encoding(file_path)) as f: for line in f: @@ -308,12 +306,12 @@ def _parse_file_header(file_path: str): if key == "CLASS" and value == "SPECIAL10": value = "SWISS10" if key in ["STARTREC", "STOPREC"]: - value = tz.localize(datetime.strptime(value, "%H:%M %d/%m/%y")) + value = TZ.localize(datetime.strptime(value, "%H:%M %d/%m/%y")) file_header[key] = value # MetroCount elif line.startswith("MetroCount"): file_header["FORMAT"] = "MC" - elif line.startswith("Place"): + elif line.startswith("Place") and "SITE" not in file_header: file_header["SITE"] = line[line.find("[") + 1 : line.find("]")].replace( "-", "" ) @@ -322,11 +320,11 @@ def _parse_file_header(file_path: str): and file_header["FORMAT"] == "MC" and "STARTREC" not in file_header ): - file_header["STARTREC"] = tz.localize( + file_header["STARTREC"] = TZ.localize( datetime.strptime(line[:19], "%Y-%m-%d %H:%M:%S") ) elif line.startswith("20") and file_header["FORMAT"] == "MC": - file_header["STOPREC"] = tz.localize( + file_header["STOPREC"] = TZ.localize( datetime.strptime(line[:19], "%Y-%m-%d %H:%M:%S") ) elif line.startswith("Type de Cat") and file_header["FORMAT"] == "MC": @@ -337,8 +335,12 @@ def _parse_file_header(file_path: str): file_header["CLASS"] = "NZ13" elif file_header["CLASS"][:5] == "FHWA ": file_header["CLASS"] = "FHWA13" - elif file_header["CLASS"] == "CAT-Cycle_dist-empat": - file_header["CLASS"] = "SPCH-MD 5C" + elif file_header["CLASS"] in ( + "CAT-Cycle_dist-empat", + "SPCH-MD5C", + "SPCH-MD 5C", + ): + file_header["CLASS"] = "SPCH-MD_5C" return file_header diff --git a/comptages/core/importer_task.py b/comptages/core/importer_task.py index 82aba356..0f95ecbd 100644 --- a/comptages/core/importer_task.py +++ b/comptages/core/importer_task.py @@ -30,14 +30,16 @@ def run(self): def finished(self, result: Any): if result: QgsMessageLog.logMessage( - "{} - Import file {} ended".format(datetime.now(), self.basename), + "{} - Task import for file {} created".format( + datetime.now(), self.basename + ), "Comptages", Qgis.Info, ) else: QgsMessageLog.logMessage( - "{} - Import file {} ended with errors: {}".format( + "{} - Task import creation for file {} ended with errors: {}".format( datetime.now(), self.basename, self.exception ), "Comptages", diff --git a/comptages/core/layers.py b/comptages/core/layers.py index 511a7c5d..8f24f57e 100644 --- a/comptages/core/layers.py +++ b/comptages/core/layers.py @@ -450,7 +450,7 @@ def get_counts_of_section_by_year(self, section_id, year): except StopIteration: return [] - print(counts) + print(f"get_counts_of_section_by_year - counts:{counts}") return counts def get_lanes_of_section(self, section_id): @@ -982,6 +982,15 @@ def get_classes_of_section(self, section_id: str): return result + def get_years_of_counts_on_section(self, section_id: str): + result = list() + counts = self.get_counts_of_section(section_id) + + for count in counts: + result.append((count.attribute("end_process_date")).year()) + + return result + def check_sensor_of_lane(self, lane_id: str): """Check id a lane is registered in the sensor table""" diff --git a/comptages/core/report.py b/comptages/core/report.py index ce9ca007..e1b79403 100644 --- a/comptages/core/report.py +++ b/comptages/core/report.py @@ -11,9 +11,8 @@ from comptages.datamodel import models -def simple_print_callback(progress: int): - if progress > 0: - print(f"Generating report... {progress}%") +def simple_print_callback(progress): + print(f"{datetime.now()}: Generating report... {progress}%") def prepare_reports( @@ -23,6 +22,7 @@ def prepare_reports( template="default", sections_ids: Optional[list[str]] = None, callback_progress=simple_print_callback, + sections_days: Optional[dict[str, list[date]]] = None, ): current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -30,14 +30,21 @@ def prepare_reports( template_name = "template.xlsx" template_path = os.path.join(current_dir, os.pardir, "report", template_name) assert count - _prepare_default_reports(file_path, count, template_path, callback_progress) + _prepare_default_reports( + file_path, count, template_path, callback_progress, sections_days + ) elif template == "yearly": template_name = "template_yearly.xlsx" template_path = os.path.join(current_dir, os.pardir, "report", template_name) assert year assert sections_ids _prepare_yearly_report( - file_path, year, template_path, sections_ids, callback_progress + file_path, + year, + template_path, + sections_ids, + callback_progress, + sections_days, ) elif template == "yearly_bike": pass @@ -56,18 +63,27 @@ def _prepare_default_reports( lane__id_installation__count=count ).distinct() assert sections + sections_qty = len(list(sections)) - mondays_qty = len(list(_mondays_of_count(count))) - mondays = _mondays_of_count(count) + mondays = list(_mondays_of_count(count)) assert mondays + mondays_qty = len(mondays) - for section in sections: + for j, section in enumerate(sections): for i, monday in enumerate(mondays): # Filter out date based on parameter if sections_days and monday not in sections_days[section.id]: continue - QgsMessageLog.logMessage("Adding to workbook", "Report", Qgis.Info) - progress = int(100 / mondays_qty * (i - 1)) + output = os.path.join( + file_path, f"{section.id}_{monday.strftime('%Y%m%d')}_r.xlsx" + ) + workbook_nbr = i + j * mondays_qty + QgsMessageLog.logMessage( + f"{datetime.now()} - Preparing reports: Adding workbook {workbook_nbr} ({output})", + "Comptages", + Qgis.Info, + ) + progress = int(100 / mondays_qty / sections_qty * workbook_nbr) callback_progress(progress) workbook = load_workbook(filename=template_path) @@ -76,9 +92,6 @@ def _prepare_default_reports( _data_speed(count, section, monday, workbook) _data_category(count, section, monday, workbook) _remove_useless_sheets(count, workbook) - output = os.path.join( - file_path, "{}_{}_r.xlsx".format(section.id, monday.strftime("%Y%m%d")) - ) workbook.save(filename=output) @@ -89,13 +102,19 @@ def _prepare_yearly_report( template_path: str, sections_ids: list[str], callback_progress, + sections_days: Optional[dict[str, list[date]]] = None, ): """Write default reports to disk (1 per section included in the count)""" # Get first count to be used as example count_qs = models.Count.objects.filter( - id_installation__lane__id_section=sections_ids[0], start_process_date__year=year + id_installation__lane__id_section=sections_ids[0], + start_process_date__year__lte=year, + end_process_date__year__gte=year, ) if not count_qs.exists(): + info_str = f"{datetime.now()}: Aucun comptage trouvé pour cette section {sections_ids[0]} et cette année {year}" + QgsMessageLog.logMessage(info_str, "Comptages", Qgis.Warning) + return count = count_qs.first() assert count @@ -231,20 +250,6 @@ def _data_count_yearly( def _data_day(count: models.Count, section: models.Section, monday, workbook: Workbook): ws = workbook["Data_day"] - # Total - row_offset = 65 - col_offset = 2 - for i in range(7): - df = statistics.get_time_data( - count, - section, - start=monday + timedelta(days=i), - end=monday + timedelta(days=i + 1), - ) - - for row in df.itertuples(): - ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) - # Monthly coefficients row_offset = 2 col_offset = 2 @@ -271,6 +276,20 @@ def _data_day(count: models.Count, section: models.Section, monday, workbook: Wo value=monthly_coefficients[day.month - 1], ) + # Total (section) + row_offset = 67 + col_offset = 2 + for i in range(7): + df = statistics.get_time_data( + count, + section, + start=monday + timedelta(days=i), + end=monday + timedelta(days=i + 1), + ) + + for row in df.itertuples(): + ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) + # Direction 1 row_offset = 5 col_offset = 2 @@ -302,7 +321,7 @@ def _data_day(count: models.Count, section: models.Section, monday, workbook: Wo # Direction 2 if len(section.lane_set.all()) == 2: - row_offset = 35 + row_offset = 36 col_offset = 2 for i in range(7): df = statistics.get_time_data( @@ -317,7 +336,7 @@ def _data_day(count: models.Count, section: models.Section, monday, workbook: Wo ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) # Light heavy direction 2 - row_offset = 60 + row_offset = 61 col_offset = 2 for i in range(7): light = statistics.get_light_numbers( @@ -345,6 +364,9 @@ def _data_day_yearly( df = statistics.get_time_data_yearly(year, section) if df is None: + print( + f"{datetime.now()}:_data_day_yearly - Pas de données pour cette section {section} et cette année {year} /!\\/!\\/!\\" + ) return for i in range(7): @@ -353,7 +375,7 @@ def _data_day_yearly( ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) # Light heavy section - row_offset = 96 + row_offset = 95 col_offset = 2 df = statistics.get_light_numbers_yearly( section, start=datetime(year, 1, 1), end=datetime(year + 1, 1, 1) @@ -386,7 +408,7 @@ def _data_day_yearly( ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) # Light heavy direction 1 - row_offset = 32 + row_offset = 31 col_offset = 2 df = statistics.get_light_numbers_yearly( section, start=datetime(year, 1, 1), end=datetime(year + 1, 1, 1), direction=1 @@ -406,7 +428,7 @@ def _data_day_yearly( # Direction 2 if len(section.lane_set.all()) == 2: - row_offset = 66 + row_offset = 37 col_offset = 2 df = statistics.get_time_data_yearly(year, section, direction=2) @@ -417,7 +439,7 @@ def _data_day_yearly( ws.cell(row=row_offset + row.hour, column=col_offset + i, value=row.thm) # Light heavy direction 2 - row_offset = 92 + row_offset = 63 col_offset = 2 df = statistics.get_light_numbers_yearly( section, diff --git a/comptages/core/report_task.py b/comptages/core/report_task.py index fdb33067..6cd92af3 100644 --- a/comptages/core/report_task.py +++ b/comptages/core/report_task.py @@ -25,7 +25,7 @@ def __init__( self.template = template self.year = year self.section_id = section_id - self.only_sections_ids = selected_sections_dates + self.sections_dates = selected_sections_dates def run(self): try: @@ -36,6 +36,7 @@ def run(self): self.template, sections_ids=[self.section_id] if self.section_id else None, callback_progress=self.setProgress, + sections_days=self.sections_dates, ) return True except Exception as e: @@ -46,16 +47,18 @@ def run(self): def finished(self, result): if result: QgsMessageLog.logMessage( - "{} - Report generation {} ended".format(datetime.now(), self.basename), + "{} - Task report generation ended for {}".format( + datetime.now(), self.file_path + ), "Comptages", Qgis.Info, ) else: QgsMessageLog.logMessage( - "{} - Report generation {} ended with errors: {}".format( - datetime.now(), self.basename, self.exception + "{} - Task report generation creation for {} ended with errors: {}".format( + datetime.now(), self.file_path, self.exception ), "Comptages", - Qgis.Info, + Qgis.Warning, ) diff --git a/comptages/core/statistics.py b/comptages/core/statistics.py index b7fa68ea..2c193775 100644 --- a/comptages/core/statistics.py +++ b/comptages/core/statistics.py @@ -1,7 +1,8 @@ from typing import Any -import pandas as pd +from pandas import DataFrame, cut from functools import reduce from datetime import timedelta, datetime +from pytz import timezone from django.db.models import F, CharField, Value, Q, Sum, QuerySet from django.db.models.functions import ExtractHour, Trunc, Concat @@ -10,6 +11,8 @@ from comptages.core import utils from comptages.datamodel import models +TZ = timezone("Europe/Zurich") + def get_time_data( count, @@ -55,7 +58,7 @@ def get_time_data( .values("import_status", "date", "hour", "thm") ) - df = pd.DataFrame.from_records(qs) + df = DataFrame.from_records(qs) if not df.empty: df["date"] = df["date"].dt.strftime("%a %d.%m.%Y") df["import_status"].replace({0: "Existant", 1: "Nouveau"}, inplace=True) @@ -64,15 +67,12 @@ def get_time_data( def get_time_data_yearly( year, section: models.Section, lane=None, direction=None -) -> pd.DataFrame: +) -> DataFrame: """Vehicles by hour and day of the week""" start = datetime(year, 1, 1) end = datetime(year + 1, 1, 1) start, end = tuple([utils.to_time_aware_utc(d) for d in (start, end)]) - print("year=", year) - print("section=", section) - # By lane/direction grouped per hour qs = models.CountDetail.objects.filter( @@ -84,15 +84,14 @@ def get_time_data_yearly( if lane is not None: qs = qs.filter(id_lane=lane) - print("lane=", lane) if direction is not None: qs = qs.filter(id_lane__direction=direction) - print("direction=", direction) - print("qs=", qs) - print("qs.count=", qs.count()) if not qs.exists(): + print( + f"statistics.py : get_time_data_yearly - Nothing found for Year: {year}, Section: {section}, Lane: {lane}, Direction: {direction}." + ) return None # Vehicles by day and hour @@ -106,15 +105,13 @@ def get_time_data_yearly( ) if not qs.exists(): print( - f"Year: {year}. Section: {section}. Lane: {lane}. Direction: {direction}. Query: {str(qs.query)}" + f"statistics.py : get_time_data_yearly - Nothing found !!! for Year: {year}. Section: {section}. Lane: {lane}. Direction: {direction}. !!!)" ) - print("qs annot=", qs) - print("qs.count=", qs.count()) - - df = pd.DataFrame.from_records(qs) - df = df.groupby([df["date"].dt.dayofweek, "hour"]).thm.sum() - df = df.reset_index() + df = DataFrame.from_records(qs) + if not df.empty: + df = df.groupby([df["date"].dt.dayofweek, "hour"]).thm.sum() + df = df.reset_index() return df @@ -128,7 +125,7 @@ def get_day_data( exclude_trash=False, start=None, end=None, -) -> tuple[pd.DataFrame, int]: +) -> tuple[DataFrame, int]: if not start: start = count.start_process_date if not end: @@ -165,15 +162,15 @@ def get_day_data( .annotate(tj=Sum("times")) .values("date", "tj", "import_status") ) + print(f"statistics.py : get_day_data - qs.query=", str(qs.query)) - df = pd.DataFrame.from_records(qs) - + df = DataFrame.from_records(qs) mean = 0 if not df.empty: mean = df["tj"].mean() df["import_status"].replace({0: "Existant", 1: "Nouveau"}, inplace=True) - return df, int(mean) + return df, mean def get_category_data( @@ -182,7 +179,7 @@ def get_category_data( status=definitions.IMPORT_STATUS_DEFINITIVE, start=None, end=None, -) -> pd.DataFrame: +) -> DataFrame: if not start: start = count.start_process_date if not end: @@ -215,8 +212,9 @@ def get_category_data( .order_by("cat_code") .values("cat_name", "cat_code", "cat_name_code", "value") ) + print(f"statistics.py : get_category_data - qs.query={str(qs.query)}") - df = pd.DataFrame.from_records(qs) + df = DataFrame.from_records(qs) return df @@ -226,7 +224,7 @@ def get_speed_data( exclude_trash=False, start=None, end=None, -) -> pd.DataFrame: +) -> DataFrame: if not start: start = count.start_process_date if not end: @@ -244,15 +242,16 @@ def get_speed_data( if exclude_trash: qs = qs.exclude(id_category__trash=True) - df = pd.DataFrame.from_records(qs.values("speed", "times", "import_status")) + print(f"statistics.py : get_speed_data - qs.query={str(qs.query)}") + df = DataFrame.from_records(qs.values("speed", "times", "import_status")) if df.empty: return df df = df.groupby( [ "import_status", - pd.cut( + cut( df["speed"], bins=[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 999], labels=[ @@ -272,7 +271,8 @@ def get_speed_data( ], right=False, # Don't include rightmost edge (e.g. bin 10-20 is actually 10-19.9999999...) ), - ] + ], + observed=False, ).sum("times") df = df.rename(columns={"speed": "speedNP"}) @@ -316,6 +316,7 @@ def get_light_numbers( .annotate(value=Sum("times")) .values_list("id_category__light", "value") ) + print(f"statistics.py : get_light_numbers - qs.query={str(qs.query)}") res = {} for r in qs: @@ -325,7 +326,7 @@ def get_light_numbers( def get_light_numbers_yearly( section: models.Section, lane=None, direction=None, start=None, end=None -) -> pd.DataFrame: +) -> DataFrame: qs = models.CountDetail.objects.filter( id_lane__id_section=section, id_category__isnull=False, @@ -341,8 +342,9 @@ def get_light_numbers_yearly( qs = qs.annotate(date=Trunc("timestamp", "day")) qs = qs.values("date", "id_category__light").annotate(value=Sum("times")) + print(f"statistics.py : get_light_numbers_yearly - qs.query={str(qs.query)}") - df = pd.DataFrame.from_records(qs) + df = DataFrame.from_records(qs) df = df.groupby([df["date"].dt.dayofweek, "id_category__light"]).value.sum() return df.reset_index() @@ -388,6 +390,7 @@ def get_speed_data_by_hour( .values("hour", "value") .values_list("hour", "value") ) + print(f"statistics.py : get_speed_data_by_hour - qs.query={str(qs.query)}") return qs @@ -400,7 +403,7 @@ def get_characteristic_speed_by_hour( start=None, end=None, v=0.15, -) -> pd.DataFrame: +) -> DataFrame: if not start: start = count.start_process_date if not end: @@ -428,8 +431,11 @@ def get_characteristic_speed_by_hour( .order_by("hour", "speed") .values("hour", "speed") ) + print( + f"statistics.py : get_characteristic_speed_by_hour - qs.query={str(qs.query)}" + ) - df = pd.DataFrame.from_records(qs.values("hour", "speed")) + df = DataFrame.from_records(qs.values("hour", "speed")) if not df.empty: df = df.set_index("hour") df = df.groupby("hour").quantile(v, interpolation="lower") @@ -444,7 +450,7 @@ def get_average_speed_by_hour( start=None, end=None, v=0.15, -) -> pd.DataFrame: +) -> DataFrame: if not start: start = count.start_process_date if not end: @@ -472,8 +478,9 @@ def get_average_speed_by_hour( .order_by("hour", "speed") .values("hour", "speed") ) + print(f"statistics.py : get_average_speed_by_hour - qs.query={str(qs.query)}") - df = pd.DataFrame.from_records(qs.values("hour", "speed")) + df = DataFrame.from_records(qs.values("hour", "speed")) if not df.empty: df = df.set_index("hour") df = df.groupby("hour").mean("speed") @@ -519,6 +526,7 @@ def get_category_data_by_hour( .values("hour", "value") .values_list("hour", "value") ) + print(f"statistics.py : get_category_data_by_hour - qs.query={str(qs.query)}") return qs @@ -528,10 +536,12 @@ def get_special_periods(first_day, last_day) -> QuerySet[models.SpecialPeriod]: Q((Q(start_date__lte=first_day) & Q(end_date__gte=last_day))) | (Q(start_date__lte=last_day) & Q(end_date__gte=first_day)) ) + print(f"statistics.py : get_special_periods - qs.query={str(qs.query)}") + return qs -def get_month_data(section: models.Section, start, end, direction=None) -> pd.DataFrame: +def get_month_data(section: models.Section, start, end, direction=None) -> DataFrame: qs = models.CountDetail.objects.filter( id_lane__id_section=section, timestamp__gte=start, timestamp__lt=end ) @@ -543,10 +553,13 @@ def get_month_data(section: models.Section, start, end, direction=None) -> pd.Da .annotate(tm=Sum("times")) .values("month", "tm", "import_status") ) + if direction is not None: qs = qs.filter(id_lane__direction=direction) - df = pd.DataFrame.from_records(qs) + print(f"statistics.py : get_month_data - qs.query={str(qs.query)}") + + df = DataFrame.from_records(qs) return df @@ -556,8 +569,8 @@ def get_valid_days(year: int, section: models.Section) -> int: where a day is deemed valid just in case there are at least 14 1-hour blocks between 6pm and 4pm with at least 1 vehicle. """ - start = datetime(year, 1, 1) - end = datetime(year + 1, 1, 1) + start = TZ.localize(datetime(year, 1, 1)) + end = TZ.localize(datetime(year + 1, 1, 1)) iterator = ( models.CountDetail.objects.filter( id_lane__id_section=section, diff --git a/comptages/metadata.txt b/comptages/metadata.txt index e82ab19f..86b289d3 100644 --- a/comptages/metadata.txt +++ b/comptages/metadata.txt @@ -2,7 +2,7 @@ name=Comptages qgisMinimumVersion=3.18 description=Canton Neuchâtel's traffic measures management -version=0.10.4.01 +version=2.1.3.4 author=OPENGIS.ch GmbH email=info@opengis.ch diff --git a/comptages/report/template.xlsx b/comptages/report/template.xlsx index c4ce664b..58fc1b64 100644 Binary files a/comptages/report/template.xlsx and b/comptages/report/template.xlsx differ diff --git a/comptages/report/template_yearly.xlsx b/comptages/report/template_yearly.xlsx index 4cd05b44..3aea05e5 100644 Binary files a/comptages/report/template_yearly.xlsx and b/comptages/report/template_yearly.xlsx differ diff --git a/comptages/report/template_yearly_bike.xlsx b/comptages/report/template_yearly_bike.xlsx index 6c499640..129b0520 100644 Binary files a/comptages/report/template_yearly_bike.xlsx and b/comptages/report/template_yearly_bike.xlsx differ diff --git a/comptages/report/yearly_report_bike.py b/comptages/report/yearly_report_bike.py index cf536588..9eea11bf 100644 --- a/comptages/report/yearly_report_bike.py +++ b/comptages/report/yearly_report_bike.py @@ -1,35 +1,45 @@ -from datetime import datetime +from datetime import datetime, timedelta from functools import reduce -import os +from os import path from typing import Any, Iterable, Optional, Union from decimal import Decimal +from qgis.core import Qgis, QgsMessageLog - -from django.db.models import Sum, F -from django.db.models.functions import Cast, TruncDate -from django.db.models.fields import DateField +from django.db.models import Sum, F, Avg # , Count, ExpressionWrapper from django.db.models.functions import ( - Cast, ExtractHour, ExtractIsoWeekDay, ExtractMonth, TruncDate, ) -from django.db.models import Sum, Avg from openpyxl import load_workbook -from comptages.core import definitions -from comptages.datamodel.models import CountDetail, Section, Lane, ClassCategory +from comptages.core import definitions, utils # , statistics +from comptages.datamodel.models import ( + CountDetail, + Section, + Lane, + ClassCategory, + Category, +) from comptages.datamodel.models import Count as modelCount class YearlyReportBike: - def __init__(self, path_to_output_dir, year, section_id): + def __init__(self, path_to_output_dir, year, section_id, classtxt): # TODO: pass section or section id? self.path_to_output_dir = path_to_output_dir self.year = year self.section_id = section_id + self.classtxt = classtxt + # Assuming seasons to run from 21 to 20 -> month20 = (date - timedelta(days=20)).month + self.seasons = { + "printemps": [3, 4, 5], + "été": [6, 7, 8], + "automne": [9, 10, 11], + "hiver": [12, 1, 2], + } def total_runs_by_directions(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": # Get all the count details for section and the year @@ -48,8 +58,31 @@ def total_runs_by_directions(self) -> "ValuesQuerySet[CountDetail, dict[str, Any .values("weekday", "id_lane__direction", "total") ) + def tjms_by_weekday_category( + self, direction=None + ) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": + # Get all the count details for section and the year + # Test/GL + qs = CountDetail.objects.filter( + id_lane__id_section__id=self.section_id, + timestamp__year=self.year, + import_status=definitions.IMPORT_STATUS_DEFINITIVE, + ) + if direction is not None: + qs = qs.filter(id_lane__direction=direction) + + # Total by day of the week (1->monday, 7->sunday) and by category (0->14) + results = ( + qs.annotate(weekday=ExtractIsoWeekDay("timestamp")) + .values("weekday") + .annotate(tjm=Sum("times")) + .values("weekday", "id_category__code", "tjm") + ) + return results + def tjms_by_weekday_hour(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": # Get all the count details for section and the year + # Doesn't produce correct results because of 2 aggregation qs = CountDetail.objects.filter( id_lane__id_section__id=self.section_id, timestamp__year=self.year, @@ -60,20 +93,20 @@ def tjms_by_weekday_hour(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": # real days (with sum) and then aggregate by weekday (with average) # Total by day of the week (0->monday, 6->sunday) and by hour (0->23) - result = ( + results = ( qs.annotate(date=TruncDate("timestamp")) .values("date") - .annotate(Sum("times")) - .annotate(weekday=ExtractIsoWeekDay("date")) - .values("weekday") - .annotate(tjm=Avg("times")) - .annotate(hour=ExtractHour("timestamp")) + .annotate(tj=Sum("times")) + .values("date", "tj") + .annotate(weekday=ExtractIsoWeekDay("date"), hour=ExtractHour("timestamp")) + .values("weekday", "hour") + .annotate(tjm=Avg("tj")) .values("weekday", "hour", "tjm") ) - return result + return results def total_runs_by_hour_and_direction( - self, directions=(1, 2), weekdays=(0, 1, 2, 3, 4, 5, 6) + self, directions=(1, 2), weekdays=(1, 2, 3, 4, 5, 6, 7) ) -> dict[int, Any]: # Get all the count details for section and the year qs = CountDetail.objects.filter( @@ -134,14 +167,15 @@ def reducer(acc: dict, val: dict) -> dict: if day not in acc: acc[day] = {} - if hour not in acc: - acc[day][hour] = val["runs"] + if hour not in acc[day]: + acc[day][hour] = {} + acc[day][hour] = val["runs"] return acc return reduce(reducer, results, {}) - def tota_runs_by_hour_weekday_one_direction( + def total_runs_by_hour_weekday_one_direction( self, direction: int ) -> "ValuesQuerySet[Countdetail, dict[str, Any]]": qs = CountDetail.objects.filter( @@ -150,7 +184,7 @@ def tota_runs_by_hour_weekday_one_direction( id_lane__direction=direction, import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) - return ( + results = ( qs.annotate(hour=ExtractHour("timestamp")) .values("hour") .annotate(runs=Sum("times")) @@ -158,6 +192,7 @@ def tota_runs_by_hour_weekday_one_direction( .annotate(day=ExtractIsoWeekDay("timestamp")) .order_by("day") ) + return results def tjms_by_weekday_and_month( self, @@ -173,19 +208,110 @@ def tjms_by_weekday_and_month( # real days (with sum) and then aggregate by weekday (with average) # Total by day of the week (0->monday, 6->sunday) and by month (1->12) - result = ( + results = ( qs.annotate(date=TruncDate("timestamp")) .values("date") - .annotate(Sum("times")) - .annotate(weekday=ExtractIsoWeekDay("timestamp")) - .values("weekday") - .annotate(tjm=Avg("times"), month=ExtractMonth("timestamp")) - .values("weekday", "month", "tjm") + .annotate(daily_runs=Sum("times")) + .values("date", "daily_runs") + .annotate(week_day=ExtractIsoWeekDay("timestamp")) + .values("date", "daily_runs", "week_day") + .annotate(month=ExtractMonth("timestamp")) + .values("week_day", "month", "daily_runs") ) - return result + # FIXME + # Aggregation via `values()` into `annotate()` all the way to the end result would be more performant. + builder = {} + for item in results: + if item["month"] not in builder: + builder[item["month"]] = {"month": item["month"]} + builder[item["month"]][item["week_day"]] = { + "days": 1, + "runs": item["daily_runs"], + "tjm": 0, + "week_day": item["week_day"], + } + elif item["week_day"] not in builder[item["month"]]: + builder[item["month"]][item["week_day"]] = { + "days": 1, + "runs": item["daily_runs"], + "tjm": 0, + "week_day": item["week_day"], + } + else: + builder[item["month"]][item["week_day"]]["days"] += 1 + builder[item["month"]][item["week_day"]]["runs"] += item["daily_runs"] + builder[item["month"]][item["week_day"]]["tjm"] = ( + builder[item["month"]][item["week_day"]]["runs"] + / builder[item["month"]][item["week_day"]]["days"] + ) + + return builder - def tjms_by_day(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": + def runs_by_weekday_and_month( + self, + ) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": + # Get all the count details for section and the year + qs = CountDetail.objects.filter( + id_lane__id_section__id=self.section_id, + timestamp__year=self.year, + import_status=definitions.IMPORT_STATUS_DEFINITIVE, + ) + + # Group by month, week_day + results = ( + qs.annotate( + month=ExtractMonth("timestamp"), week_day=ExtractIsoWeekDay("timestamp") + ) + .values("month", "week_day") + .annotate(daily_runs=Sum("times")) + .values("month", "week_day", "daily_runs") + ) + print( + f"yearly_report_bike.py : runs_by_weekday_and_month - results.query:{str(results.query)}" + ) + + return results + + def nb_weekday_by_month(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": + # Get all the count details for section and the year + qs = CountDetail.objects.filter( + id_lane__id_section__id=self.section_id, + timestamp__year=self.year, + import_status=definitions.IMPORT_STATUS_DEFINITIVE, + ) + + # Group by date then by month, week_day + results = ( + qs.annotate(date=TruncDate("timestamp")) + .values("date") + .annotate(daily_runs=Sum("times")) + .values("date", "daily_runs") + .annotate( + month=ExtractMonth("timestamp"), week_day=ExtractIsoWeekDay("timestamp") + ) + .values("date", "month", "week_day") + ) + print( + f"yearly_report_bike.py : nb_weekday_by_month - results.query:{str(results.query)}" + ) + + def reducer(acc: dict, item) -> dict: + + if item["month"] not in acc: + acc[item["month"]] = {} + + if item["week_day"] not in acc[item["month"]]: + acc[item["month"]][item["week_day"]] = 0 + + acc[item["month"]][item["week_day"]] += 1 + + return acc + + # Collecting + return reduce(reducer, results, {}) + + def total_runs_by_day(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": # Get all the count details for section and the year qs = CountDetail.objects.filter( id_lane__id_section__id=self.section_id, @@ -194,14 +320,14 @@ def tjms_by_day(self) -> "ValuesQuerySet[CountDetail, dict[str, Any]]": ) # Group by date - result = ( - qs.annotate(date=Cast("timestamp", DateField())) + results = ( + qs.annotate(date=TruncDate("timestamp")) .values("date") - .annotate(tjm=Sum("times")) - .values("date", "tjm") + .annotate(daily_runs=Sum("times")) + .values("date", "daily_runs") ) - return result + return results def tjms_total_runs_by_day_of_week(self) -> dict[str, Any]: # Get all the count details for section and the year @@ -210,17 +336,20 @@ def tjms_total_runs_by_day_of_week(self) -> dict[str, Any]: timestamp__year=self.year, import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) - result = ( + results = ( qs.annotate(date=TruncDate("timestamp")) .values("date") .annotate(daily_runs=Sum("times"), week_day=ExtractIsoWeekDay("timestamp")) .values("week_day", "daily_runs") .order_by("week_day") ) + print( + f"yearly_report_bike.py : tjms_total_runs_by_day_of_week - results.query:{str(results.query)}" + ) # FIXME # Aggregation via `values()` into `annotate()` all the way to the end result would be more performant. builder = {} - for item in result: + for item in results: if item["week_day"] not in builder: builder[item["week_day"]] = { "days": 1, @@ -231,7 +360,7 @@ def tjms_total_runs_by_day_of_week(self) -> dict[str, Any]: else: builder[item["week_day"]]["days"] += 1 builder[item["week_day"]]["runs"] += item["daily_runs"] - builder[item["week_day"]]["tjm"] = round( + builder[item["week_day"]]["tjm"] = ( builder[item["week_day"]]["runs"] / builder[item["week_day"]]["days"] ) @@ -266,7 +395,7 @@ def reducer(acc: dict, i: dict): return reduce(reducer, results, {}) def tjms_by_direction_bike( - self, categories, direction, weekdays=[0, 1, 2, 3, 4, 5, 6] + self, categories, direction, weekdays=[1, 2, 3, 4, 5, 6, 7] ) -> float: qs = CountDetail.objects.filter( id_lane__id_section__id=self.section_id, @@ -277,9 +406,9 @@ def tjms_by_direction_bike( import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) assert qs.exists() - + results = qs.aggregate(res=Sum("times"))["res"] # TODO: avoid the division? - return qs.aggregate(res=Sum("times"))["res"] / 365 + return results / 365 def total(self, categories=[1]) -> float: qs = CountDetail.objects.filter( @@ -287,8 +416,8 @@ def total(self, categories=[1]) -> float: id_category__code__in=categories, import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) - - return qs.aggregate(res=Sum("times"))["res"] + results = qs.aggregate(res=Sum("times"))["res"] + return results def max_day(self, categories=[1]) -> tuple[str, Any]: qs = ( @@ -297,7 +426,7 @@ def max_day(self, categories=[1]) -> tuple[str, Any]: id_category__code__in=categories, import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) - .annotate(date=Cast("timestamp", DateField())) + .annotate(date=TruncDate("timestamp")) .values("date") .annotate(total=Sum("times")) .order_by("-total") @@ -335,46 +464,12 @@ def min_month(self, categories=[1]) -> tuple[str, Any]: return qs[0]["total"], qs[0]["month"] - @staticmethod - def count_details_by_day_month(count: modelCount) -> dict[int, Any]: - # Preparing to filter out categories that don't reference the class picked out by `class_name` - class_name = "SPCH-MD 5C" - # Excluding irrelevant - categories_name_to_exclude = ("TRASH", "ELSE") - categories_ids = ( - ClassCategory.objects.filter(id_class__name=class_name) - .exclude(id_category__name__in=categories_name_to_exclude) - .values_list("id_category", flat=True) - ) - qs = ( - CountDetail.objects.filter( - id_count=count.id, id_category__in=categories_ids - ) - .annotate( - month=ExtractMonth("timestamp"), day=ExtractIsoWeekDay("timestamp") - ) - .values("month", "day") - .annotate(Sum("times")) - ) - - def reducer(acc, item): - month = item["month"] - day = item["day"] - runs = item["times__sum"] - if month not in acc: - acc[month] = {} - if day not in acc[month]: - acc[month][day] = runs - return acc - - return reduce(reducer, qs, {}) - - @staticmethod def count_details_by_various_criteria( + self, count: modelCount, ) -> dict[str, tuple["ValueQuerySet[CountDetail]", Optional[str]]]: # Preparing to filter out categories that don't reference the class picked out by `class_name` - class_name = "SPCH-MD 5C" + class_name = self.classtxt # Excluding irrelevant categories_name_to_exclude = ("TRASH", "ELSE") categories_ids = ( @@ -384,7 +479,9 @@ def count_details_by_various_criteria( ) # Base QuerySet base_qs = CountDetail.objects.filter( - id_count=count.id, id_category__in=categories_ids + id_count=count.id, + id_category__in=categories_ids, + timestamp__year=self.year, ) # Specialized QuerySets @@ -394,16 +491,17 @@ def count_details_by_various_criteria( .annotate(value=Sum("times")) ) - qs = ( + busy_date = ( base_qs.annotate( - category_name=F("id_category__name"), date=TruncDate("timestamp") + date=TruncDate("timestamp"), category_name=F("id_category__name") ) .values("date", "category_name") .annotate(value=Sum("times")) .order_by("-value") ) - busiest_date = qs.first() - least_busy_date = qs.last() + + busiest_date = busy_date.first() + least_busy_date = busy_date.last() assert busiest_date assert least_busy_date @@ -415,6 +513,7 @@ def count_details_by_various_criteria( .values("date", "category_name") .annotate(value=Sum("times")) ) + least_busy_date_row = ( base_qs.annotate( date=TruncDate("timestamp"), category_name=F("id_category__name") @@ -424,14 +523,15 @@ def count_details_by_various_criteria( .annotate(value=Sum("times")) ) - qs = ( + busy_month = ( base_qs.annotate(month=ExtractMonth("timestamp")) .values("month") .annotate(value=Sum("times")) .order_by("-value") ) - busiest_month = qs.first() - least_busy_month = qs.last() + + busiest_month = busy_month.first() + least_busy_month = busy_month.last() assert busiest_month assert least_busy_month @@ -452,7 +552,7 @@ def count_details_by_various_criteria( .annotate(value=Sum("times")) ) - qs = ( + busiest_hour = ( base_qs.annotate( category_name=F("id_category__name"), date=TruncDate("timestamp"), @@ -463,11 +563,11 @@ def count_details_by_various_criteria( .annotate(value=Sum("times")) .order_by("-value") ) - total_runs_busiest_hour_weekday = qs.exclude(week_day__gt=5) - total_runs_busiest_hour_weekend = qs.exclude(week_day__lt=6) + total_runs_busiest_hour_weekday = busiest_hour.exclude(week_day__gt=5) + total_runs_busiest_hour_weekend = busiest_hour.exclude(week_day__lt=6) busiest_weekday = total_runs_busiest_hour_weekday.first() - busiest_weekend = total_runs_busiest_hour_weekend[:2] + busiest_weekend = total_runs_busiest_hour_weekend.first() assert busiest_weekday assert busiest_weekend @@ -481,27 +581,19 @@ def count_details_by_various_criteria( ), "total_runs_busiest_hour_weekday": ( total_runs_busiest_hour_weekday, - str(busiest_weekday["date"]), + f'{busiest_weekday["date"]} {busiest_weekday["hour"]}:00', ), "total_runs_busiest_hour_weekend": ( total_runs_busiest_hour_weekend, - ", ".join(str(item["date"]) for item in busiest_weekend), + f'{busiest_weekend["date"]} {busiest_weekend["hour"]}:00', ), "total_runs_in_year": (total_runs_in_year, None), } - @staticmethod - def count_details_by_season(count: modelCount) -> dict[int, Any]: + def count_details_by_season(self, count_id) -> dict[int, Any]: """Break down count details by season x section x class""" - # Assuming seasons to run from 20 to 21 - seasons = { - "printemps": [3, 4, 5], - "été": [6, 7, 8], - "automne": [9, 10, 11], - "hiver": [12, 1, 2], - } # Preparing to filter out categories that don't reference the class picked out by `class_name` - class_name = "SPCH-MD 5C" + class_name = self.classtxt # Excluding irrelevant categories_name_to_exclude = ("TRASH", "ELSE") categories_ids = ( @@ -512,36 +604,35 @@ def count_details_by_season(count: modelCount) -> dict[int, Any]: # Getting data count_details = ( CountDetail.objects.filter( - id_count=count.id, id_category__in=categories_ids + id_count=count_id, + id_category__in=categories_ids, + timestamp__year=self.year, ) .annotate( - section=F("id_lane__id_section"), category_name=F("id_category__name") + date=TruncDate("timestamp"), + category_name=F("id_category__name"), ) - .values("id", "section", "category_name", "times", "timestamp") + .values("date", "category_name") + .annotate(value=Sum("times")) + .values("date", "category_name", "value") ) # Preparing to collect data def reducer(acc: dict, detail) -> dict: - timestamp: datetime = detail["timestamp"] + date: datetime = detail["date"] + month20 = (date - timedelta(days=20)).month - for season, _range in seasons.items(): - if timestamp.month in _range and ( - timestamp.month != _range[0] or timestamp.day >= 21 - ): - section_id = detail["section"] + for season, _range in self.seasons.items(): + if month20 in _range: category_name = detail["category_name"] - times = detail["times"] if season not in acc: acc[season] = {} if category_name not in acc[season]: - acc[season][category_name] = {} + acc[season][category_name] = 0 - if section_id not in acc[season][category_name]: - acc[season][category_name][section_id] = 0 - - acc[season][category_name][section_id] += times + acc[season][category_name] += detail["value"] break return acc @@ -573,9 +664,50 @@ def write_to_row( else: cell.value = "-" + @staticmethod + def get_category_data_by_dow( + count: modelCount, + section=None, + categoryid=None, + lane=None, + direction=None, + start=None, + end=None, + ) -> "ValuesQuerySet[models.CountDetail, Any]": + if not start: + start = count.start_process_date + if not end: + end = count.end_process_date + timedelta(days=1) + start, end = tuple([utils.to_time_aware_utc(d) for d in (start, end)]) + + qs = CountDetail.objects.filter( + id_lane__id_section=section, + id_category=categoryid, + timestamp__gte=start, + timestamp__lt=end, + ) + + if count is not None: + qs = qs.filter(id_count=count) + + if lane is not None: + qs = qs.filter(id_lane=lane) + + if direction is not None: + qs = qs.filter(id_lane__direction=direction) + + qs = ( + qs.annotate(week_day=ExtractIsoWeekDay("timestamp")) + .values("week_day", "times") + .annotate(value=Sum("times")) + .values("week_day", "value") + .values_list("week_day", "value") + ) + return qs + def run(self): - current_dir = os.path.dirname(os.path.abspath(__file__)) - template = os.path.join(current_dir, "template_yearly_bike.xlsx") + current_dir = path.dirname(path.abspath(__file__)) + template = path.join(current_dir, "template_yearly_bike.xlsx") workbook = load_workbook(filename=template) """ Data_count """ @@ -614,10 +746,10 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: import_status=definitions.IMPORT_STATUS_DEFINITIVE, ) if not count_detail.exists(): - print( - "Aucun conmptage pour cette année ({}) et cette section ({})".format( - self.year, self.section_id - ) + QgsMessageLog.logMessage( + f"{datetime.now()} - Aucun conmptage pour cette année ({self.year}) et cette section ({self.section_id})", + "Comptages", + Qgis.Info, ) return @@ -637,62 +769,17 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: ws["B11"] = lanes[0].id_section.place_name - """ AN_TE""" - - ws = workbook["AN_TE"] - - row_offset = 14 - column_offset = 1 - data = self.tjms_by_weekday_hour() - for i in data: - ws.cell( - row=i["hour"] + row_offset, - column=i["weekday"] + column_offset, - value=i["tjm"], - ) - - row_offset = 47 - column_offset = 1 - - data = self.tjms_by_weekday_and_month() - for i in data: - ws.cell( - row=i["month"] + row_offset, - column=i["weekday"] + column_offset, - value=i["tjm"], - ) - - """ CV_LV """ - # Is this superseded by the `Data_yearly_stats` tab ? - - # ws = workbook["CV_LV"] - - # ws["F12"] = self.tjms_by_direction_bike([1], 1) - # ws["G12"] = self.tjms_by_direction_bike([1], 2) - # ws["H12"] = self.tjms_by_direction_bike([2, 3, 4, 5], 1) - # ws["I12"] = self.tjms_by_direction_bike([2, 3, 4, 5], 2) - - # ws["K35"] = self.total() - # ws["J39"] = self.max_day()[0] - # ws["K39"] = self.max_day()[1] - - # ws["J40"] = self.max_month()[0] - # ws["K40"] = self.max_month()[1] - - # ws["J41"] = self.min_month()[0] - # ws["k41"] = self.min_month()[1] - """ Data_year """ ws = workbook["Data_year"] row_offset = 4 column_offset = 1 - data = self.tjms_by_day() + data = self.total_runs_by_day() row = row_offset for i in data: ws.cell(row=row, column=column_offset, value=i["date"]) - ws.cell(row=row, column=column_offset + 1, value=i["tjm"]) + ws.cell(row=row, column=column_offset + 1, value=i["daily_runs"]) row += 1 """ Data_week """ @@ -715,9 +802,7 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: # Data hour > Whole weeks print_area = ws["C5:D28"] data = self.total_runs_by_hour_and_direction(directions=(1, 2)) - for hour, row in enumerate(print_area, 1): - if hour == 24: - hour = 0 + for hour, row in enumerate(print_area, 0): for direction, cell in enumerate(row, 1): if hour not in data or direction not in data[hour]: cell.value = 0 @@ -726,10 +811,8 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: # Data hour > Weekends only print_area = ws["C37:D60"] - data = self.total_runs_by_hour_and_direction(directions=(1, 2), weekdays=(5, 6)) - for hour, row in enumerate(print_area, 1): - if hour == 24: - hour = 0 + data = self.total_runs_by_hour_and_direction(directions=(1, 2), weekdays=(6, 7)) + for hour, row in enumerate(print_area, 0): for direction, cell in enumerate(row, 1): if hour not in data or direction not in data[hour]: cell.value = 0 @@ -739,9 +822,7 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: # Data hour > Only dir 1 print_area = ws["B69:H92"] data = self.total_runs_by_hour_one_direction(1) - for hour, row in enumerate(print_area, 1): - if hour == 24: - hour = 0 + for hour, row in enumerate(print_area, 0): for day, cell in enumerate(row, 1): if day not in data or hour not in data[day]: cell.value = 0 @@ -751,9 +832,7 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: # Data hour > Only dir 2 print_area = ws["B101:H124"] data = self.total_runs_by_hour_one_direction(2) - for hour, row in enumerate(print_area, 1): - if hour == 24: - hour = 0 + for hour, row in enumerate(print_area, 0): for day, cell in enumerate(row, 1): if day not in data or hour not in data[day]: cell.value = 0 @@ -764,7 +843,7 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: ws = workbook["Data_yearly_stats"] print_area = ws["B2:G8"] - data = YearlyReportBike.count_details_by_various_criteria(count) + data = self.count_details_by_various_criteria(count) column_names = ( "VELO", "MONO", @@ -792,26 +871,134 @@ def render_section_dist(value: Union[str, Decimal, None]) -> str: column_names=column_names, ) - """ Data_class """ + """ Section Passages saisonniers pour chaque catégories Adrien""" + row_offset = 22 + column_offset = 2 - ws = workbook["Data_class"] - print_area = ws["B4:H18"] - for code, row in enumerate(print_area, 0): - for day, cell in enumerate(row, 1): - if code not in data or day not in data[code]: - cell.value = 0 - else: - cell.value = data[code][day] + data = self.count_details_by_season(count) + for saison in data: + for i, season in enumerate(self.seasons): + if saison == season: + for j, cat in enumerate(column_names): + if cat in data[saison]: + ws.cell( + column=j + column_offset, + row=i + row_offset, + value=data[saison][cat], + ) + + """ Section Passages mensuels Adrien""" + row_offset = 40 + column_offset = 1 + + data = self.runs_by_weekday_and_month() + for i in data: + ws.cell( + column=i["week_day"] + column_offset, + row=i["month"] + row_offset, + value=i["daily_runs"], + ) + + row_offset = 60 + data = self.nb_weekday_by_month() + for mois in data: + for jours in data[mois]: + ws.cell( + column=jours + column_offset, + row=mois + row_offset, + value=data[mois][jours], + ) + + """ Data_category """ + """ Direction Mario""" + ws = workbook["Data_category"] + + start = datetime(self.year, 1, 1) + end = datetime(self.year + 1, 1, 1) + section = Section(self.section_id) + + categories = ( + Category.objects.filter(countdetail__id_count=count) + .distinct() + .order_by("code") + ) + print( + f"yearly_report_bike.py : Data_category, Direction Mario, categories.query={str(categories.query)}" + ) + + # Direction 1 + row_offset = 5 + col_offset = 1 + for category in categories: + res = self.get_category_data_by_dow( + count=None, + section=self.section_id, + categoryid=category.id, + lane=None, + direction=1, + start=start, + end=end, + ) + + for row in res: + row_num = row_offset + category.code + col_num = col_offset + row[0] + val = ws.cell(row_num, col_num).value + # Add to previous value because with class convertions multiple categories can converge into a single one + value = val + row[1] if isinstance(val, (int, float)) else row[1] + + ws.cell(row=row_num, column=col_num, value=value) + + # Direction 2 + if len(section.lane_set.all()) == 2: + row_offset = 29 + col_offset = 1 + for category in categories: + res = self.get_category_data_by_dow( + None, + section, + categoryid=category.id, + direction=2, + start=start, + end=end, + ) + + for row in res: + row_num = row_offset + category.code + col_num = col_offset + row[0] + val = ws.cell(row_num, col_num).value + value = ( + val + row[1] if isinstance(val, (int, float)) else row[1] + ) # Add to previous value because with class convertions multiple categories can converge into a single one + + ws.cell(row=row_num, column=col_num, value=value) + + # Section + row_offset = 53 + col_offset = 1 + for category in categories: + res = self.get_category_data_by_dow( + count=None, + section=self.section_id, + categoryid=category.id, + lane=None, + direction=None, + start=start, + end=end, + ) - ws = workbook["AN_GR"] - ws.print_area = "A1:Z62" + for row in res: + row_num = row_offset + category.code + col_num = col_offset + row[0] + val = ws.cell(row_num, col_num).value + # Add to previous value because with class convertions multiple categories can converge into a single one + value = val + row[1] if isinstance(val, (int, float)) else row[1] - ws = workbook["CAT"] - ws.print_area = "A1:Z62" + ws.cell(row=row_num, column=col_num, value=value) # Save the file - output = os.path.join( + output = path.join( self.path_to_output_dir, "{}_{}_r.xlsx".format(self.section_id, self.year) ) workbook.save(filename=output) - print(f"Saved report to {output}") + print(f"{datetime.now()}: YRB_run - end: Saved report to {output}") diff --git a/comptages/test/test_report.py b/comptages/test/test_report.py index be224d89..4a621b85 100644 --- a/comptages/test/test_report.py +++ b/comptages/test/test_report.py @@ -222,12 +222,14 @@ def test_yearly_bike_report(self): path_to_output_dir=self.test_outputs, year=2021, section_id=sections_ids.first(), + classtxt="SPCH-MD 5C", ) report.run() def test_busiest_by_season(self): # Import test data pertaining to "mobilité douce" - installation = models.Installation.objects.get(name="00107695") + installation_name = "00107695" + installation = models.Installation.objects.get(name=installation_name) model = models.Model.objects.all().first() device = models.Device.objects.all().first() sensor_type = models.SensorType.objects.all().first() @@ -254,8 +256,26 @@ def test_busiest_by_season(self): importer.import_file(str(path_to_file), count) print("Imported 1 count files!") + models.CountDetail.objects.update(import_status=0) + print("Forced import status to 'definitive' for testing purposes") + + sections_ids = ( + models.Section.objects.filter(lane__id_installation__name=installation_name) + .distinct() + .values_list("id", flat=True) + ) + self.assertTrue(sections_ids.exists()) + + report = YearlyReportBike( + path_to_output_dir=self.test_outputs, + year=2021, + section_id=sections_ids.first(), + classtxt="SPCH-MD 5C", + ) + report.run() + # Collect count details - details = YearlyReportBike.count_details_by_season(count) + details = report.count_details_by_season(count) assert details def inspect_leaves(d, res) -> list[int]: @@ -281,13 +301,20 @@ def inspect_leaves(d, res) -> list[int]: for cell, category in zip( row, ("VELO", "MONO", "SHORT", "SPECIAL", "MULTI") ): - cell.value = sum(details[season][category].values()) + category_data = details.get(season, {}).get(category, []) + if isinstance(category_data, (list, tuple)): + cell.value = sum(details[season][category].values()) + elif isinstance(category_data, int): + cell.value = details[season][category] + else: + cell.value = 0 wb.save(path_to_outputs) - def test_busiest_by_day_month(self): + def test_busiest_by_various_criteria(self): # Import test data pertaining to "mobilité douce" - installation = models.Installation.objects.get(name="00107695") + installation_name = "00107695" + installation = models.Installation.objects.get(name=installation_name) model = models.Model.objects.all().first() device = models.Device.objects.all().first() sensor_type = models.SensorType.objects.all().first() @@ -314,54 +341,26 @@ def test_busiest_by_day_month(self): importer.import_file(str(path_to_file), count) print("Imported 1 count files!") - # Collecting count details - data = YearlyReportBike.count_details_by_day_month(count) - - # Prepare workbook - path_to_inputs = Path("comptages/report").joinpath("template_yearly_bike.xlsx") - path_to_outputs = self.test_outputs.joinpath("yearly_bike.xlsx") - wb = load_workbook(path_to_inputs) - - # Write data & save - ws = wb["Data_yearly_stats"] - print_area = ws["B31:H42"] - for row_idx, row in enumerate(print_area, 1): - for column_idx, cell in enumerate(row, 1): - cell.value = data[row_idx][column_idx] - - wb.save(path_to_outputs) - - def test_busiest_by_various_criteria(self): - # Import test data pertaining to "mobilité douce" - installation = models.Installation.objects.get(name="00107695") - model = models.Model.objects.all().first() - device = models.Device.objects.all().first() - sensor_type = models.SensorType.objects.all().first() - class_ = models.Class.objects.get(name="SPCH-MD 5C") - tz = pytz.timezone("Europe/Zurich") + models.CountDetail.objects.update(import_status=0) + print("Forced import status to 'definitive' for testing purposes") - count = models.Count.objects.create( - start_service_date=tz.localize(datetime(2021, 2, 1)), - end_service_date=tz.localize(datetime(2021, 12, 10)), - start_process_date=tz.localize(datetime(2021, 2, 10)), - end_process_date=tz.localize(datetime(2021, 12, 15)), - start_put_date=tz.localize(datetime(2021, 1, 1)), - end_put_date=tz.localize(datetime(2021, 1, 5)), - id_model=model, - id_device=device, - id_sensor_type=sensor_type, - id_class=class_, - id_installation=installation, + sections_ids = ( + models.Section.objects.filter(lane__id_installation__name=installation_name) + .distinct() + .values_list("id", flat=True) ) + self.assertTrue(sections_ids.exists()) - path_to_file = Path("/test_data").joinpath( - "64540060_Latenium_PS2021_ChMixte.txt" + report = YearlyReportBike( + path_to_output_dir=self.test_outputs, + year=2021, + section_id=sections_ids.first(), + classtxt="SPCH-MD 5C", ) - importer.import_file(str(path_to_file), count) - print("Imported 1 count files!") + report.run() # Collecting count details - data = YearlyReportBike.count_details_by_various_criteria(count) + data = report.count_details_by_various_criteria(count) # Prepare workbook path_to_inputs = Path("comptages/report").joinpath("template_yearly_bike.xlsx") diff --git a/docs/installation.md b/docs/installation.md index 3bb72acc..12a3e50f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -62,31 +62,35 @@ If starting from zero, restore a backup of opencomptages database or follow [Dev ### Deploying plugin -1. Look into `requirements.txt` file and install dependencies by opening QGIS > Python Console: +1. Make sure Django will be aware of GDAL by setting `GDAL_LIBRARY_PATH` environment variable in QGIS settings. +You'll have to find the GDAL dll path in the `bin` folder of your QGIS installation. +For instance `C:/Program Files/QGIS 3.36.3/bin/gdal309.dll`. + +2. Look into `requirements.txt` file and install dependencies by opening QGIS > Python Console: ```python subprocess.check_call(['python', '-m', 'pip', 'install', '', '']) ``` -2. Deploy to your custom qgis plugin repository: +3. Deploy to your custom qgis plugin repository: ```powershell cd .\scripts\windows .\deploy.ps1 ``` -3. Install plugin from your custom repository. +4. Install plugin from your custom repository. -4. Make sure the user in the plugin settings is owner of the database otherwise migrations will not work. +5. Make sure the user in the plugin settings is owner of the database otherwise migrations will not work. -5. Run migrations. Open Python console in QGIS: +6. Run migrations. Open Python console in QGIS: ```python from django.core.management import call_command call_command('migrate', 'comptages') ``` -6. Revert ownership of database if required +7. Revert ownership of database if required ## Recalculate TJM of the counts diff --git a/pyproject.toml b/pyproject.toml index beeeee7c..be5f4c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" package-dir = { "" = "comptages" } [project] -requires-python = ">=3.9.0,<=3.9.18" +requires-python = ">=3.9.0,<=3.9.19" name = "comptages" version = "0.1" dynamic = ["dependencies", "optional-dependencies"]