Merge pull request #2476 from ziv17/2357-add-junctions-to-suburban-wi…

…dgets

2357 add junctions to suburban widgets

.gitignore

@@ -25,6 +25,9 @@ develop-eggs
 lib
 lib64
 
+# newsflash-ifografics
+anyway-newsflash-infographics/anyway-newsflash-infographics/
+
 # Installer logs
 pip-log.txt
 

alembic/versions/7ea883c8a245_add_road_junction_km_table.py

@@ -0,0 +1,36 @@
+"""add suburban junction table
+
+Revision ID: 7ea883c8a245
+Revises: 881e7b1dba8a
+Create Date: 2023-06-04 17:43:13.170728
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7ea883c8a245'
+down_revision = '881e7b1dba8a'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# noinspection PyUnresolvedReferences
+def upgrade():
+    op.create_table("road_junction_km",
+                    sa.Column('road', sa.Integer(), nullable=False),
+                    sa.Column('non_urban_intersection', sa.Integer(), nullable=False),
+                    sa.Column('km', sa.Float(), nullable=False),
+                    sa.PrimaryKeyConstraint('road', 'non_urban_intersection')
+                    )
+    op.create_index(op.f('road_junction_km_idx'),
+                    "road_junction_km",
+                    ['road', 'non_urban_intersection'],
+                    unique=True)
+
+
+# noinspection PyUnresolvedReferences
+def downgrade():
+    op.drop_index(op.f('road_junction_km_idx'), table_name="road_junction_km")
+    op.drop_table("road_junction_km")

alembic/versions/fe08b885a23f_add_non_urban_intersection_indexes.py

@@ -0,0 +1,32 @@
+"""add non_urban_intersection indexes
+
+Revision ID: fe08b885a23f
+Revises: 7ea883c8a245
+Create Date: 2023-12-22 07:18:15.227413
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'fe08b885a23f'
+down_revision = '7ea883c8a245'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+tables = ["vehicles_markers_hebrew", "involved_markers_hebrew", "markers_hebrew"]
+
+
+def upgrade():
+    for table in tables:
+        op.create_index(f'ix_{table}_non_urban_intersection',
+                        table,
+                        ['non_urban_intersection'], unique=False
+                        )
+
+
+def downgrade():
+    for table in tables:
+        op.drop_index(op.f(f'ix_{table}_non_urban_intersection'), table_name=table)

anyway/models.py

@@ -1258,6 +1258,21 @@ def serialize(self):
         }
 
 
+class RoadJunctionKM(Base):
+    __tablename__ = "road_junction_km"
+    MAX_NAME_LEN = 100
+    road = Column(Integer(), primary_key=True, nullable=False)
+    non_urban_intersection = Column(Integer(), primary_key=True, nullable=False)
+    km = Column(Float(), nullable=False)
+
+    def serialize(self):
+        return {
+            "road": self.road,
+            "non_urban_intersection": self.non_urban_intersection,
+            "km": self.km,
+        }
+
+
 class RegisteredVehicle(Base):
     __tablename__ = "cities_vehicles_registered"
     id = Column(Integer(), primary_key=True)

anyway/parsers/__init__.py

@@ -16,6 +16,7 @@
         "road1",
         "road2",
         "road_segment_name",
+        "road_segment_id",
     ],
 }
 

anyway/parsers/cbs/executor.py

@@ -77,7 +77,6 @@
     ProviderCode,
     VehicleDamage,
     Streets,
-    SuburbanJunction,
     AccidentMarkerView,
     InvolvedView,
     InvolvedMarkerView,
@@ -569,7 +568,6 @@ def import_accidents(provider_code, accidents, streets, roads, non_urban_interse
     accidents_result = []
     for _, accident in accidents.iterrows():
         marker = create_marker(provider_code, accident, streets, roads, non_urban_intersection)
-        add_suburban_junction_from_marker(marker)
         accidents_result.append(marker)
     db.session.bulk_insert_mappings(AccidentMarker, accidents_result)
     db.session.commit()
@@ -793,8 +791,6 @@ def import_streets_into_db():
 
 yishuv_street_dict: Dict[Tuple[int, int], str] = {}
 yishuv_name_dict: Dict[Tuple[int, str], int] = {}
-suburban_junctions_dict: Dict[int, dict] = {}
-SUBURBAN_JUNCTION = "suburban_junction"
 
 
 def load_existing_streets():
@@ -842,67 +838,6 @@ def add_street_remove_name_duplicates(street: Dict[str, Any]):
         yishuv_name_dict[k] = street["street"]
 
 
-def import_suburban_junctions_into_db():
-    items = [{"non_urban_intersection": k,
-              NON_URBAN_INTERSECTION_HEBREW: fix_name_len(v[NON_URBAN_INTERSECTION_HEBREW]),
-              ROADS: v[ROADS]} for
-             k, v in suburban_junctions_dict.items()]
-    logging.debug(
-        f"Writing to db: {len(items)} suburban junctions"
-    )
-    db.session.query(SuburbanJunction).delete()
-    db.session.bulk_insert_mappings(SuburbanJunction, items)
-    db.session.commit()
-    logging.debug(f"Done.")
-
-
-def fix_name_len(name: str) -> str:
-    if not isinstance(name, str):
-        return name
-    if len(name) > SuburbanJunction.MAX_NAME_LEN:
-        logging.error(f"Suburban_junction name too long ({len(name)}>"
-                      f"{SuburbanJunction.MAX_NAME_LEN}):{name}.")
-    return name[: SuburbanJunction.MAX_NAME_LEN]
-
-def load_existing_suburban_junctions():
-    junctions: List[SuburbanJunction] = db.session.query(SuburbanJunction).all()
-    for j in junctions:
-        add_suburban_junction(j)
-    logging.debug(f"Loaded suburban junctions: {len(suburban_junctions_dict)}.")
-
-
-def add_suburban_junction(added: SuburbanJunction):
-    if added.non_urban_intersection in suburban_junctions_dict:
-        existing_junction = suburban_junctions_dict[added.non_urban_intersection]
-        added_heb = added.non_urban_intersection_hebrew
-        if existing_junction[NON_URBAN_INTERSECTION_HEBREW] != added_heb and added_heb is not None:
-            logging.error(
-                f"Duplicate non-urban intersection name: {added.non_urban_intersection}: existing:"
-                f"{existing_junction[NON_URBAN_INTERSECTION_HEBREW]}, added: {added_heb}"
-            )
-            existing_junction[NON_URBAN_INTERSECTION_HEBREW] = added_heb
-        existing_junction[ROADS].update(set(added.roads))
-    else:
-        suburban_junctions_dict[added.non_urban_intersection] = {
-            NON_URBAN_INTERSECTION_HEBREW: added.non_urban_intersection_hebrew,
-            ROADS: set(added.roads),
-        }
-
-
-def add_suburban_junction_from_marker(marker: dict):
-    intersection = marker[NON_URBAN_INTERSECTION]
-    if intersection is not None:
-        j = SuburbanJunction()
-        j.non_urban_intersection = intersection
-        j.non_urban_intersection_hebrew = marker[NON_URBAN_INTERSECTION_HEBREW]
-        roads = set()
-        for k in ["road1", "road2"]:
-            if marker[k] is not None:
-                roads.add(marker[k])
-        j.roads = roads
-        add_suburban_junction(j)
-
-
 def delete_invalid_entries(batch_size):
     """
     deletes all markers in the database with null latitude or longitude
@@ -1177,7 +1112,6 @@ def get_file_type_and_year(file_path):
 def main(batch_size, source, load_start_year=None):
     try:
         load_existing_streets()
-        load_existing_suburban_junctions()
         total = 0
         started = datetime.now()
         if source == "s3":
@@ -1234,7 +1168,6 @@ def main(batch_size, source, load_start_year=None):
                 add_to_streets(streets)
 
         import_streets_into_db()
-        import_suburban_junctions_into_db()
 
         fill_db_geo_data()
 

anyway/parsers/suburban_junctions.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import sys
+from typing import Dict, Tuple, Iterator
+import logging
+from openpyxl import load_workbook
+from anyway.app_and_db import db
+from anyway.models import SuburbanJunction, RoadJunctionKM
+
+
+SUBURBAN_JUNCTION = "suburban_junction"
+ACCIDENTS = "accidents"
+CITIES = "cities"
+STREETS = "streets"
+ROADS = "roads"
+URBAN_INTERSECTION = "urban_intersection"
+NON_URBAN_INTERSECTION = "non_urban_intersection"
+NON_URBAN_INTERSECTION_HEBREW = "non_urban_intersection_hebrew"
+DICTIONARY = "dictionary"
+INVOLVED = "involved"
+VEHICLES = "vehicles"
+ID = "id"
+NAME = "name"
+KM = "km"
+ROAD1 = "road1"
+suburban_junctions_dict: Dict[int, dict] = {}
+# (road, junction) -> km
+road_junction_km_dict: Dict[Tuple[int, int], int] = {}
+
+
+def parse(filename):
+    read_from_file(filename)
+    import_suburban_junctions_into_db()
+    import_road_junction_km_into_db()
+
+
+def read_from_file(filename: str):
+    for j in _iter_rows(filename):
+        add_suburban_junction(j)
+        add_road_junction_km(j)
+
+
+def _iter_rows(filename) -> Iterator[dict]:
+    workbook = load_workbook(filename, read_only=True)
+    sheet = workbook["מילון צמתים לא עירוניים"]
+    rows = sheet.rows
+    first_row = next(rows)
+    headers = [
+        "ZOMET",
+        "SUG_DEREH",
+        "REHOV1_KVISH1",
+        "REHOV2_KVISH2",
+        "KM",
+        "IKS",
+        "IGREK",
+        "IDF",
+        "SHEM_ZOMET",
+        "SUG_ZOMET",
+        "KVISH_RASHI",
+        "KM_RASHI",
+        "SHNAT_ZOMET_SGIRA",
+        "MAHOZ",
+        "NAFA",
+        "EZOR_TIVI",
+        "METROPOLIN",
+        "MAAMAD_MINIZIPALI",
+        "EZOR_STAT",
+    ]
+    assert [cell.value for cell in first_row] == headers, "File does not have expected headers"
+    for row in rows:
+        # In order to ignore empty lines
+        if not row[0].value:
+            continue
+        yield {ID: row[0].value, NAME: row[8].value, ROAD1: row[2].value, KM: row[4].value}
+
+
+def add_road_junction_km(junction: dict):
+    road_junction_km_dict[(junction[ROAD1], junction[ID])] = junction[KM] / 10
+
+
+def import_suburban_junctions_into_db():
+    items = [
+        {
+            "non_urban_intersection": k,
+            NON_URBAN_INTERSECTION_HEBREW: fix_name_len(v[NON_URBAN_INTERSECTION_HEBREW]),
+            ROADS: v[ROADS],
+        }
+        for k, v in suburban_junctions_dict.items()
+    ]
+    logging.debug(f"Writing to db: {len(items)} suburban junctions")
+    db.session.query(SuburbanJunction).delete()
+    db.session.bulk_insert_mappings(SuburbanJunction, items)
+    db.session.commit()
+    logging.debug(f"Done writing SuburbanJunction.")
+
+
+def import_road_junction_km_into_db():
+    items = [
+        {"road": k[0], "non_urban_intersection": k[1], "km": v}
+        for k, v in road_junction_km_dict.items()
+    ]
+    logging.debug(f"Writing to db: {len(items)} road junction km rows")
+    db.session.query(RoadJunctionKM).delete()
+    db.session.bulk_insert_mappings(RoadJunctionKM, items)
+    db.session.commit()
+    logging.debug(f"Done writing RoadJunctionKM.")
+
+
+def fix_name_len(name: str) -> str:
+    if not isinstance(name, str):
+        return name
+    if len(name) > SuburbanJunction.MAX_NAME_LEN:
+        logging.error(
+            f"Suburban_junction name too long ({len(name)}>"
+            f"{SuburbanJunction.MAX_NAME_LEN}):{name}."
+        )
+    return name[: SuburbanJunction.MAX_NAME_LEN]
+
+
+def add_suburban_junction(junction: dict):
+    j_id = junction[ID]
+    j_name = junction[NAME]
+    road1 = junction[ROAD1]
+    if j_id in suburban_junctions_dict:
+        existing_junction = suburban_junctions_dict[j_id]
+        existing_junction[ROADS].add(road1)
+    else:
+        suburban_junctions_dict[j_id] = {
+            NON_URBAN_INTERSECTION_HEBREW: j_name,
+            ROADS: {road1},
+        }
+
+
+if __name__ == "__main__":
+    parse(sys.argv[1])

anyway/request_params.py

@@ -218,7 +218,7 @@ def extract_road_segment_location(road_segment_id):
     road1, road_segment_name = get_road_segment_name_and_number(road_segment_id)
     data["road1"] = int(road1)
     data["road_segment_name"] = road_segment_name
-    data["road_segment_id"] = road_segment_id
+    data["road_segment_id"] = int(road_segment_id)
     text = get_road_segment_location_text(road1, road_segment_name)
     # fake gps - todo: fix
     gps = {"lat": 32.825610, "lon": 35.165395}
@@ -309,10 +309,12 @@ def fill_missing_non_urban_intersection_values(vals: dict) -> dict:
     else:
         raise ValueError(f"Cannot get non_urban_intersection from input: {vals}")
     #   TODO: temporarily removing "roads" field, as it is not used correctly in the filters.
-    if res.get("road1") is None or res.get("road2") is None and len(res.get("roads")) > 2:
+    if res.get("road1") is None or res.get("road2") is None:
         roads = list(res["roads"])
-        res["road1"] = roads[0]
-        res["road2"] = roads[1]
+        if len(roads) > 0:
+            res["road1"] = roads[0]
+        if len(roads) > 1:
+            res["road2"] = roads[1]
     if "roads" in res:
         res.pop("roads")
     return res

anyway/widgets/all_locations_widgets/accident_count_by_severity_widget.py

@@ -98,7 +98,7 @@ def get_transcription(request_params: RequestParams, items: Dict):
                 segment_name=_(request_params.location_info.get('road_segment_name')),
             )
         else:
-            raise Exception(f"cannot convert to hebrew for resolution : {request_params.resolution.get('resolution')}")
+            raise Exception(f"cannot convert to hebrew for resolution : {request_params.resolution.value}")
         text += "{between_years_keyword} {start_year} - {end_year}, {separator_keyword} {incidents_num} {incident_keyword}, {out_of_them_keywoard} ".format(
             between_years_keyword=_("between the years"),
             start_year=request_params.start_time.year,
@@ -170,4 +170,4 @@ def localize_items(request_params: RequestParams, items: Dict) -> Dict:
 _("out of them")
 _("accidents")
 _(" and ")
-_("in the selected time")
+_("in the selected time")