Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

changed single polygon to merged polygon POI if uploaded multi polygon during project area edit #829

Merged
merged 9 commits into from
Sep 25, 2023
Merged

changed single polygon to merged polygon POI if uploaded multi polygon during project area edit #829

Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d63b9ec
Merge remote-tracking branch 'upstream/feat-updateSchemas' into fix-x…
Sep 5, 2023
308a0d9
Merge remote-tracking branch 'upstream/development' into fix-xlsforms
Sep 11, 2023
3a8778c
Merge remote-tracking branch 'upstream/development' into fix-xlsforms
Sep 12, 2023
970d056
fix: changed single polygon to merged polygon POI if uploaded multi p…
Sep 12, 2023
1b32c1b
fix: changed single polygon to merged polygon POI if uploaded multi p…
Sep 12, 2023
0b338b4
fix: changed single polygon to multipolygon preview for multipolygon …
Sep 12, 2023
fa14cab
fix: multi polygon task in custom data extract using area as task for…
Sep 14, 2023
0e1736b
Merge branch 'fix-xlsforms' of github.com:Sujanadh/fmtm into fix-xlsf…
Sep 14, 2023
b2caa44
fix: added missing try except block
Sep 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
318 changes: 176 additions & 142 deletions src/backend/app/projects/project_crud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,10 +487,20 @@ def remove_z_dimension(coord):
)

""" Apply the lambda function to each coordinate in its geometry ro remove the z-dimension - if it exists"""
multi_polygons = []
for feature in features:
list(map(remove_z_dimension, feature["geometry"]["coordinates"][0]))
if feature["geometry"]["type"] == "MultiPolygon":
multi_polygons.append(Polygon(feature["geometry"]["coordinates"][0][0]))


boundary = shape(features[0]["geometry"])
"""Update the boundary polyon on the database."""
if multi_polygons:
boundary = multi_polygons[0]
for geom in multi_polygons[1:]:
boundary = boundary.union(geom)
else:
boundary = shape(features[0]["geometry"])

minx, miny, maxx, maxy = boundary.bounds

Expand Down Expand Up @@ -830,15 +840,20 @@ def remove_z_dimension(coord):
)

""" Apply the lambda function to each coordinate in its geometry """
multi_polygons = []
for feature in features:
list(map(remove_z_dimension, feature["geometry"]["coordinates"][0]))
if feature["geometry"]["type"] == "MultiPolygon":
multi_polygons.append(Polygon(feature["geometry"]["coordinates"][0][0]))


"""Update the boundary polyon on the database."""
outline = shape(features[0]["geometry"])

# If the outline is a multipolygon, use the first polygon
if isinstance(outline, MultiPolygon):
outline = outline.geoms[0]
if multi_polygons:
outline = multi_polygons[0]
for geom in multi_polygons[1:]:
outline = outline.union(geom)
else:
outline = shape(features[0]["geometry"])

db_project.outline = outline.wkt
db_project.centroid = outline.centroid.wkt
Expand Down Expand Up @@ -1141,27 +1156,46 @@ def upload_custom_data_extracts(

for feature in features_data["features"]:
feature_shape = shape(feature["geometry"])

if not (shape(project_geojson).contains(feature_shape)):
continue
if isinstance(feature_shape, MultiPolygon):
wkb_element = from_shape(Polygon(feature["geometry"]["coordinates"][0][0]), srid=4326)
else:
wkb_element = from_shape(feature_shape, srid=4326)

# If the osm extracts contents do not have a title, provide an empty text for that.
feature["properties"]["title"] = ""
properties = flatten_dict(feature["properties"])

feature_shape = shape(feature["geometry"])

wkb_element = from_shape(feature_shape, srid=4326)
feature_mapping = {
"project_id": project_id,
"geometry": wkb_element,
"properties": feature["properties"],
}
featuree = db_models.DbFeatures(**feature_mapping)
db.add(featuree)
db.commit()
db_feature = db_models.DbFeatures(
project_id=project_id,
geometry = wkb_element,
properties=properties
)
db.add(db_feature)
db.commit()

return True

def flatten_dict(d, parent_key='', sep='_'):
"""
Recursively flattens a nested dictionary into a single-level dictionary.

Args:
d (dict): The input dictionary.
parent_key (str): The parent key (used for recursion).
sep (str): The separator character to use in flattened keys.

Returns:
dict: The flattened dictionary.
"""
items = {}
for k, v in d.items():
new_key = f"{parent_key}{sep}{k}" if parent_key else k
if isinstance(v, dict):
items.update(flatten_dict(v, new_key, sep=sep))
else:
items[new_key] = v
return items


def generate_task_files(
db: Session,
Expand Down Expand Up @@ -1322,153 +1356,153 @@ def generate_appuser_files(
- form_type: weather the form is xls, xlsx or xml
- background_task_id: the task_id of the background task running this function.
"""
try:
Sujanadh marked this conversation as resolved.
Show resolved Hide resolved
project_log = log.bind(task="create_project", project_id=project_id)

project_log.info(f"Starting generate_appuser_files for project {project_id}")

# Get the project table contents.
project = table(
"projects",
column("project_name_prefix"),
column("xform_title"),
column("id"),
column("odk_central_url"),
column("odk_central_user"),
column("odk_central_password"),
column("outline"),
)
# try:
project_log = log.bind(task="create_project", project_id=project_id)

project_log.info(f"Starting generate_appuser_files for project {project_id}")

where = f"id={project_id}"
sql = select(
project.c.project_name_prefix,
project.c.xform_title,
project.c.id,
project.c.odk_central_url,
project.c.odk_central_user,
project.c.odk_central_password,
geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label("outline"),
).where(text(where))
result = db.execute(sql)
# Get the project table contents.
project = table(
"projects",
column("project_name_prefix"),
column("xform_title"),
column("id"),
column("odk_central_url"),
column("odk_central_user"),
column("odk_central_password"),
column("outline"),
)

# There should only be one match
if result.rowcount != 1:
log.warning(str(sql))
if result.rowcount < 1:
raise HTTPException(status_code=400, detail="Project not found")
else:
raise HTTPException(status_code=400, detail="Multiple projects found")
where = f"id={project_id}"
sql = select(
project.c.project_name_prefix,
project.c.xform_title,
project.c.id,
project.c.odk_central_url,
project.c.odk_central_user,
project.c.odk_central_password,
geoalchemy2.functions.ST_AsGeoJSON(project.c.outline).label("outline"),
).where(text(where))
result = db.execute(sql)

# There should only be one match
if result.rowcount != 1:
log.warning(str(sql))
if result.rowcount < 1:
raise HTTPException(status_code=400, detail="Project not found")
else:
raise HTTPException(status_code=400, detail="Multiple projects found")

one = result.first()
one = result.first()

if one:
prefix = one.project_name_prefix
if one:
prefix = one.project_name_prefix

# Get odk credentials from project.
odk_credentials = {
"odk_central_url": one.odk_central_url,
"odk_central_user": one.odk_central_user,
"odk_central_password": one.odk_central_password,
}
# Get odk credentials from project.
odk_credentials = {
"odk_central_url": one.odk_central_url,
"odk_central_user": one.odk_central_user,
"odk_central_password": one.odk_central_password,
}

odk_credentials = project_schemas.ODKCentral(**odk_credentials)
odk_credentials = project_schemas.ODKCentral(**odk_credentials)

xform_title = one.xform_title if one.xform_title else None
xform_title = one.xform_title if one.xform_title else None

if upload:
xlsform = f"/tmp/custom_form.{form_type}"
contents = upload
with open(xlsform, "wb") as f:
f.write(contents)
else:
xlsform = f"{xlsforms_path}/{xform_title}.xls"
if upload:
xlsform = f"/tmp/custom_form.{form_type}"
contents = upload
with open(xlsform, "wb") as f:
f.write(contents)
else:
xlsform = f"{xlsforms_path}/{xform_title}.xls"

category = xform_title
category = xform_title

# Data Extracts
if extracts_contents is not None:
project_log.info(f"Uploading data extracts")
upload_custom_data_extracts(db, project_id, extracts_contents)
# Data Extracts
if extracts_contents is not None:
project_log.info(f"Uploading data extracts")
upload_custom_data_extracts(db, project_id, extracts_contents)

else:
project_log.info(f"Extracting Data from OSM")

# OSM Extracts for whole project
pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass")
# This file will store osm extracts
outfile = f"/tmp/{prefix}_{xform_title}.geojson"

outline = json.loads(one.outline)
outline_geojson = pg.getFeatures(
boundary=outline,
filespec=outfile,
polygon=extract_polygon,
xlsfile=f"{category}.xls",
category=category,
)
else:
project_log.info(f"Extracting Data from OSM")

# OSM Extracts for whole project
pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass")
# This file will store osm extracts
outfile = f"/tmp/{prefix}_{xform_title}.geojson"

outline = json.loads(one.outline)
outline_geojson = pg.getFeatures(
boundary=outline,
filespec=outfile,
polygon=extract_polygon,
xlsfile=f"{category}.xls",
category=category,
)

updated_outline_geojson = {"type": "FeatureCollection", "features": []}
updated_outline_geojson = {"type": "FeatureCollection", "features": []}

# Collect feature mappings for bulk insert
feature_mappings = []
# Collect feature mappings for bulk insert
feature_mappings = []

for feature in outline_geojson["features"]:
# If the osm extracts contents do not have a title, provide an empty text for that.
feature["properties"]["title"] = ""
for feature in outline_geojson["features"]:
# If the osm extracts contents do not have a title, provide an empty text for that.
feature["properties"]["title"] = ""

feature_shape = shape(feature["geometry"])
feature_shape = shape(feature["geometry"])

# If the centroid of the Polygon is not inside the outline, skip the feature.
if extract_polygon and (
not shape(outline).contains(shape(feature_shape.centroid))
):
continue
# If the centroid of the Polygon is not inside the outline, skip the feature.
if extract_polygon and (
not shape(outline).contains(shape(feature_shape.centroid))
):
continue

wkb_element = from_shape(feature_shape, srid=4326)
feature_mapping = {
"project_id": project_id,
"category_title": category,
"geometry": wkb_element,
"properties": feature["properties"],
}
updated_outline_geojson["features"].append(feature)
feature_mappings.append(feature_mapping)
wkb_element = from_shape(feature_shape, srid=4326)
feature_mapping = {
"project_id": project_id,
"category_title": category,
"geometry": wkb_element,
"properties": feature["properties"],
}
updated_outline_geojson["features"].append(feature)
feature_mappings.append(feature_mapping)

# Bulk insert the osm extracts into the db.
db.bulk_insert_mappings(db_models.DbFeatures, feature_mappings)
# Bulk insert the osm extracts into the db.
db.bulk_insert_mappings(db_models.DbFeatures, feature_mappings)

# Generating QR Code, XForm and uploading OSM Extracts to the form.
# Creating app users and updating the role of that user.
tasks_list = tasks_crud.get_task_lists(db, project_id)
# Generating QR Code, XForm and uploading OSM Extracts to the form.
# Creating app users and updating the role of that user.
tasks_list = tasks_crud.get_task_lists(db, project_id)

# info = get_cpu_info()
# cores = info["count"]
# with concurrent.futures.ThreadPoolExecutor(max_workers=cores) as executor:
# futures = {executor.submit(generate_task_files_wrapper, project_id, task, xlsform, form_type, odk_credentials): task for task in tasks_list}
# info = get_cpu_info()
# cores = info["count"]
# with concurrent.futures.ThreadPoolExecutor(max_workers=cores) as executor:
# futures = {executor.submit(generate_task_files_wrapper, project_id, task, xlsform, form_type, odk_credentials): task for task in tasks_list}

# for future in concurrent.futures.as_completed(futures):
# log.debug(f"Waiting for thread to complete..")
# for future in concurrent.futures.as_completed(futures):
# log.debug(f"Waiting for thread to complete..")

for task in tasks_list:
try:
generate_task_files(
db, project_id, task, xlsform, form_type, odk_credentials,
)
except Exception as e:
log.warning(str(e))
continue
# # Update background task status to COMPLETED
update_background_task_status_in_database(
db, background_task_id, 4
) # 4 is COMPLETED
for task in tasks_list:
try:
generate_task_files(
db, project_id, task, xlsform, form_type, odk_credentials,
)
except Exception as e:
log.warning(str(e))
continue
# # Update background task status to COMPLETED
update_background_task_status_in_database(
db, background_task_id, 4
) # 4 is COMPLETED

except Exception as e:
log.warning(str(e))
# except Exception as e:
# log.warning(str(e))

# Update background task status to FAILED
update_background_task_status_in_database(
db, background_task_id, 2, str(e)
) # 2 is FAILED
# update_background_task_status_in_database(
# db, background_task_id, 2, str(e)
# ) # 2 is FAILED


def create_qrcode(
Expand Down
Loading