Merge branch 'development' into staging

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   # Lint / autoformat: Python code
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.6.5"
+    rev: "v0.6.7"
     hooks:
       # Run the linter
       - id: ruff
@@ -71,7 +71,7 @@ repos:
 
   # Lint & Autoformat: SQL
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: 3.1.1
+    rev: 3.2.0
     hooks:
       - id: sqlfluff-lint
         files: ^src/backend/migrations/(?:.*/)*.*$

docs/dev/Production.md

@@ -209,3 +209,39 @@ pg_restore --verbose -U fmtm -d fmtm
 # Run the entire docker compose stack
 docker compose -f docker-compose.$GIT_BRANCH.yml up -d
 ```
+
+## Help! FMTM Prod Is Broken 😨
+
+### Debugging
+
+- Log into the production server, fmtm.hotosm.org and view the container logs:
+
+  ```bash
+  docker logs fmtm-main-api-1
+  docker logs fmtm-main-api-2
+  docker logs fmtm-main-api-3
+  docker logs fmtm-main-api-4
+  ```
+
+  > Note there are four replica containers running, and any one of them
+  > could have handled the request. You should check them all.
+
+  They often provide useful traceback information, including timestamps.
+
+- View error reports on Sentry: <https://humanitarian-openstreetmap-tea.sentry.io>
+  You can get very detailed tracebacks here, even with the SQL executed on
+  the database amongst other things.
+
+- Reproduce the error on your local machine!
+
+### Making a hotfix
+
+- Sometimes fixes just can't wait to go through the development -->
+  staging --> production cycle. We need the fix now!
+
+- In this case, a `hotfix` can be made directly to the `main` branch:
+  - Create a branch `hotfix/something-i-fixed`, add your code and test
+    it works locally.
+  - Push your branch, then create a PR against the `main` branch in Github.
+  - Merge in the PR and wait for the deployment.
+  - Later the code can be pulled back into develop / staging.

src/backend/app/central/central_crud.py

@@ -321,7 +321,6 @@ async def append_fields_to_user_xlsform(
     xlsform: BytesIO,
     form_category: str = "buildings",
     additional_entities: list[str] = None,
-    task_count: int = None,
     existing_id: str = None,
 ) -> tuple[str, BytesIO]:
     """Helper to return the intermediate XLSForm prior to convert."""
@@ -330,7 +329,6 @@ async def append_fields_to_user_xlsform(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
-        task_count=task_count,
         existing_id=existing_id,
     )
 
@@ -339,15 +337,13 @@ async def validate_and_update_user_xlsform(
     xlsform: BytesIO,
     form_category: str = "buildings",
     additional_entities: list[str] = None,
-    task_count: int = None,
     existing_id: str = None,
 ) -> BytesIO:
     """Wrapper to append mandatory fields and validate user uploaded XLSForm."""
     xform_id, updated_file_bytes = await append_fields_to_user_xlsform(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
-        task_count=task_count,
         existing_id=existing_id,
     )
 
@@ -361,7 +357,6 @@ async def update_project_xform(
     odk_id: int,
     xlsform: BytesIO,
     category: str,
-    task_count: int,
     odk_credentials: project_schemas.ODKCentralDecrypted,
 ) -> None:
     """Update and publish the XForm for a project.
@@ -371,7 +366,6 @@ async def update_project_xform(
         odk_id (int): ODK Central form ID.
         xlsform (UploadFile): XForm data.
         category (str): Category of the XForm.
-        task_count (int): The number of tasks in a project.
         odk_credentials (project_schemas.ODKCentralDecrypted): ODK Central creds.
 
     Returns: None

src/backend/app/projects/project_routes.py

@@ -675,7 +675,6 @@ async def validate_form(
     if debug:
         xform_id, updated_form = await central_crud.append_fields_to_user_xlsform(
             xlsform,
-            task_count=1,  # NOTE this must be included to append task_filter choices
         )
         return StreamingResponse(
             updated_form,
@@ -687,7 +686,6 @@ async def validate_form(
     else:
         await central_crud.validate_and_update_user_xlsform(
             xlsform,
-            task_count=1,  # NOTE this must be included to append task_filter choices
         )
         return JSONResponse(
             status_code=HTTPStatus.OK,
@@ -735,7 +733,6 @@ async def generate_files(
     project = project_user_dict.get("project")
     project_id = project.id
     form_category = project.xform_category
-    task_count = len(project.tasks)
 
     log.debug(f"Generating additional files for project: {project.id}")
 
@@ -746,7 +743,6 @@ async def generate_files(
         await central_crud.validate_and_update_user_xlsform(
             xlsform=xlsform_upload,
             form_category=form_category,
-            task_count=task_count,
             additional_entities=additional_entities,
         )
         xlsform = xlsform_upload
@@ -762,7 +758,6 @@ async def generate_files(
     xform_id, project_xlsform = await central_crud.append_fields_to_user_xlsform(
         xlsform=xlsform,
         form_category=form_category,
-        task_count=task_count,
         additional_entities=additional_entities,
     )
     # Write XLS form content to db

src/backend/app/submissions/submission_crud.py

@@ -465,9 +465,18 @@ async def get_submission_detail(
     """
     odk_credentials = await project_deps.get_odk_credentials(db, project.id)
     odk_form = get_odk_form(odk_credentials)
-    submission = json.loads(
-        odk_form.getSubmissions(project.odkid, project.odk_form_id, submission_id)
+
+    project_submissions = odk_form.getSubmissions(
+        project.odkid, project.odk_form_id, submission_id
     )
+    if not project_submissions:
+        log.warning("Failed to download submissions due to unknown error")
+        raise HTTPException(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            detail="Failed to download submissions",
+        )
+
+    submission = json.loads(project_submissions)
     return submission.get("value", [])[0]
 
 

src/backend/app/submissions/submission_routes.py

@@ -412,20 +412,6 @@ async def submission_table(
     return response
 
 
-@router.get("/{submission_id}")
-async def submission_detail(
-    submission_id: str,
-    db: Session = Depends(database.get_db),
-    project_user: ProjectUserDict = Depends(mapper),
-) -> dict:
-    """This api returns the submission detail of individual submission."""
-    project = project_user.get("project")
-    submission_detail = await submission_crud.get_submission_detail(
-        submission_id, project, db
-    )
-    return submission_detail
-
-
 @router.post("/update_review_state")
 async def update_review_state(
     instance_id: str,
@@ -527,6 +513,20 @@ async def conflate_geojson(
         ) from e
 
 
+@router.get("/{submission_id}")
+async def submission_detail(
+    submission_id: str,
+    db: Session = Depends(database.get_db),
+    project_user: ProjectUserDict = Depends(mapper),
+) -> dict:
+    """This api returns the submission detail of individual submission."""
+    project = project_user.get("project")
+    submission_detail = await submission_crud.get_submission_detail(
+        submission_id, project, db
+    )
+    return submission_detail
+
+
 @router.get("/{submission_id}/photos")
 async def submission_photo(
     submission_id: str,

src/backend/migrations/archived/2024-02-24/003-project-roles.sql

@@ -21,10 +21,10 @@ END $$;
 ALTER TYPE public.projectrole OWNER TO fmtm;
 
 ALTER TABLE public.user_roles
-ALTER COLUMN "role" TYPE VARCHAR(24);
+ALTER COLUMN role TYPE VARCHAR(24);
 
 ALTER TABLE public.user_roles
-ALTER COLUMN "role" TYPE public.projectrole USING role::public.projectrole;
+ALTER COLUMN role TYPE public.projectrole USING role::public.projectrole;
 
 -- Commit the transaction
 COMMIT;

src/backend/migrations/archived/2024-02-24/revert/003-project-roles.sql

@@ -2,9 +2,9 @@
 BEGIN;
 
 -- Revert user_roles table changes
-ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE VARCHAR(24);
+ALTER TABLE public.user_roles ALTER COLUMN role TYPE VARCHAR(24);
 
-ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE public.userrole
+ALTER TABLE public.user_roles ALTER COLUMN role TYPE public.userrole
 USING role::public.userrole;
 
 -- Drop the public.projectrole enum

src/backend/migrations/init/fmtm_base_schema.sql

@@ -155,11 +155,11 @@ SET default_table_access_method = heap;
 
 -- Tables
 
-CREATE TABLE IF NOT EXISTS public."_migrations" (
+CREATE TABLE IF NOT EXISTS public._migrations (
     date_executed TIMESTAMP,
     script_name TEXT
 );
-ALTER TABLE public."_migrations" OWNER TO fmtm;
+ALTER TABLE public._migrations OWNER TO fmtm;
 
 
 CREATE TABLE public.background_tasks (
@@ -193,11 +193,11 @@ ALTER TABLE public.mbtiles_path_id_seq OWNER TO fmtm;
 ALTER SEQUENCE public.mbtiles_path_id_seq OWNED BY public.mbtiles_path.id;
 
 
-CREATE TABLE public."_migrations" (
+CREATE TABLE public._migrations (
     script_name text,
     date_executed timestamp without time zone
 );
-ALTER TABLE public."_migrations" OWNER TO fmtm;
+ALTER TABLE public._migrations OWNER TO fmtm;
 
 
 CREATE TABLE public.organisation_managers (
@@ -436,8 +436,8 @@ ALTER TABLE ONLY public.submission_photos ALTER COLUMN id SET DEFAULT nextval(
 
 -- Constraints for primary keys
 
-ALTER TABLE public."_migrations"
-ADD CONSTRAINT "_migrations_pkey" PRIMARY KEY (script_name);
+ALTER TABLE public._migrations
+ADD CONSTRAINT _migrations_pkey PRIMARY KEY (script_name);
 
 ALTER TABLE ONLY public.background_tasks
 ADD CONSTRAINT background_tasks_pkey PRIMARY KEY (id);

src/backend/pyproject.toml

@@ -44,7 +44,7 @@ dependencies = [
     "cryptography>=42.0.8",
     "pyjwt>=2.8.0",
     "async-lru>=2.0.4",
-    "osm-fieldwork>=0.16.5rc0",
+    "osm-fieldwork>=0.16.5",
     "osm-login-python==2.0.0",
     "osm-rawdata==0.3.2",
     "fmtm-splitter==1.3.1",
@@ -147,7 +147,7 @@ update_changelog_on_bump = true
 [tool.sqlfluff.core]
 dialect = "postgres"
 large_file_skip_byte_limit = 30000  # Required to process fmtm_base_schema.sql
-exclude_rules = "CP05"  # Avoid capitalisation of enums
+exclude_rules = "CP05, RF02"  # Avoid capitalisation of enums
 
 [tool.codespell]
 skip = "contrib/*.py,*languages_and_countries.py,*pnpm-lock.yaml,*CHANGELOG.md"

src/frontend/src/components/DialogTaskActions.tsx

@@ -232,7 +232,7 @@ export default function Dialog({ taskId, feature }: dialogPropType) {
               );
 
               if (isMobile) {
-                document.location.href = `odkcollect://form/${projectInfo.xform_id}?task_filter=${taskId}`;
+                document.location.href = `odkcollect://form/${projectInfo.xform_id}`;
               } else {
                 dispatch(
                   CommonActions.SetSnackBar({

src/frontend/src/views/SubmissionDetails.tsx

@@ -94,11 +94,7 @@ const SubmissionDetails = () => {
   const projectDashboardLoading = useAppSelector((state) => state.project.projectDashboardLoading);
   const submissionDetails = useAppSelector((state) => state.submission.submissionDetails);
   const submissionDetailsLoading = useAppSelector((state) => state.submission.submissionDetailsLoading);
-  const taskId = submissionDetails?.task_id
-    ? submissionDetails?.task_id
-    : submissionDetails?.task_filter
-      ? submissionDetails?.task_filter
-      : '-';
+  const taskId = submissionDetails?.task_id ? submissionDetails?.task_id : '-';
 
   const { start, end, today, deviceid, ...restSubmissionDetails } = submissionDetails || {};
   const dateDeviceDetails = { start, end, today, deviceid };
@@ -158,7 +154,7 @@ const SubmissionDetails = () => {
   const newFeaturePoint = {
     type: 'Feature',
     geometry: {
-      ...restSubmissionDetails?.new_feature_point,
+      ...restSubmissionDetails?.new_feature,
     },
     properties: {},
   };
@@ -250,7 +246,7 @@ const SubmissionDetails = () => {
                 featureGeojson={
                   submissionDetailsLoading
                     ? {}
-                    : restSubmissionDetails?.new_feature === 'yes'
+                    : restSubmissionDetails?.new_feature
                       ? newFeaturePoint
                       : coordinatesArray
                         ? geojsonFeature