Merge pull request #265 from hotosm/fix/mvt-tiles

Fix : MVT Tiles  Zip Dir

API/api_worker.py

@@ -108,11 +108,12 @@ def zip_binding(
             "Using memory optimized zip",
         )
 
-        paths = [
-            {"fs": str(file_path), "n": file_path.name}
-            for file_path in pathlib.Path(working_dir).iterdir()
-            if file_path.is_file()
-        ]
+        paths = []
+        for root, dirs, files in os.walk(working_dir):
+            for file in files:
+                file_path = os.path.join(root, file)
+                rel_path = os.path.relpath(file_path, working_dir)
+                paths.append({"fs": str(file_path), "n": rel_path})
 
         zfly = zipfly.ZipFly(paths=paths)
         generator = zfly.generator()
@@ -129,9 +130,11 @@ def zip_binding(
             compresslevel=9,
             allowZip64=True,
         ) as zf:
-            for file_path in pathlib.Path(working_dir).iterdir():
-                if file_path.is_file():
-                    zf.write(file_path, arcname=file_path.name)
+            for root, dirs, files in os.walk(working_dir):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    rel_path = os.path.relpath(file_path, working_dir)
+                    zf.write(file_path, arcname=rel_path)
 
     logging.debug("Zip Binding Done!")
     return upload_file_path, inside_file_size

docs/src/installation/local.md

@@ -109,3 +109,12 @@ Flower dashboard should be available on port `5000` on your localhost.
 ```
 http://127.0.0.1:5000/
 ```
+
+
+
+### DEBUG
+
+If you are running in to worker crash on mac , You can use  the OBJC_DISABLE_INITIALIZE_FORK_SAFETY environment variable to disable the Objective-C runtime's fork safety checks. However use this with caution and may result to other errrors 
+```
+export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+```

src/app.py

@@ -610,11 +610,11 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params):
             format_options[RawDataOutputType.MVT.value] = {
                 "format": "MVT",
                 "extra": (
-                    "-dsco MINZOOM={} -dsco MAXZOOM={} -dsco FORMAT=MBTILES".format(
+                    "-dsco MINZOOM={} -dsco MAXZOOM={}".format(
                         params.min_zoom, params.max_zoom
                     )
                     if params.min_zoom and params.max_zoom
-                    else "-dsco MINZOOM=10 -dsco MAXZOOM=15 -dsco FORMAT=MBTILES"
+                    else "-dsco MINZOOM=10 -dsco MAXZOOM=15"
                 ),
             }
 
@@ -627,7 +627,14 @@ def ogr_export(query, outputtype, working_dir, dump_temp_path, params):
 
         format_option = format_options.get(outputtype, {"format": "", "extra": ""})
 
-        cmd = f"ogr2ogr -overwrite -f {format_option['format']} {dump_temp_path} PG:\"host={db_items.get('host')} port={db_items.get('port')} user={db_items.get('user')} dbname={db_items.get('dbname')} password={db_items.get('password')}\" -sql @{query_path} -lco ENCODING=UTF-8 -progress {format_option['extra']} {file_name_option}"
+        if format_option["format"] in [
+            "Parquet"
+        ]:  # those layers which doesn't support overwrite if layer is not present
+            begin = "ogr2ogr -f"
+        else:
+            begin = "ogr2ogr -overwrite -f"
+
+        cmd = f"{begin} {format_option['format']} {dump_temp_path} PG:\"host={db_items.get('host')} port={db_items.get('port')} user={db_items.get('user')} dbname={db_items.get('dbname')} password={db_items.get('password')}\" -sql @{query_path} -lco ENCODING=UTF-8 -progress {format_option['extra']} {file_name_option}"
         run_ogr2ogr_cmd(cmd)
 
         os.remove(query_path)
@@ -731,8 +738,9 @@ def extract_current_data(self, exportname):
 
         dump_temp_file_path = os.path.join(
             working_dir,
-            f"{self.params.file_name if self.params.file_name else 'Export'}.{output_type.lower()}",
+            f"{self.params.file_name if self.params.file_name else 'Export'}{'' if output_type == RawDataOutputType.MVT.value else f'.{output_type.lower()}'}",
         )
+
         try:
             # currently we have only geojson binding function written other than that we have depend on ogr
             if ENABLE_TILES: