Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolves #36 #39

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Resolves #36 #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
208fcb4
Use GDAL to convert StringList to String.
Mtk112 Sep 20, 2024
dae7e05
Cleanup and update readme
Mtk112 Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ This plugin is available from the official plugin repository and can be installe

<img width="500" alt="Project view" src="documentation/project.png">

## Requirements

- **GDAL 3.7 or higher.**

## Usage and installation

Please read [Instructions.md](documentation/instructions.md) for more information.
Expand Down
191 changes: 91 additions & 100 deletions nlsgpkgloader/nls_geopackage_loader_tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,17 @@
# You should have received a copy of the GNU General Public License
# along with NLSgpkgloadert. If not, see <https://www.gnu.org/licenses/>.

import os
import sqlite3
from pathlib import Path

from osgeo import ogr
from osgeo import gdal, ogr
from qgis import processing
from qgis.core import (
QgsFeature,
QgsFeatureRequest,
QgsMessageLog,
QgsProcessingContext,
QgsTask,
QgsVectorFileWriter,
QgsVectorLayer,
)

Expand All @@ -43,109 +42,45 @@ def __init__(self, description, urls, dlcount, products, dlpath, path):
self.all_urls = urls
self.total_download_count = dlcount
self.products = products
self.data_download_dir = dlpath
self.gpkg_path = path
self.data_download_dir = Path(dlpath)
self.gpkg_path = Path(path)

def run(self):
for dl_index in range(0, self.total_download_count):
url = self.all_urls[dl_index][0]
url_parts = url.split("/")
file_name = url_parts[-1].split("?")[0]
data_dir_name = self.all_urls[dl_index][1]
data_dir_name = data_dir_name.replace(":", "_suhde_")
dir_path = os.path.join(self.data_download_dir, data_dir_name)
dir_path = os.path.join(dir_path, file_name.split(".")[0])
data_type = self.all_urls[dl_index][3]

percentage = dl_index / float(self.total_download_count) * 100.0
self.setProgress(percentage)

if not os.path.exists(dir_path):
QgsMessageLog.logMessage(
"Skipping directory: " + dir_path, "NLSgpkgloader", 1
gdal.UseExceptions()
all_gml_files = []
try:
for dl_index in range(0, self.total_download_count):
url = self.all_urls[dl_index][0]
url_parts = url.split("/")
file_name = url_parts[-1].split("?")[0]
data_dir_name = self.all_urls[dl_index][1]
data_dir_name = data_dir_name.replace(":", "_suhde_")
dir_path = (
self.data_download_dir / data_dir_name / file_name.split(".")[0]
)
continue
data_type = self.all_urls[dl_index][3]

for listed_file_name in os.listdir(dir_path):
if data_type == "gml" and listed_file_name.endswith(".xml"):
driver = ogr.GetDriverByName("GML")
data_source = driver.Open(
os.path.join(dir_path, listed_file_name), 0
)
layer_count = data_source.GetLayerCount()

mtk_layer_count = 0 # Used for breaking from the for loop
# when all MTK layers chosen by the user have been added
for i in range(layer_count):
if self.isCanceled():
return False
layer = data_source.GetLayerByIndex(i)
layer_name = layer.GetName()
if layer_name in self.products:
new_layer = QgsVectorLayer(
os.path.join(dir_path, listed_file_name)
+ "|layerid="
+ str(i),
layer_name,
"ogr",
)
if new_layer.isValid():
options = QgsVectorFileWriter.SaveVectorOptions()
options.layerName = layer_name
options.driverName = "GPKG"
options.fileEncoding = "UTF-8"
if os.path.isfile(self.gpkg_path):
if QgsVectorLayer(
self.gpkg_path + "|layername=" + layer_name
).isValid():
options.actionOnExistingFile = (
QgsVectorFileWriter.AppendToLayerNoNewFields
)
else:
options.actionOnExistingFile = (
QgsVectorFileWriter.CreateOrOverwriteLayer
)
else:
options.actionOnExistingFile = (
QgsVectorFileWriter.CreateOrOverwriteFile
)
e = QgsVectorFileWriter.writeAsVectorFormat(
new_layer, self.gpkg_path, options
)
if e[0]:
QgsMessageLog.logMessage(
"Failed to write layer "
+ layer_name
+ " to geopackage",
"NLSgpkgloader",
2,
)
break
mtk_layer_count += 1
else:
layer_name = ""
# TODO: handle invalid layer error
# QgsMessageLog.logMessage(
# "Invalid layer: {} : {}".format(
# listed_file_name, layer_name
# ),
# "NLSgpkgloader",
# 2,
# )
pass

if mtk_layer_count == len(self.products):
break
else:
percentage = dl_index / float(self.total_download_count) * 100.0
self.setProgress(percentage)

if not dir_path.exists():
QgsMessageLog.logMessage(
"cannot add the data type "
+ data_type
+ ", listed_file_name: "
+ listed_file_name,
"NLSgpkgloader",
0,
"Skipping directory: " + str(dir_path), "NLSgpkgloader", 1
)
return True
continue

for listed_file_name in dir_path.iterdir():
if data_type == "gml" and listed_file_name.suffix == ".xml":
all_gml_files.append(listed_file_name)

if all_gml_files:
merge_gmls(all_gml_files, self.gpkg_path)

return True

except Exception as e:
QgsMessageLog.logMessage(f"Error: {str(e)}", "NLSgpkgloader", 2)
return False

def finished(self, result):
if not result:
Expand All @@ -154,6 +89,62 @@ def finished(self, result):
)


def merge_gmls(gmls: list[Path], output: Path) -> None:
gdal.UseExceptions()

try:
gpkg_driver = ogr.GetDriverByName("GPKG")
if not output.exists():
gpkg_driver.CreateDataSource(str(output))
creation_options = gdal.VectorTranslateOptions(
layerCreationOptions=["FID=id", "GEOMETRY_NAME=geom", "SPATIAL_INDEX=NONE"],
mapFieldType="StringList=String",
)
append_options = gdal.VectorTranslateOptions(
accessMode="append",
mapFieldType="StringList=String",
)
for i, gml in enumerate(gmls):
QgsMessageLog.logMessage(f"Processing GML file: {gml}", "NLSgpkgloader", 1)

source_datasource = gdal.OpenEx(
str(gml),
nOpenFlags=gdal.OF_VECTOR,
)

if source_datasource is None:
QgsMessageLog.logMessage(
f"Failed to open GML file: {gml}", "NLSgpkgloader", 2
)
continue
QgsMessageLog.logMessage(
f"Merging into GeoPackage: {output}", "NLSgpkgloader", 1
)
options = creation_options if i == 0 else append_options

result = gdal.VectorTranslate(
destNameOrDestDS=str(output),
srcDS=source_datasource,
options=options,
)
if not result:
QgsMessageLog.logMessage(
f"Merge failed for GML file: {gml}", "NLSgpkgloader", 2
)
else:
QgsMessageLog.logMessage(
f"Successfully merged GML file: {gml} into {output}",
"NLSgpkgloader",
1,
)

except Exception as e:
QgsMessageLog.logMessage(
f"Error merging GML files: {str(e)}", "NLSgpkgloader", 2
)
raise e


class DissolveFeaturesTask(QgsTask):
def __init__(self, description, path):
super().__init__(description, QgsTask.CanCancel)
Expand Down
Loading