From 04f4593fb75066462ed43fa637578129d4539d83 Mon Sep 17 00:00:00 2001 From: signedav Date: Thu, 14 Nov 2024 14:31:28 +0100 Subject: [PATCH 1/2] On windows backslashes where created with topping exporter file paths. This leaded to issues when the relative path has been attached to an url. The IliDataCache.download_file now: - replaces the backslashes with slashes on URL - normalizes the path to the current system Still ili2db does not that. The IliToppingMaker creating those paths is now storing everything with backslashes as well, nerveless if it's an url or a file path (because it cannot know that). Windows can handel slashes in the path as well AFAIK, what means it's more stable to have slashes. --- modelbaker/dbconnector/gpkg_connector.py | 4 +++- modelbaker/dbconnector/mssql_connector.py | 6 +++++- modelbaker/dbconnector/pg_connector.py | 6 +++++- modelbaker/ilitoppingmaker/ilitarget.py | 2 +- modelbaker/iliwrapper/ilicache.py | 13 +++++++------ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 95552bb..0cf4155 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -875,7 +875,9 @@ def rename_dataset(self, tid, datasetname): return False, self.tr('Could not rename dataset to "{}".').format(datasetname) def get_topics_info(self): - if self._table_exists("T_ILI2DB_CLASSNAME"): + if self._table_exists("T_ILI2DB_CLASSNAME") and self._table_exists( + GPKG_METAATTRS_TABLE + ): cursor = self.conn.cursor() cursor.execute( """ diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index eb3f435..849a4ef 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -993,7 +993,11 @@ def rename_dataset(self, tid, datasetname): def get_topics_info(self): result = {} - if self.schema and self._table_exists("t_ili2db_classname"): + if ( + self.schema + and self._table_exists("t_ili2db_classname") + and self._table_exists(METAATTRS_TABLE) + ): cur = self.conn.cursor() cur.execute( """ diff --git a/modelbaker/dbconnector/pg_connector.py b/modelbaker/dbconnector/pg_connector.py index e3df395..80f8c37 100644 --- a/modelbaker/dbconnector/pg_connector.py +++ b/modelbaker/dbconnector/pg_connector.py @@ -1076,7 +1076,11 @@ def rename_dataset(self, tid, datasetname): return False, self.tr('Could not rename dataset "{}".').format(datasetname) def get_topics_info(self): - if self.schema and self._table_exists("t_ili2db_classname"): + if ( + self.schema + and self._table_exists("t_ili2db_classname") + and self._table_exists(PG_METAATTRS_TABLE) + ): cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur.execute( sql.SQL( diff --git a/modelbaker/ilitoppingmaker/ilitarget.py b/modelbaker/ilitoppingmaker/ilitarget.py index dd038c4..70d9359 100644 --- a/modelbaker/ilitoppingmaker/ilitarget.py +++ b/modelbaker/ilitoppingmaker/ilitarget.py @@ -62,7 +62,7 @@ def ilidata_path_resolver(target, name, type): _, relative_filedir_path = target.filedir_path(type) id = target.unique_id_in_target_scope(target, slugify(f"{type}_{name}_001")) - path = os.path.join(relative_filedir_path, name) + path = os.path.join(relative_filedir_path, name).replace("\\", "/") type = type toppingfile = {"id": id, "path": path, "type": type} target.toppingfileinfo_list.append(toppingfile) diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index b7d2a47..7c79f9b 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -685,20 +685,21 @@ def download_file(self, netloc, url, file, dataset_id=None): file_url = self.file_url(url, file) if url is None or os.path.isdir(url): - file_path = file_url + file_path = os.path.normpath(file_url) # continue with the local file - if os.path.exists(file_url): - self.file_download_succeeded.emit(dataset_id, file_url) + if os.path.exists(file_path): + self.file_download_succeeded.emit(dataset_id, file_path) else: self.file_download_failed.emit( dataset_id, - self.tr("Could not find local file {}").format(file_url), + self.tr("Could not find local file {}").format(file_path), ) else: - file_path = os.path.join(self.CACHE_PATH, netloc, file) + file_path = os.path.normpath(os.path.join(self.CACHE_PATH, netloc, file)) file_dir = os.path.dirname(file_path) os.makedirs(file_dir, exist_ok=True) - + # in case there are backslashes in the url, remove them + file_url = file_url.replace("\\", "/") download_file( file_url, file_path, From 21f64d62df792684fbab7a1e5fae0f88d132336a Mon Sep 17 00:00:00 2001 From: signedav Date: Thu, 14 Nov 2024 22:02:58 +0100 Subject: [PATCH 2/2] use pathlib to noramlize paths (means having only / instead of \) --- modelbaker/ilitoppingmaker/ilitarget.py | 5 ++++- modelbaker/iliwrapper/ilicache.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modelbaker/ilitoppingmaker/ilitarget.py b/modelbaker/ilitoppingmaker/ilitarget.py index 70d9359..51bac2c 100644 --- a/modelbaker/ilitoppingmaker/ilitarget.py +++ b/modelbaker/ilitoppingmaker/ilitarget.py @@ -19,6 +19,7 @@ import datetime import os +import pathlib from ..libs.toppingmaker import Target from ..libs.toppingmaker.utils import slugify @@ -62,7 +63,9 @@ def ilidata_path_resolver(target, name, type): _, relative_filedir_path = target.filedir_path(type) id = target.unique_id_in_target_scope(target, slugify(f"{type}_{name}_001")) - path = os.path.join(relative_filedir_path, name).replace("\\", "/") + path = pathlib.PureWindowsPath( + os.path.join(relative_filedir_path, name) + ).as_posix() type = type toppingfile = {"id": id, "path": path, "type": type} target.toppingfileinfo_list.append(toppingfile) diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index 7c79f9b..01c961a 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -19,6 +19,7 @@ import glob import logging import os +import pathlib import re import shutil import urllib.parse @@ -699,7 +700,7 @@ def download_file(self, netloc, url, file, dataset_id=None): file_dir = os.path.dirname(file_path) os.makedirs(file_dir, exist_ok=True) # in case there are backslashes in the url, remove them - file_url = file_url.replace("\\", "/") + file_url = pathlib.PureWindowsPath(file_url).as_posix() download_file( file_url, file_path,