From 71609f8afe26612e98efe99267988cdc1c77b35e Mon Sep 17 00:00:00 2001 From: SC-McD Date: Sun, 10 Dec 2023 21:51:15 +0100 Subject: [PATCH 1/5] add flexibility to backup_project_directory() --- bw2io/backup.py | 56 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/bw2io/backup.py b/bw2io/backup.py index 7d55ac36..54ea1c8a 100644 --- a/bw2io/backup.py +++ b/bw2io/backup.py @@ -34,15 +34,25 @@ def backup_data_directory(): tar.add(projects.dir, arcname=os.path.basename(projects.dir)) -def backup_project_directory(project: str): +def backup_project_directory( + project: str, timestamp: Optional[bool] = True, dir_backup: Optional[str] = None +): """ - Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory. + Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory, or a directory specified by ``dir_backup``. + + File name is of the form ``brightway2-project-{project}-backup.{timestamp}.tar.gz``, unless ``timestamp`` is False, in which case the file name is ``brightway2-project-{project}-backup.tar.gz``. Parameters ---------- project : str Name of the project to backup. + timestamp : bool, optional + If True, append a timestamp to the backup file name. + + dir_backup : str, optional + Directory to backup. If None, use the default (home)). + Returns ------- project_name : str @@ -61,25 +71,38 @@ def backup_project_directory(project: str): if project not in projects: raise ValueError("Project {} does not exist".format(project)) + dir_backup = dir_backup or os.path.expanduser("~") + + if timestamp: + timestamp = f'.{datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p")}' + else: + timestamp = "" + fp = os.path.join( - os.path.expanduser("~"), - "brightway2-project-{}-backup.{}.tar.gz".format( - project, datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p") - ), + dir_backup, f"brightway2-project-{project}-backup{timestamp}.tar.gz" ) + dir_path = os.path.join(projects._base_data_dir, safe_filename(project)) + with open(os.path.join(dir_path, ".project-name.json"), "w") as f: json.dump({"name": project}, f) + print("Creating project backup archive - this could take a few minutes...") + with tarfile.open(fp, "w:gz") as tar: tar.add(dir_path, arcname=safe_filename(project)) + print(f"Saved to: {fp}") return project -def restore_project_directory(fp: str, project_name: Optional[str] = None, overwrite_existing: Optional[bool] = False): +def restore_project_directory( + fp: str, + project_name: Optional[str] = None, + overwrite_existing: Optional[bool] = False, +): """ - Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) in the user's home directory. + Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) specified by ``fp``. Parameters ---------- @@ -101,7 +124,7 @@ def restore_project_directory(fp: str, project_name: Optional[str] = None, overw See Also -------- - bw2io.backup.backup_project_directory: To restore a project directory from a backup. + bw2io.backup.backup_project_directory: To backup a project directory. """ def get_project_name(fp): @@ -122,8 +145,8 @@ def get_project_name(fp): with tempfile.TemporaryDirectory() as td: with tarfile.open(fp, "r:gz") as tar: - def is_within_directory(directory, target): + def is_within_directory(directory, target): abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) @@ -132,7 +155,6 @@ def is_within_directory(directory, target): return prefix == abs_directory def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): @@ -143,9 +165,15 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): safe_extract(tar, td) # Find single extracted directory; don't know it ahead of time - extracted_dir = [(Path(td) / dirname) for dirname in Path(td).iterdir() if (Path(td) / dirname).is_dir()] + extracted_dir = [ + (Path(td) / dirname) + for dirname in Path(td).iterdir() + if (Path(td) / dirname).is_dir() + ] if not len(extracted_dir) == 1: - raise ValueError("Can't find single directory extracted from project archive") + raise ValueError( + "Can't find single directory extracted from project archive" + ) extracted_path = extracted_dir[0] _current = projects.current @@ -153,4 +181,6 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): shutil.copytree(extracted_path, projects.dir, dirs_exist_ok=True) projects.set_current(_current) + print(f"Restored project: {project_name}") + return project_name From 47ceb1b0f8cf160bda156f2e48d9836f862ee8aa Mon Sep 17 00:00:00 2001 From: SC-McD Date: Sun, 10 Dec 2023 21:51:15 +0100 Subject: [PATCH 2/5] update-for-CMs-comments --- bw2io/backup.py | 67 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/bw2io/backup.py b/bw2io/backup.py index 7d55ac36..66196817 100644 --- a/bw2io/backup.py +++ b/bw2io/backup.py @@ -34,15 +34,25 @@ def backup_data_directory(): tar.add(projects.dir, arcname=os.path.basename(projects.dir)) -def backup_project_directory(project: str): +def backup_project_directory( + project: str, timestamp: Optional[bool] = True, dir_backup: Optional[str] = None +): """ - Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory. + Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory, or a directory specified by ``dir_backup``. + + File name is of the form ``brightway2-project-{project}-backup.{timestamp}.tar.gz``, unless ``timestamp`` is False, in which case the file name is ``brightway2-project-{project}-backup.tar.gz``. Parameters ---------- project : str Name of the project to backup. + timestamp : bool, optional + If True, append a timestamp to the backup file name. + + dir_backup : str, optional + Directory to backup. If None, use the default (home)). + Returns ------- project_name : str @@ -61,25 +71,41 @@ def backup_project_directory(project: str): if project not in projects: raise ValueError("Project {} does not exist".format(project)) - fp = os.path.join( - os.path.expanduser("~"), - "brightway2-project-{}-backup.{}.tar.gz".format( - project, datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p") - ), + dir_backup = Path(dir_backup or Path.home()) + + # Check if the backup directory exists and is writable + if not dir_backup.exists(): + raise FileNotFoundError(f"The directory {dir_backup} does not exist.") + if not os.access(dir_backup, os.W_OK): + raise PermissionError(f"The directory {dir_backup} is not writable.") + + timestamp_str = ( + f'.{datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p")}' if timestamp else "" ) - dir_path = os.path.join(projects._base_data_dir, safe_filename(project)) - with open(os.path.join(dir_path, ".project-name.json"), "w") as f: - json.dump({"name": project}, f) + backup_filename = f"brightway2-project-{project}-backup{timestamp_str}.tar.gz" + fp = dir_backup / backup_filename + + dir_path = Path(projects._base_data_dir) / safe_filename(project) + + (dir_path / ".project-name.json").write_text(json.dumps({"name": project})) + print("Creating project backup archive - this could take a few minutes...") + with tarfile.open(fp, "w:gz") as tar: tar.add(dir_path, arcname=safe_filename(project)) + print(f"Saved to: {fp}") + return project -def restore_project_directory(fp: str, project_name: Optional[str] = None, overwrite_existing: Optional[bool] = False): +def restore_project_directory( + fp: str, + project_name: Optional[str] = None, + overwrite_existing: Optional[bool] = False, +): """ - Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) in the user's home directory. + Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) specified by ``fp``. Parameters ---------- @@ -101,7 +127,7 @@ def restore_project_directory(fp: str, project_name: Optional[str] = None, overw See Also -------- - bw2io.backup.backup_project_directory: To restore a project directory from a backup. + bw2io.backup.backup_project_directory: To backup a project directory. """ def get_project_name(fp): @@ -122,8 +148,8 @@ def get_project_name(fp): with tempfile.TemporaryDirectory() as td: with tarfile.open(fp, "r:gz") as tar: - def is_within_directory(directory, target): + def is_within_directory(directory, target): abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) @@ -132,7 +158,6 @@ def is_within_directory(directory, target): return prefix == abs_directory def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): @@ -143,9 +168,15 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): safe_extract(tar, td) # Find single extracted directory; don't know it ahead of time - extracted_dir = [(Path(td) / dirname) for dirname in Path(td).iterdir() if (Path(td) / dirname).is_dir()] + extracted_dir = [ + (Path(td) / dirname) + for dirname in Path(td).iterdir() + if (Path(td) / dirname).is_dir() + ] if not len(extracted_dir) == 1: - raise ValueError("Can't find single directory extracted from project archive") + raise ValueError( + "Can't find single directory extracted from project archive" + ) extracted_path = extracted_dir[0] _current = projects.current @@ -153,4 +184,6 @@ def safe_extract(tar, path=".", members=None, *, numeric_owner=False): shutil.copytree(extracted_path, projects.dir, dirs_exist_ok=True) projects.set_current(_current) + print(f"Restored project: {project_name}") + return project_name From 0b5e924b76eabe8bc4bfebfb706a3abb84cb06e0 Mon Sep 17 00:00:00 2001 From: SC-McD Date: Mon, 11 Dec 2023 19:04:56 +0100 Subject: [PATCH 3/5] fix --- bw2io/backup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bw2io/backup.py b/bw2io/backup.py index 3fb42570..92d1672d 100644 --- a/bw2io/backup.py +++ b/bw2io/backup.py @@ -34,7 +34,9 @@ def backup_data_directory(): tar.add(projects.dir, arcname=os.path.basename(projects.dir)) -def backup_project_directory(project: str): +def backup_project_directory( + project: str, timestamp: Optional[bool] = True, dir_backup: Optional[str] = None +): """ Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory, or a directory specified by ``dir_backup``. @@ -77,9 +79,7 @@ def backup_project_directory(project: str): if not os.access(dir_backup, os.W_OK): raise PermissionError(f"The directory {dir_backup} is not writable.") - timestamp_str = ( - f'.{datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p")}' if timestamp else "" - ) + timestamp_str = datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p") if timestamp else "" backup_filename = f"brightway2-project-{project}-backup{timestamp_str}.tar.gz" fp = dir_backup / backup_filename From bd3892bb84ba05e7e99051f450d9dc27e9493d79 Mon Sep 17 00:00:00 2001 From: SC-McD Date: Mon, 11 Dec 2023 19:04:56 +0100 Subject: [PATCH 4/5] amend-to-CM-comments --- bw2io/backup.py | 99 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/bw2io/backup.py b/bw2io/backup.py index 3fb42570..9bb2df89 100644 --- a/bw2io/backup.py +++ b/bw2io/backup.py @@ -1,5 +1,3 @@ -from pathlib import Path -from typing import Optional import codecs import datetime import json @@ -7,38 +5,81 @@ import shutil import tarfile import tempfile +from pathlib import Path +from typing import Optional, Union from bw2data import projects from bw_processing import safe_filename -def backup_data_directory(): +def backup_data_directory( + timestamp: Optional[bool] = True, dir_backup: Optional[Union[str, Path]] = None +): """ - Backup data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory. - Restoration is done manually. + Backup the Brightway2 data directory to a `.tar.gz` (compressed tar archive) in a specified directory, or in the user's home directory by default. + + The file name is of the form "brightway2-data-backup.{timestamp}.tar.gz", unless `timestamp` is False, in which case the file name is "brightway2-data-backup.tar.gz". + + Parameters + ---------- + timestamp : bool, optional + If True, append a timestamp to the backup file name. + + dir_backup : str, Path, optional + Directory to backup. If None, use the user's home directory. + + Raises + ------ + FileNotFoundError + If the backup directory does not exist. + PermissionError + If the backup directory is not writable. Examples -------- + >>> import bw2io >>> bw2io.bw2setup() >>> bw2io.backup.backup_data_directory() Creating backup archive - this could take a few minutes... """ - fp = os.path.join( - os.path.expanduser("~"), - "brightway2-data-backup.{}.tar.gz".format( - datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p") - ), + + dir_backup = Path(dir_backup or Path.home()) + + # Check if the backup directory exists and is writable + if not dir_backup.exists(): + raise FileNotFoundError(f"The directory {dir_backup} does not exist.") + if not os.access(dir_backup, os.W_OK): + raise PermissionError(f"The directory {dir_backup} is not writable.") + + # Construct the backup file name + timestamp_str = ( + f'.{datetime.datetime.now().strftime("%d-%B-%Y-%I-%M%p")}' if timestamp else "" ) - print("Creating backup archive - this could take a few minutes...") + backup_filename = f"brightway2-data-backup{timestamp_str}.tar.gz" + fp = dir_backup / backup_filename + + # Create the backup archive + print("Creating backup archive of data directory - this could take a few minutes...") with tarfile.open(fp, "w:gz") as tar: - tar.add(projects.dir, arcname=os.path.basename(projects.dir)) + data_directory = Path( + projects.dir + ) # Assuming projects.dir is a valid directory path + tar.add(data_directory, arcname=data_directory.name) + + print(f"Saved to: {fp}") + + return fp -def backup_project_directory(project: str): +def backup_project_directory( + project: str, + timestamp: Optional[bool] = True, + dir_backup: Optional[Union[str, Path]] = None, +): """ Backup project data directory to a ``.tar.gz`` (compressed tar archive) in the user's home directory, or a directory specified by ``dir_backup``. - File name is of the form ``brightway2-project-{project}-backup.{timestamp}.tar.gz``, unless ``timestamp`` is False, in which case the file name is ``brightway2-project-{project}-backup.tar.gz``. + File name is of the form ``brightway2-project-{project}-backup{timestamp}.tar.gz``, unless ``timestamp`` is False, in which case the file name is ``brightway2-project-{project}-backup.tar.gz``. Parameters ---------- @@ -48,7 +89,7 @@ def backup_project_directory(project: str): timestamp : bool, optional If True, append a timestamp to the backup file name. - dir_backup : str, optional + dir_backup : str, Path, optional Directory to backup. If None, use the default (home)). Returns @@ -60,6 +101,10 @@ def backup_project_directory(project: str): ------ ValueError If the project does not exist. + FileNotFoundError + If the backup directory does not exist. + PermissionError + If the backup directory is not writable. See Also -------- @@ -93,20 +138,21 @@ def backup_project_directory(project: str): tar.add(dir_path, arcname=safe_filename(project)) print(f"Saved to: {fp}") - return project + + return fp def restore_project_directory( - fp: str, + fp: Union[str, Path], project_name: Optional[str] = None, overwrite_existing: Optional[bool] = False, ): """ - Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) specified by ``fp``. + Restore a backed up project data directory from a ``.tar.gz`` (compressed tar archive) specified by ``fp``. Choose a custom name, or use the name of the project in the archive. If the project already exists, you must set ``overwrite_existing`` to True. Parameters ---------- - fp : str + fp : str, Path File path of the project to restore. project_name : str, optional Name of new project to create @@ -114,18 +160,24 @@ def restore_project_directory( Returns ------- - project_name : str + project_name : str, Path Name of the project that was restored. Raises ------ + FileNotFoundError + If the file path does not exist. ValueError - If the project does not exist. + If the project name cannot be found in the archive. + If the project exists and ``overwrite_existing`` is False. See Also -------- bw2io.backup.backup_project_directory: To backup a project directory. """ + fp_path = Path(fp) + if not fp_path.is_file(): + raise ValueError(f"Can't find file at path: {fp}") def get_project_name(fp): reader = codecs.getreader("utf-8") @@ -136,12 +188,13 @@ def get_project_name(fp): return json.load(reader(tar.extractfile(member)))["name"] raise ValueError("Couldn't find project name file in archive") - assert os.path.isfile(fp), "Can't find file at path: {}".format(fp) print("Restoring project backup archive - this could take a few minutes...") project_name = get_project_name(fp) if project_name is None else project_name if project_name in projects and not overwrite_existing: - raise ValueError("Project {} already exists".format(project_name)) + raise ValueError( + f"Project {project_name} already exists, set overwrite_existing=True to overwrite" + ) with tempfile.TemporaryDirectory() as td: with tarfile.open(fp, "r:gz") as tar: From 8c55b960874b05dc514cb69f5d9e73d506a40520 Mon Sep 17 00:00:00 2001 From: SC-McD Date: Mon, 11 Dec 2023 21:52:08 +0100 Subject: [PATCH 5/5] fix-path-for-datadir-backup --- bw2io/backup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bw2io/backup.py b/bw2io/backup.py index 4ec8aaaf..d72213bb 100644 --- a/bw2io/backup.py +++ b/bw2io/backup.py @@ -61,9 +61,7 @@ def backup_data_directory( # Create the backup archive print("Creating backup archive of data directory - this could take a few minutes...") with tarfile.open(fp, "w:gz") as tar: - data_directory = Path( - projects.dir - ) # Assuming projects.dir is a valid directory path + data_directory = Path(projects._base_data_dir) tar.add(data_directory, arcname=data_directory.name) print(f"Saved to: {fp}")