diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07b1a60afd..24d9688979 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,6 +97,7 @@ repos: aiida/orm/nodes/data/jsonable.py| aiida/orm/nodes/node.py| aiida/orm/nodes/process/.*py| + aiida/orm/nodes/repository.py| aiida/orm/utils/links.py| aiida/plugins/entry_point.py| aiida/plugins/factories.py| diff --git a/aiida/cmdline/commands/cmd_calcjob.py b/aiida/cmdline/commands/cmd_calcjob.py index 2d8dcdc387..119f7f1eb1 100644 --- a/aiida/cmdline/commands/cmd_calcjob.py +++ b/aiida/cmdline/commands/cmd_calcjob.py @@ -111,7 +111,7 @@ def calcjob_inputcat(calcjob, path): try: # When we `cat`, it makes sense to directly send the output to stdout as it is - with calcjob.open(path, mode='rb') as fhandle: + with calcjob.base.repository.open(path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. @@ -163,7 +163,7 @@ def calcjob_outputcat(calcjob, path): try: # When we `cat`, it makes sense to directly send the output to stdout as it is - with retrieved.open(path, mode='rb') as fhandle: + with retrieved.base.repository.open(path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. diff --git a/aiida/cmdline/commands/cmd_node.py b/aiida/cmdline/commands/cmd_node.py index 1607f992d0..28a049a5ac 100644 --- a/aiida/cmdline/commands/cmd_node.py +++ b/aiida/cmdline/commands/cmd_node.py @@ -44,7 +44,7 @@ def repo_cat(node, relative_path): import sys try: - with node.open(relative_path, mode='rb') as fhandle: + with node.base.repository.open(relative_path, mode='rb') as fhandle: copyfileobj(fhandle, sys.stdout.buffer) except OSError as exception: # The sepcial case is breakon pipe error, which is usually OK. @@ -96,7 +96,7 @@ def _copy_tree(key, output_dir): # pylint: disable=too-many-branches Recursively copy the content at the ``key`` path in the given node to the ``output_dir``. """ - for file in node.list_objects(key): + for file in node.base.repository.list_objects(key): # Not using os.path.join here, because this is the "path" # in the AiiDA node, not an actual OS - level path. file_key = file.name if not key else f'{key}/{file.name}' @@ -110,7 +110,7 @@ def _copy_tree(key, output_dir): # pylint: disable=too-many-branches assert file.file_type == FileType.FILE out_file_path = output_dir / file.name assert not out_file_path.exists() - with node.open(file_key, 'rb') as in_file: + with node.base.repository.open(file_key, 'rb') as in_file: with out_file_path.open('wb') as out_file: shutil.copyfileobj(in_file, out_file) diff --git a/aiida/cmdline/utils/repository.py b/aiida/cmdline/utils/repository.py index c507488fb8..01c18a096b 100644 --- a/aiida/cmdline/utils/repository.py +++ b/aiida/cmdline/utils/repository.py @@ -20,6 +20,6 @@ def list_repository_contents(node, path, color): """ from aiida.repository import FileType - for entry in node.list_objects(path): + for entry in node.base.repository.list_objects(path): bold = bool(entry.file_type == FileType.DIRECTORY) echo.echo(entry.name, bold=bold, fg='blue' if color and entry.file_type == FileType.DIRECTORY else None) diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py index 31d0f49e0b..e50397b147 100644 --- a/aiida/engine/daemon/execmanager.py +++ b/aiida/engine/daemon/execmanager.py @@ -175,12 +175,12 @@ def upload_calculation( for code in input_codes: if code.is_local(): # Note: this will possibly overwrite files - for filename in code.list_object_names(): + for filename in code.base.repository.list_object_names(): # Note, once #2579 is implemented, use the `node.open` method instead of the named temporary file in # combination with the new `Transport.put_object_from_filelike` # Since the content of the node could potentially be binary, we read the raw bytes and pass them on with NamedTemporaryFile(mode='wb+') as handle: - handle.write(code.get_object_content(filename, mode='rb')) + handle.write(code.base.repository.get_object_content(filename, mode='rb')) handle.flush() transport.put(handle.name, filename) transport.chmod(code.get_local_executable(), 0o755) # rwxr-xr-x @@ -212,14 +212,14 @@ def upload_calculation( filepath_target = pathlib.Path(folder.abspath) / filename_target filepath_target.parent.mkdir(parents=True, exist_ok=True) - if data_node.get_object(filename_source).file_type == FileType.DIRECTORY: + if data_node.base.repository.get_object(filename_source).file_type == FileType.DIRECTORY: # If the source object is a directory, we copy its entire contents - data_node.copy_tree(filepath_target, filename_source) - provenance_exclude_list.extend(data_node.list_object_names(filename_source)) + data_node.base.repository.copy_tree(filepath_target, filename_source) + provenance_exclude_list.extend(data_node.base.repository.list_object_names(filename_source)) else: # Otherwise, simply copy the file with folder.open(target, 'wb') as handle: - with data_node.open(filename, 'rb') as source: + with data_node.base.repository.open(filename, 'rb') as source: shutil.copyfileobj(source, handle) provenance_exclude_list.append(target) @@ -320,12 +320,12 @@ def upload_calculation( dirname not in provenance_exclude_list for dirname in dirnames ): with open(filepath, 'rb') as handle: # type: ignore[assignment] - node._repository.put_object_from_filelike(handle, relpath) # pylint: disable=protected-access + node.base.repository._repository.put_object_from_filelike(handle, relpath) # pylint: disable=protected-access # Since the node is already stored, we cannot use the normal repository interface since it will raise a # `ModificationNotAllowed` error. To bypass it, we go straight to the underlying repository instance to store the # files, however, this means we have to manually update the node's repository metadata. - node._update_repository_metadata() # pylint: disable=protected-access + node.base.repository._update_repository_metadata() # pylint: disable=protected-access if not dry_run: # Make sure that attaching the `remote_folder` with a link is the last thing we do. This gives the biggest @@ -465,7 +465,7 @@ def retrieve_calculation(calculation: CalcJobNode, transport: Transport, retriev with SandboxFolder() as folder: retrieve_files_from_list(calculation, transport, folder.abspath, retrieve_list) # Here I retrieved everything; now I store them inside the calculation - retrieved_files.put_object_from_tree(folder.abspath) + retrieved_files.base.repository.put_object_from_tree(folder.abspath) # Retrieve the temporary files in the retrieved_temporary_folder if any files were # specified in the 'retrieve_temporary_list' key diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index d815389ed8..c6fa45cfb0 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -519,7 +519,7 @@ def parse_scheduler_output(self, retrieved: orm.Node) -> Optional[ExitCode]: self.logger.warning('could not determine `stderr` filename because `scheduler_stderr` option was not set.') else: try: - scheduler_stderr = retrieved.get_object_content(filename_stderr) + scheduler_stderr = retrieved.base.repository.get_object_content(filename_stderr) except FileNotFoundError: scheduler_stderr = None self.logger.warning(f'could not parse scheduler output: the `{filename_stderr}` file is missing') @@ -528,7 +528,7 @@ def parse_scheduler_output(self, retrieved: orm.Node) -> Optional[ExitCode]: self.logger.warning('could not determine `stdout` filename because `scheduler_stdout` option was not set.') else: try: - scheduler_stdout = retrieved.get_object_content(filename_stdout) + scheduler_stdout = retrieved.base.repository.get_object_content(filename_stdout) except FileNotFoundError: scheduler_stdout = None self.logger.warning(f'could not parse scheduler output: the `{filename_stdout}` file is missing') diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index dcf7f66104..a20396ea7f 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -72,7 +72,7 @@ 'Node', 'NodeEntityLoader', 'NodeLinksManager', - 'NodeRepositoryMixin', + 'NodeRepository', 'NumericType', 'OrbitalData', 'OrderSpecifier', diff --git a/aiida/orm/implementation/storage_backend.py b/aiida/orm/implementation/storage_backend.py index 02f3c8ba29..fe3f023107 100644 --- a/aiida/orm/implementation/storage_backend.py +++ b/aiida/orm/implementation/storage_backend.py @@ -40,7 +40,7 @@ class StorageBackend(abc.ABC): # pylint: disable=too-many-public-methods - Searchable data, which is stored in the database and can be queried using the QueryBuilder - Non-searchable (binary) data, which is stored in the repository and can be loaded using the RepositoryBackend - The two sources are inter-linked by the ``Node.repository_metadata``. + The two sources are inter-linked by the ``Node.base.repository.metadata``. Once stored, the leaf values of this dictionary must be valid pointers to object keys in the repository. The class methods,`version_profile` and `migrate`, diff --git a/aiida/orm/nodes/__init__.py b/aiida/orm/nodes/__init__.py index 99e32294d6..9cd3c65371 100644 --- a/aiida/orm/nodes/__init__.py +++ b/aiida/orm/nodes/__init__.py @@ -40,7 +40,7 @@ 'KpointsData', 'List', 'Node', - 'NodeRepositoryMixin', + 'NodeRepository', 'NumericType', 'OrbitalData', 'ProcessNode', diff --git a/aiida/orm/nodes/data/array/array.py b/aiida/orm/nodes/data/array/array.py index 1bb97e94b3..43088209a4 100644 --- a/aiida/orm/nodes/data/array/array.py +++ b/aiida/orm/nodes/data/array/array.py @@ -45,11 +45,11 @@ def delete_array(self, name): :param name: The name of the array to delete from the node. """ fname = f'{name}.npy' - if fname not in self.list_object_names(): + if fname not in self.base.repository.list_object_names(): raise KeyError(f"Array with name '{name}' not found in node pk= {self.pk}") # remove both file and attribute - self.delete_object(fname) + self.base.repository.delete_object(fname) try: self.delete_attribute(f'{self.array_prefix}{name}') except (KeyError, AttributeError): @@ -71,7 +71,7 @@ def _arraynames_from_files(self): Return a list of all arrays stored in the node, listing the files (and not relying on the properties). """ - return [i[:-4] for i in self.list_object_names() if i.endswith('.npy')] + return [i[:-4] for i in self.base.repository.list_object_names() if i.endswith('.npy')] def _arraynames_from_properties(self): """ @@ -111,11 +111,11 @@ def get_array_from_file(self, name): """Return the array stored in a .npy file""" filename = f'{name}.npy' - if filename not in self.list_object_names(): + if filename not in self.base.repository.list_object_names(): raise KeyError(f'Array with name `{name}` not found in ArrayData<{self.pk}>') # Open a handle in binary read mode as the arrays are written as binary files as well - with self.open(filename, mode='rb') as handle: + with self.base.repository.open(filename, mode='rb') as handle: return numpy.load(handle, allow_pickle=False) # pylint: disable=unexpected-keyword-arg # Return with proper caching if the node is stored, otherwise always re-read from disk @@ -171,7 +171,7 @@ def set_array(self, name, array): handle.seek(0) # Write the numpy array to the repository, keeping the byte representation - self.put_object_from_filelike(handle, f'{name}.npy') + self.base.repository.put_object_from_filelike(handle, f'{name}.npy') # Store the array name and shape for querying purposes self.set_attribute(f'{self.array_prefix}{name}', list(array.shape)) diff --git a/aiida/orm/nodes/data/code.py b/aiida/orm/nodes/data/code.py index 4306ca5aaf..081d80445b 100644 --- a/aiida/orm/nodes/data/code.py +++ b/aiida/orm/nodes/data/code.py @@ -93,7 +93,7 @@ def set_files(self, files): for filename in files: if os.path.isfile(filename): with open(filename, 'rb') as handle: - self.put_object_from_filelike(handle, os.path.split(filename)[1]) + self.base.repository.put_object_from_filelike(handle, os.path.split(filename)[1]) def __str__(self): local_str = 'Local' if self.is_local() else 'Remote' @@ -282,12 +282,12 @@ def _validate(self): 'You have to set which file is the local executable ' 'using the set_exec_filename() method' ) - if self.get_local_executable() not in self.list_object_names(): + if self.get_local_executable() not in self.base.repository.list_object_names(): raise exceptions.ValidationError( f"The local executable '{self.get_local_executable()}' is not in the list of files of this code" ) else: - if self.list_object_names(): + if self.base.repository.list_object_names(): raise exceptions.ValidationError('The code is remote but it has files inside') if not self.get_remote_computer(): raise exceptions.ValidationError('You did not specify a remote computer') diff --git a/aiida/orm/nodes/data/data.py b/aiida/orm/nodes/data/data.py index 0c80acc188..f204fc2587 100644 --- a/aiida/orm/nodes/data/data.py +++ b/aiida/orm/nodes/data/data.py @@ -69,8 +69,8 @@ def clone(self): backend_clone = self.backend_entity.clone() clone = self.__class__.from_backend_entity(backend_clone) - clone.reset_attributes(copy.deepcopy(self.attributes)) # pylint: disable=no-member - clone._repository.clone(self._repository) # pylint: disable=no-member,protected-access + clone.reset_attributes(copy.deepcopy(self.attributes)) + clone.base.repository._clone(self.base.repository) # pylint: disable=protected-access return clone diff --git a/aiida/orm/nodes/data/folder.py b/aiida/orm/nodes/data/folder.py index e85a4235f6..38c679d9ef 100644 --- a/aiida/orm/nodes/data/folder.py +++ b/aiida/orm/nodes/data/folder.py @@ -37,4 +37,4 @@ def __init__(self, **kwargs): tree = kwargs.pop('tree', None) super().__init__(**kwargs) if tree: - self.put_object_from_tree(tree) + self.base.repository.put_object_from_tree(tree) diff --git a/aiida/orm/nodes/data/singlefile.py b/aiida/orm/nodes/data/singlefile.py index 452e496f86..32e6ae63ca 100644 --- a/aiida/orm/nodes/data/singlefile.py +++ b/aiida/orm/nodes/data/singlefile.py @@ -56,7 +56,7 @@ def open(self, path=None, mode='r'): if path is None: path = self.filename - with super().open(path, mode=mode) as handle: + with self.base.repository.open(path, mode=mode) as handle: yield handle def get_content(self): @@ -94,7 +94,7 @@ def set_file(self, file, filename=None): key = filename or key - existing_object_names = self.list_object_names() + existing_object_names = self.base.repository.list_object_names() try: # Remove the 'key' from the list of currently existing objects such that it is not deleted after storing @@ -103,13 +103,13 @@ def set_file(self, file, filename=None): pass if is_filelike: - self.put_object_from_filelike(file, key) + self.base.repository.put_object_from_filelike(file, key) else: - self.put_object_from_file(file, key) + self.base.repository.put_object_from_file(file, key) # Delete any other existing objects (minus the current `key` which was already removed from the list) for existing_key in existing_object_names: - self.delete_object(existing_key) + self.base.repository.delete_object(existing_key) self.set_attribute('filename', key) @@ -122,7 +122,7 @@ def _validate(self): except AttributeError: raise exceptions.ValidationError('the `filename` attribute is not set.') - objects = self.list_object_names() + objects = self.base.repository.list_object_names() if [filename] != objects: raise exceptions.ValidationError( diff --git a/aiida/orm/nodes/node.py b/aiida/orm/nodes/node.py index 29fd3c3252..19bbfeee13 100644 --- a/aiida/orm/nodes/node.py +++ b/aiida/orm/nodes/node.py @@ -9,7 +9,6 @@ ########################################################################### # pylint: disable=too-many-lines,too-many-arguments """Package for node ORM classes.""" -import copy import datetime from functools import cached_property import importlib @@ -36,6 +35,7 @@ from aiida.common.hashing import make_hash from aiida.common.lang import classproperty, type_check from aiida.common.links import LinkType +from aiida.common.warnings import warn_deprecation from aiida.manage import get_manager from aiida.orm.utils.links import LinkManager, LinkTriple from aiida.orm.utils.node import AbstractNodeMeta @@ -46,7 +46,7 @@ from ..entities import Entity, EntityAttributesMixin, EntityExtrasMixin from ..querybuilder import QueryBuilder from ..users import User -from .repository import NodeRepositoryMixin +from .repository import NodeRepository if TYPE_CHECKING: from ..implementation import BackendNode, StorageBackend @@ -60,7 +60,7 @@ class NodeCollection(EntityCollection[NodeType], Generic[NodeType]): """The collection of nodes.""" @staticmethod - def _entity_base_cls() -> Type['Node']: + def _entity_base_cls() -> Type['Node']: # type: ignore return Node def delete(self, pk: int) -> None: @@ -109,10 +109,13 @@ def __init__(self, node: 'Node') -> None: """Construct a new instance of the base namespace.""" self._node: 'Node' = node + @cached_property + def repository(self) -> 'NodeRepository': + """Return the repository for this node.""" + return NodeRepository(self._node) + -class Node( - Entity['BackendNode'], NodeRepositoryMixin, EntityAttributesMixin, EntityExtrasMixin, metaclass=AbstractNodeMeta -): +class Node(Entity['BackendNode'], EntityAttributesMixin, EntityExtrasMixin, metaclass=AbstractNodeMeta): """ Base class for all nodes in AiiDA. @@ -160,7 +163,7 @@ class Node( Collection = NodeCollection @classproperty - def objects(cls: Type[NodeType]) -> NodeCollection[NodeType]: # pylint: disable=no-self-argument + def objects(cls: Type[NodeType]) -> NodeCollection[NodeType]: # type: ignore # pylint: disable=no-self-argument return NodeCollection.get_cached(cls, get_manager().get_profile_storage()) # type: ignore[arg-type] def __init__( @@ -339,22 +342,6 @@ def description(self, value: str) -> None: """ self.backend_entity.description = value - @property - def repository_metadata(self) -> Dict[str, Any]: - """Return the node repository metadata. - - :return: the repository metadata - """ - return self.backend_entity.repository_metadata - - @repository_metadata.setter - def repository_metadata(self, value: Dict[str, Any]) -> None: - """Set the repository metadata. - - :param value: the new value to set - """ - self.backend_entity.repository_metadata = value - @property def computer(self) -> Optional[Computer]: """Return the computer of this node.""" @@ -740,18 +727,7 @@ def _store(self, with_transaction: bool = True, clean: bool = True) -> 'Node': :param with_transaction: if False, do not use a transaction because the caller will already have opened one. :param clean: boolean, if True, will clean the attributes and extras before attempting to store """ - from aiida.repository import Repository - from aiida.repository.backend import SandboxRepositoryBackend - - # Only if the backend repository is a sandbox do we have to clone its contents to the permanent repository. - if isinstance(self._repository.backend, SandboxRepositoryBackend): - repository_backend = self.backend.get_repository() - repository = Repository(backend=repository_backend) - repository.clone(self._repository) - # Swap the sandbox repository for the new permanent repository instance which should delete the sandbox - self._repository_instance = repository - - self.repository_metadata = self._repository.serialize() + self.base.repository._store() # pylint: disable=protected-access links = self._incoming_cache self._backend_entity.store(links, with_transaction=with_transaction, clean=clean) @@ -786,7 +762,6 @@ def _store_from_cache(self, cache_node: 'Node', with_transaction: bool) -> None: """ from aiida.orm.utils.mixins import Sealable - from aiida.repository import Repository assert self.node_type == cache_node.node_type # Make sure the node doesn't have any RETURN links @@ -797,7 +772,7 @@ def _store_from_cache(self, cache_node: 'Node', with_transaction: bool) -> None: self.description = cache_node.description # Make sure to reinitialize the repository instance of the clone to that of the source node. - self._repository: Repository = copy.copy(cache_node._repository) # pylint: disable=protected-access + self.base.repository._copy(cache_node.base.repository) # pylint: disable=protected-access for key, value in cache_node.attributes.items(): if key != Sealable.SEALED_KEY: @@ -843,7 +818,6 @@ def _get_hash(self, ignore_errors: bool = True, **kwargs: Any) -> Optional[str]: def _get_objects_to_hash(self) -> List[Any]: """Return a list of objects which should be included in the hash.""" - assert self._repository is not None, 'repository not initialised' top_level_module = self.__module__.split('.', 1)[0] try: version = importlib.import_module(top_level_module).__version__ @@ -856,7 +830,7 @@ def _get_objects_to_hash(self) -> List[Any]: for key, val in self.attributes_items() if key not in self._hash_ignored_attributes and key not in self._updatable_attributes # pylint: disable=unsupported-membership-test }, - self._repository.hash(), + self.base.repository.hash(), self.computer.uuid if self.computer is not None else None ] return objects @@ -960,3 +934,36 @@ def get_description(self) -> str: """ # pylint: disable=no-self-use return '' + + _deprecated_repo_methods = { + 'copy_tree': 'copy_tree', + 'delete_object': 'delete_object', + 'get_object': 'get_object', + 'get_object_content': 'get_object_content', + 'glob': 'glob', + 'list_objects': 'list_objects', + 'list_object_names': 'list_object_names', + 'open': 'open', + 'put_object_from_filelike': 'put_object_from_filelike', + 'put_object_from_file': 'put_object_from_file', + 'put_object_from_tree': 'put_object_from_tree', + 'walk': 'walk', + 'repository_metadata': 'metadata', + } + + def __getattr__(self, name: str) -> Any: + """ + This method is called when an attribute is not found in the instance. + + It allows for the handling of deprecated mixin methods. + """ + if name in self._deprecated_repo_methods: + new_name = self._deprecated_repo_methods[name] + kls = self.__class__.__name__ + warn_deprecation( + f'`{kls}.{name}` is deprecated, use `{kls}.base.repository.{new_name}` instead.', + version=3, + stacklevel=3 + ) + return getattr(self.base.repository, new_name) + raise AttributeError(name) diff --git a/aiida/orm/nodes/process/calculation/calcjob.py b/aiida/orm/nodes/process/calculation/calcjob.py index bf4376e83d..ea06c8871f 100644 --- a/aiida/orm/nodes/process/calculation/calcjob.py +++ b/aiida/orm/nodes/process/calculation/calcjob.py @@ -494,7 +494,7 @@ def get_scheduler_stdout(self) -> Optional[AnyStr]: return None try: - stdout = retrieved_node.get_object_content(filename) + stdout = retrieved_node.base.repository.get_object_content(filename) except IOError: stdout = None @@ -512,7 +512,7 @@ def get_scheduler_stderr(self) -> Optional[AnyStr]: return None try: - stderr = retrieved_node.get_object_content(filename) + stderr = retrieved_node.base.repository.get_object_content(filename) except IOError: stderr = None diff --git a/aiida/orm/nodes/repository.py b/aiida/orm/nodes/repository.py index 67dc7bb3db..faaba4626d 100644 --- a/aiida/orm/nodes/repository.py +++ b/aiida/orm/nodes/repository.py @@ -1,21 +1,25 @@ # -*- coding: utf-8 -*- """Interface to the file repository of a node instance.""" import contextlib +import copy import io import pathlib import tempfile -from typing import BinaryIO, Dict, Iterable, Iterator, List, Tuple, Union +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, Iterator, List, Optional, TextIO, Tuple, Union from aiida.common import exceptions from aiida.repository import File, Repository from aiida.repository.backend import SandboxRepositoryBackend -__all__ = ('NodeRepositoryMixin',) +if TYPE_CHECKING: + from .node import Node + +__all__ = ('NodeRepository',) FilePath = Union[str, pathlib.PurePosixPath] -class NodeRepositoryMixin: +class NodeRepository: """Interface to the file repository of a node instance. This is the compatibility layer between the `Node` class and the `Repository` class. The repository in principle has @@ -26,19 +30,42 @@ class NodeRepositoryMixin: hierarchy if the instance was constructed normally, or from a specific hierarchy if reconstructred through the ``Repository.from_serialized`` classmethod. This is only the case for stored nodes, because unstored nodes do not have any files yet when they are constructed. Once the node get's stored, the repository is asked to serialize its - metadata contents which is then stored in the ``repository_metadata`` attribute of the node in the database. This - layer explicitly does not update the metadata of the node on a mutation action. The reason is that for stored nodes - these actions are anyway forbidden and for unstored nodes, the final metadata will be stored in one go, once the - node is stored, so there is no need to keep updating the node metadata intermediately. Note that this does mean that - ``repository_metadata`` does not give accurate information as long as the node is not yet stored. + metadata contents which is then stored in the ``repository_metadata`` field of the backend node. + This layer explicitly does not update the metadata of the node on a mutation action. + The reason is that for stored nodes these actions are anyway forbidden and for unstored nodes, + the final metadata will be stored in one go, once the node is stored, + so there is no need to keep updating the node metadata intermediately. + Note that this does mean that ``repository_metadata`` does not give accurate information, + as long as the node is not yet stored. """ - _repository_instance = None + def __init__(self, node: 'Node') -> None: + """Construct a new instance of the repository interface.""" + self._node: 'Node' = node + self._repository_instance: Optional[Repository] = None + + @property + def metadata(self) -> Dict[str, Any]: + """Return the repository metadata, representing the virtual file hierarchy. + + Note, this is only accurate if the node is stored. + + :return: the repository metadata + """ + return self._node.backend_entity.repository_metadata def _update_repository_metadata(self): - """Refresh the repository metadata of the node if it is stored and the decorated method returns successfully.""" - if self.is_stored: - self.repository_metadata = self._repository.serialize() + """Refresh the repository metadata of the node if it is stored.""" + if self._node.is_stored: + self._node.backend_entity.repository_metadata = self.serialize() + + def _check_mutability(self): + """Check if the node is mutable. + + :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. + """ + if self._node.is_stored: + raise exceptions.ModificationNotAllowed('the node is stored and therefore the repository is immutable.') @property def _repository(self) -> Repository: @@ -49,10 +76,9 @@ def _repository(self) -> Repository: :return: the file repository instance. """ if self._repository_instance is None: - if self.is_stored: - backend = self.backend.get_repository() - serialized = self.repository_metadata - self._repository_instance = Repository.from_serialized(backend=backend, serialized=serialized) + if self._node.is_stored: + backend = self._node.backend.get_repository() + self._repository_instance = Repository.from_serialized(backend=backend, serialized=self.metadata) else: self._repository_instance = Repository(backend=SandboxRepositoryBackend()) @@ -69,20 +95,49 @@ def _repository(self, repository: Repository) -> None: self._repository_instance = repository - def repository_serialize(self) -> Dict: + def _store(self) -> None: + """Store the repository in the backend.""" + if isinstance(self._repository.backend, SandboxRepositoryBackend): + # Only if the backend repository is a sandbox do we have to clone its contents to the permanent repository. + repository_backend = self._node.backend.get_repository() + repository = Repository(backend=repository_backend) + repository.clone(self._repository) + # Swap the sandbox repository for the new permanent repository instance which should delete the sandbox + self._repository_instance = repository + # update the metadata on the node backend + self._node.backend_entity.repository_metadata = self.serialize() + + def _copy(self, repo: 'NodeRepository') -> None: + """Copy a repository from another instance. + + This is used when storing cached nodes. + + :param repo: the repository to clone. + """ + self._repository = copy.copy(repo._repository) # pylint: disable=protected-access + + def _clone(self, repo: 'NodeRepository') -> None: + """Clone the repository from another instance. + + This is used when cloning a node. + + :param repo: the repository to clone. + """ + self._repository.clone(repo._repository) # pylint: disable=protected-access + + def serialize(self) -> Dict: """Serialize the metadata of the repository content into a JSON-serializable format. :return: dictionary with the content metadata. """ return self._repository.serialize() - def check_mutability(self): - """Check if the node is mutable. + def hash(self) -> str: + """Generate a hash of the repository's contents. - :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. + :return: the hash representing the contents of the repository. """ - if self.is_stored: - raise exceptions.ModificationNotAllowed('the node is stored and therefore the repository is immutable.') + return self._repository.hash() def list_objects(self, path: str = None) -> List[File]: """Return a list of the objects contained in this repository sorted by name, optionally in given sub directory. @@ -107,7 +162,7 @@ def list_object_names(self, path: str = None) -> List[str]: return self._repository.list_object_names(path) @contextlib.contextmanager - def open(self, path: str, mode='r') -> Iterator[BinaryIO]: + def open(self, path: str, mode='r') -> Iterator[Union[BinaryIO, TextIO]]: """Open a file handle to an object stored under the given key. .. note:: this should only be used to open a handle to read an existing file. To write a new file use the method @@ -156,6 +211,18 @@ def get_object_content(self, path: str, mode='r') -> Union[str, bytes]: return self._repository.get_object_content(path) + def put_object_from_bytes(self, content: bytes, path: str) -> None: + """Store the given content in the repository at the given path. + + :param path: the relative path where to store the object in the repository. + :param content: the content to store. + :raises TypeError: if the path is not a string and relative path. + :raises FileExistsError: if an object already exists at the given path. + """ + self._check_mutability() + self._repository.put_object_from_filelike(io.BytesIO(content), path) + self._update_repository_metadata() + def put_object_from_filelike(self, handle: io.BufferedReader, path: str): """Store the byte contents of a file in the repository. @@ -164,7 +231,7 @@ def put_object_from_filelike(self, handle: io.BufferedReader, path: str): :raises TypeError: if the path is not a string and relative path. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() if isinstance(handle, io.StringIO): handle = io.BytesIO(handle.read().encode('utf-8')) @@ -186,7 +253,7 @@ def put_object_from_file(self, filepath: str, path: str): :raises TypeError: if the path is not a string and relative path, or the handle is not a byte stream. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.put_object_from_file(filepath, path) self._update_repository_metadata() @@ -198,7 +265,7 @@ def put_object_from_tree(self, filepath: str, path: str = None): :raises TypeError: if the path is not a string and relative path. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.put_object_from_tree(filepath, path) self._update_repository_metadata() @@ -241,7 +308,7 @@ def delete_object(self, path: str): :raises OSError: if the file could not be deleted. :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.delete_object(path) self._update_repository_metadata() @@ -250,6 +317,6 @@ def erase(self): :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is stored and therefore immutable. """ - self.check_mutability() + self._check_mutability() self._repository.erase() self._update_repository_metadata() diff --git a/aiida/orm/utils/mixins.py b/aiida/orm/utils/mixins.py index 9857de86d7..95ddefad65 100644 --- a/aiida/orm/utils/mixins.py +++ b/aiida/orm/utils/mixins.py @@ -9,8 +9,6 @@ ########################################################################### """Mixin classes for ORM classes.""" import inspect -import io -import tempfile from aiida.common import exceptions from aiida.common.lang import classproperty, override @@ -56,7 +54,7 @@ def store_source_info(self, func): try: source_file_path = inspect.getsourcefile(func) with open(source_file_path, 'rb') as handle: - self.put_object_from_filelike(handle, self.FUNCTION_SOURCE_FILE_PATH) + self.base.repository.put_object_from_filelike(handle, self.FUNCTION_SOURCE_FILE_PATH) except (IOError, OSError): pass @@ -110,7 +108,7 @@ def get_function_source_code(self): :returns: the absolute path of the source file in the repository, or None if it does not exist """ - return self.get_object_content(self.FUNCTION_SOURCE_FILE_PATH) + return self.base.repository.get_object_content(self.FUNCTION_SOURCE_FILE_PATH) class Sealable: @@ -123,14 +121,6 @@ class Sealable: def _updatable_attributes(cls): # pylint: disable=no-self-argument return (cls.SEALED_KEY,) - def check_mutability(self): - """Check if the node is mutable. - - :raises `~aiida.common.exceptions.ModificationNotAllowed`: when the node is sealed and therefore immutable. - """ - if self.is_stored: - raise exceptions.ModificationNotAllowed('the node is sealed and therefore the repository is immutable.') - def validate_incoming(self, source, link_type, link_label): """Validate adding a link of the given type from a given node to ourself. @@ -204,67 +194,3 @@ def delete_attribute(self, key): raise exceptions.ModificationNotAllowed(f'`{key}` is not an updatable attribute') self.backend_entity.delete_attribute(key) - - @override - def put_object_from_filelike(self, handle: io.BufferedReader, path: str): - """Store the byte contents of a file in the repository. - - :param handle: filelike object with the byte content to be stored. - :param path: the relative path where to store the object in the repository. - :raises TypeError: if the path is not a string and relative path. - :raises aiida.common.exceptions.ModificationNotAllowed: when the node is sealed and therefore immutable. - """ - self.check_mutability() - - if isinstance(handle, io.StringIO): - handle = io.BytesIO(handle.read().encode('utf-8')) - - if isinstance(handle, tempfile._TemporaryFileWrapper): # pylint: disable=protected-access - if 'b' in handle.file.mode: - handle = io.BytesIO(handle.read()) - else: - handle = io.BytesIO(handle.read().encode('utf-8')) - - self._repository.put_object_from_filelike(handle, path) - self._update_repository_metadata() - - @override - def put_object_from_file(self, filepath: str, path: str): - """Store a new object under `path` with contents of the file located at `filepath` on the local file system. - - :param filepath: absolute path of file whose contents to copy to the repository - :param path: the relative path where to store the object in the repository. - :raises TypeError: if the path is not a string and relative path, or the handle is not a byte stream. - :raises aiida.common.exceptions.ModificationNotAllowed: when the node is sealed and therefore immutable. - """ - self.check_mutability() - self._repository.put_object_from_file(filepath, path) - self._update_repository_metadata() - - @override - def put_object_from_tree(self, filepath: str, path: str = None): - """Store the entire contents of `filepath` on the local file system in the repository with under given `path`. - - :param filepath: absolute path of the directory whose contents to copy to the repository. - :param path: the relative path where to store the objects in the repository. - :raises TypeError: if the path is not a string and relative path. - :raises aiida.common.exceptions.ModificationNotAllowed: when the node is sealed and therefore immutable. - """ - self.check_mutability() - self._repository.put_object_from_tree(filepath, path) - self._update_repository_metadata() - - @override - def delete_object(self, path: str): - """Delete the object from the repository. - - :param key: fully qualified identifier for the object within the repository. - :raises TypeError: if the path is not a string and relative path. - :raises FileNotFoundError: if the file does not exist. - :raises IsADirectoryError: if the object is a directory and not a file. - :raises OSError: if the file could not be deleted. - :raises aiida.common.exceptions.ModificationNotAllowed: when the node is sealed and therefore immutable. - """ - self.check_mutability() - self._repository.delete_object(path) - self._update_repository_metadata() diff --git a/aiida/parsers/plugins/arithmetic/add.py b/aiida/parsers/plugins/arithmetic/add.py index f043f3fc63..ffa1e82d96 100644 --- a/aiida/parsers/plugins/arithmetic/add.py +++ b/aiida/parsers/plugins/arithmetic/add.py @@ -21,7 +21,7 @@ def parse(self, **kwargs): from aiida.orm import Int try: - with self.retrieved.open(self.node.get_option('output_filename'), 'r') as handle: + with self.retrieved.base.repository.open(self.node.get_option('output_filename'), 'r') as handle: result = int(handle.read()) except OSError: return self.exit_codes.ERROR_READING_OUTPUT_FILE @@ -43,7 +43,7 @@ def parse(self, **kwargs): output_folder = self.retrieved - with output_folder.open(self.node.get_option('output_filename'), 'r') as handle: + with output_folder.base.repository.open(self.node.get_option('output_filename'), 'r') as handle: result = int(handle.read()) self.out('sum', Int(result)) diff --git a/aiida/repository/backend/abstract.py b/aiida/repository/backend/abstract.py index 17c7946d2a..85023b1a72 100644 --- a/aiida/repository/backend/abstract.py +++ b/aiida/repository/backend/abstract.py @@ -39,7 +39,7 @@ def key_format(self) -> Optional[str]: """Return the format for the keys of the repository. Important for when migrating between backends (e.g. archive -> main), as if they are not equal then it is - necessary to re-compute all the `Node.repository_metadata` before importing (otherwise they will not match + necessary to re-compute all the `Node.base.repository.metadata` before importing (otherwise they will not match with the repository). """ diff --git a/aiida/restapi/translator/nodes/node.py b/aiida/restapi/translator/nodes/node.py index 3f3c920ea8..b550f10e0e 100644 --- a/aiida/restapi/translator/nodes/node.py +++ b/aiida/restapi/translator/nodes/node.py @@ -495,7 +495,7 @@ def get_repo_list(node, filename=''): :return: folder list """ try: - flist = node.list_objects(filename) + flist = node.base.repository.list_objects(filename) except NotADirectoryError: raise RestInputValidationError(f'{filename} is not a directory in this repository') response = [] @@ -515,7 +515,7 @@ def get_repo_contents(node, filename=''): if filename: try: - data = node.get_object_content(filename, mode='rb') + data = node.base.repository.get_object_content(filename, mode='rb') return data except FileNotFoundError: raise RestInputValidationError('No such file is present') diff --git a/aiida/tools/archive/create.py b/aiida/tools/archive/create.py index acb5a200fe..0e28e1e8a4 100644 --- a/aiida/tools/archive/create.py +++ b/aiida/tools/archive/create.py @@ -571,7 +571,7 @@ def _stream_repo_files( repository = backend.get_repository() if not repository.key_format == key_format: - # Here we would have to go back and replace all the keys in the `Node.repository_metadata`s + # Here we would have to go back and replace all the keys in the `BackendNode.repository_metadata`s raise NotImplementedError( f'Backend repository key format incompatible: {repository.key_format!r} != {key_format!r}' ) diff --git a/docs/source/howto/plugin_codes.rst b/docs/source/howto/plugin_codes.rst index 49e39c9150..f387484af7 100644 --- a/docs/source/howto/plugin_codes.rst +++ b/docs/source/howto/plugin_codes.rst @@ -210,7 +210,7 @@ To create a parser plugin, subclass the |Parser| class in a file called ``parser Before the ``parse()`` method is called, two important attributes are set on the |Parser| instance: - 1. ``self.retrieved``: An instance of |FolderData|, which points to the folder containing all output files that the |CalcJob| instructed to retrieve, and provides the means to :py:meth:`~aiida.orm.nodes.repository.NodeRepositoryMixin.open` any file it contains. + 1. ``self.retrieved``: An instance of |FolderData|, which points to the folder containing all output files that the |CalcJob| instructed to retrieve, and provides the means to :py:meth:`~aiida.orm.nodes.repository.NodeRepository.open` any file it contains. 2. ``self.node``: The :py:class:`~aiida.orm.nodes.process.calculation.calcjob.CalcJobNode` representing the finished calculation, which, among other things, provides access to all of its inputs (``self.node.inputs``). diff --git a/docs/source/internals/storage/repository.rst b/docs/source/internals/storage/repository.rst index b3f354ba35..a4295b237e 100644 --- a/docs/source/internals/storage/repository.rst +++ b/docs/source/internals/storage/repository.rst @@ -59,7 +59,7 @@ To understand how the file repository frontend integrates the ORM and the file r The file repository backend is interfaced through the :class:`~aiida.repository.repository.Repository` class. It maintains a reference of an instance of one of the available file repository backend implementations, be it the sandbox or disk object store variant, to store file objects and to retrieve the content for stored objects. Internally, it keeps a virtual file hierarchy, which allows clients to address files by their path in the file hierarchy as opposed to have the unique key identifiers created by the backend. - Finally, the :class:`~aiida.orm.nodes.node.Node` class, which is the main point of interaction of users with the entire API, mixes in the :class:`~aiida.orm.nodes.repository.NodeRepositoryMixin` class. + Finally, the :class:`~aiida.orm.nodes.node.Node` class, which is the main point of interaction of users with the entire API, mixes in the :class:`~aiida.orm.nodes.repository.NodeRepository` class. The latter keeps an instance of the :class:`~aiida.repository.repository.Repository` class to which all repository operations are forwarded, after the check of node mutability is performed. As opposed to the backend interface, the frontend :class:`~aiida.repository.repository.Repository` *does* understand the concept of a file hierarchy and keeps it fully in memory. @@ -73,7 +73,7 @@ The node database model has a JSONB column called ``repository_metadata`` that c This serialized form is generated by the :meth:`~aiida.repository.repository.Repository.serialize` method, and the :meth:`~aiida.repository.repository.Repository.from_serialized` class method can be used to reconstruct a repository instance with a pre-existing file hierarchy. Note that upon constructing from a serialized file hierarchy, the :class:`~aiida.repository.repository.Repository` will not actually validate that the file objects contained within the hierarchy are *actually* contained in the backend. -The final integration of the :class:`~aiida.repository.repository.Repository` class with the ORM is through the :class:`~aiida.orm.nodes.repository.NodeRepositoryMixin` class, which is mixed into the :class:`~aiida.orm.nodes.node.Node` class. +The final integration of the :class:`~aiida.repository.repository.Repository` class with the ORM is through the :class:`~aiida.orm.nodes.repository.NodeRepository` class, which is mixed into the :class:`~aiida.orm.nodes.node.Node` class. This layer serves a couple of functions: * It implements the mutability rules of nodes @@ -82,10 +82,10 @@ This layer serves a couple of functions: The first is necessary because after a node has been stored, its content is considered immutable, which includes the content of its file repository. The :class:`~aiida.orm.utils.mixins.Sealable` mixin overrides the :class:`~aiida.repository.repository.Repository` methods that mutate repository content, to ensure that process nodes *can* mutate their content, as long as they are not sealed. -The second *raison-d'être* of the :class:`~aiida.orm.nodes.repository.NodeRepositoryMixin` is to allow clients to work with string streams instead of byte streams. +The second *raison-d'être* of the :class:`~aiida.orm.nodes.repository.NodeRepository` is to allow clients to work with string streams instead of byte streams. As explained in the section on the :ref:`file repository backend `, it only works with byte streams. However, users of the frontend API are often more used to working with strings and files with a given encoding. -The :class:`~aiida.orm.nodes.repository.NodeRepositoryMixin` overrides the *put*-methods and accepts string streams, and enables returning file handles to existing file objects that automatically decode the byte content. +The :class:`~aiida.orm.nodes.repository.NodeRepository` overrides the *put*-methods and accepts string streams, and enables returning file handles to existing file objects that automatically decode the byte content. The only additional requirement for operating with strings instead of bytes is that the client specifies the encoding. Since the file repository backend does not store any sort of metadata, it is impossible to deduce the file encoding and automatically decode it. Likewise, using the default file encoding of the system may yield the wrong result since the file could have been imported and actually have been originally written on another system with a different encoding. diff --git a/docs/source/topics/data_types.rst b/docs/source/topics/data_types.rst index 31f7a6d3df..3ff39c4639 100644 --- a/docs/source/topics/data_types.rst +++ b/docs/source/topics/data_types.rst @@ -333,14 +333,14 @@ or from `file-like objects ` shows how to read their content which can then be written elsewhere. However, sometimes you want to copy the entire contents of the node's repository, or a subdirectory of it. -The :meth:`~aiida.orm.nodes.repository.NodeRepositoryMixin.copy_tree` method makes this easy and can be used as follows: +The :meth:`~aiida.orm.nodes.repository.NodeRepository.copy_tree` method makes this easy and can be used as follows: .. code:: python @@ -185,7 +185,7 @@ If you only want to copy a particular subdirectory of the repository, you can pa node.copy_tree('/some/target/directory', path='sub/directory') -This method, combined with :meth:`~aiida.orm.nodes.repository.NodeRepositoryMixin.put_object_from_tree`, makes it easy to copy the entire repository content (or a subdirectory) from one node to another: +This method, combined with :meth:`~aiida.orm.nodes.repository.NodeRepository.put_object_from_tree`, makes it easy to copy the entire repository content (or a subdirectory) from one node to another: .. code:: python @@ -198,7 +198,7 @@ This method, combined with :meth:`~aiida.orm.nodes.repository.NodeRepositoryMixi node_target.put_object_from_tree(dirpath) Note that this method is not the most efficient as the files are first written from ``node_a`` to a temporary directory on disk, before they are read in memory again and written to the repository of ``node_b``. -There is a more efficient method which requires a bit more code and that directly uses the :meth:`~aiida.orm.nodes.repository.NodeRepositoryMixin.walk` method explained in the section on :ref:`listing repository content `. +There is a more efficient method which requires a bit more code and that directly uses the :meth:`~aiida.orm.nodes.repository.NodeRepository.walk` method explained in the section on :ref:`listing repository content `. .. code:: python diff --git a/tests/benchmark/test_archive.py b/tests/benchmark/test_archive.py index 63af4a335f..4431237f0a 100644 --- a/tests/benchmark/test_archive.py +++ b/tests/benchmark/test_archive.py @@ -39,7 +39,7 @@ def recursive_provenance(in_node, depth, breadth, num_objects=0): out_node = Dict(dict={str(i): i for i in range(10)}) for idx in range(num_objects): - out_node.put_object_from_filelike(StringIO('a' * 10000), f'key{str(idx)}') + out_node.base.repository.put_object_from_filelike(StringIO('a' * 10000), f'key{str(idx)}') out_node.add_incoming(calcfunc, link_type=LinkType.CREATE, link_label='output') out_node.store() diff --git a/tests/calculations/test_transfer.py b/tests/calculations/test_transfer.py index 22735382ee..77f5288caa 100644 --- a/tests/calculations/test_transfer.py +++ b/tests/calculations/test_transfer.py @@ -239,9 +239,9 @@ def test_integration_transfer(aiida_localhost, tmp_path): output_retrieved = output_nodes['retrieved'] # Check the retrieved folder - assert sorted(output_retrieved.list_object_names()) == sorted(['file_local.txt', 'file_remote.txt']) - assert output_retrieved.get_object_content('file_local.txt') == content_local - assert output_retrieved.get_object_content('file_remote.txt') == content_remote + assert sorted(output_retrieved.base.repository.list_object_names()) == sorted(['file_local.txt', 'file_remote.txt']) + assert output_retrieved.base.repository.get_object_content('file_local.txt') == content_local + assert output_retrieved.base.repository.get_object_content('file_remote.txt') == content_remote # Check the remote folder assert 'file_local.txt' in output_remotedir.listdir() diff --git a/tests/cmdline/commands/test_calcjob.py b/tests/cmdline/commands/test_calcjob.py index 7107d447ed..e189f86abd 100644 --- a/tests/cmdline/commands/test_calcjob.py +++ b/tests/cmdline/commands/test_calcjob.py @@ -100,7 +100,7 @@ def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable # Get the imported ArithmeticAddCalculation node ArithmeticAddCalculation = CalculationFactory('core.arithmetic.add') calculations = orm.QueryBuilder().append(ArithmeticAddCalculation).all()[0] - self.arithmetic_job = calculations[0] + self.arithmetic_job: orm.CalcJobNode = calculations[0] self.cli_runner = CliRunner() @@ -194,16 +194,16 @@ def test_calcjob_inputcat(self): assert get_result_lines(result)[0] == '2 3' # Test cat binary files - self.arithmetic_job._repository.put_object_from_filelike(io.BytesIO(b'COMPRESS'), 'aiida.in') - self.arithmetic_job._update_repository_metadata() + self.arithmetic_job.base.repository._repository.put_object_from_filelike(io.BytesIO(b'COMPRESS'), 'aiida.in') + self.arithmetic_job.base.repository._update_repository_metadata() options = [self.arithmetic_job.uuid, 'aiida.in'] result = self.cli_runner.invoke(command.calcjob_inputcat, options) assert result.stdout_bytes == b'COMPRESS' # Restore the file - self.arithmetic_job._repository.put_object_from_filelike(io.BytesIO(b'2 3\n'), 'aiida.in') - self.arithmetic_job._update_repository_metadata() + self.arithmetic_job.base.repository._repository.put_object_from_filelike(io.BytesIO(b'2 3\n'), 'aiida.in') + self.arithmetic_job.base.repository._update_repository_metadata() def test_calcjob_outputcat(self): """Test verdi calcjob outputcat""" @@ -226,16 +226,16 @@ def test_calcjob_outputcat(self): # Test cat binary files retrieved = self.arithmetic_job.outputs.retrieved - retrieved._repository.put_object_from_filelike(io.BytesIO(b'COMPRESS'), 'aiida.out') - retrieved._update_repository_metadata() + retrieved.base.repository._repository.put_object_from_filelike(io.BytesIO(b'COMPRESS'), 'aiida.out') + retrieved.base.repository._update_repository_metadata() options = [self.arithmetic_job.uuid, 'aiida.out'] result = self.cli_runner.invoke(command.calcjob_outputcat, options) assert result.stdout_bytes == b'COMPRESS' # Restore the file - retrieved._repository.put_object_from_filelike(io.BytesIO(b'5\n'), 'aiida.out') - retrieved._update_repository_metadata() + retrieved.base.repository._repository.put_object_from_filelike(io.BytesIO(b'5\n'), 'aiida.out') + retrieved.base.repository._update_repository_metadata() def test_calcjob_cleanworkdir(self): """Test verdi calcjob cleanworkdir""" diff --git a/tests/cmdline/commands/test_node.py b/tests/cmdline/commands/test_node.py index b09bccf46e..57888a97c7 100644 --- a/tests/cmdline/commands/test_node.py +++ b/tests/cmdline/commands/test_node.py @@ -63,8 +63,8 @@ def get_unstored_folder_node(cls): cls.content_file2 = 'the minister of silly walks' cls.key_file1 = 'some/nested/folder/filename.txt' cls.key_file2 = 'some_other_file.txt' - folder_node.put_object_from_filelike(io.StringIO(cls.content_file1), cls.key_file1) - folder_node.put_object_from_filelike(io.StringIO(cls.content_file2), cls.key_file2) + folder_node.base.repository.put_object_from_filelike(io.StringIO(cls.content_file1), cls.key_file1) + folder_node.base.repository.put_object_from_filelike(io.StringIO(cls.content_file2), cls.key_file2) return folder_node def test_node_show(self): @@ -173,7 +173,7 @@ def test_node_repo_cat(self): # Test cat binary files folder_node = orm.FolderData() bytestream = gzip.compress(b'COMPRESS') - folder_node.put_object_from_filelike(io.BytesIO(bytestream), 'filename.txt.gz') + folder_node.base.repository.put_object_from_filelike(io.BytesIO(bytestream), 'filename.txt.gz') folder_node.store() options = [str(folder_node.pk), 'filename.txt.gz'] diff --git a/tests/cmdline/utils/test_repository.py b/tests/cmdline/utils/test_repository.py index d5937325ff..6a74fc1efb 100644 --- a/tests/cmdline/utils/test_repository.py +++ b/tests/cmdline/utils/test_repository.py @@ -28,8 +28,8 @@ def runner(): def folder_data(): """Create a `FolderData` instance with basic file and directory structure.""" node = FolderData() - node.put_object_from_filelike(io.StringIO(''), 'nested/file.txt') - node.put_object_from_filelike(io.StringIO(''), 'file.txt') + node.base.repository.put_object_from_filelike(io.StringIO(''), 'nested/file.txt') + node.base.repository.put_object_from_filelike(io.StringIO(''), 'file.txt') return node diff --git a/tests/engine/daemon/test_execmanager.py b/tests/engine/daemon/test_execmanager.py index c90dfebea9..c4d872d349 100644 --- a/tests/engine/daemon/test_execmanager.py +++ b/tests/engine/daemon/test_execmanager.py @@ -141,7 +141,7 @@ def test_upload_local_copy_list(fixture_sandbox, aiida_localhost, aiida_local_co create_file_hierarchy(file_hierarchy, tmp_path) folder = FolderData() - folder.put_object_from_tree(tmp_path) + folder.base.repository.put_object_from_tree(tmp_path) inputs = { 'file_x': SinglefileData(io.BytesIO(b'content_x')).store(), @@ -170,7 +170,7 @@ def test_upload_local_copy_list(fixture_sandbox, aiida_localhost, aiida_local_co # Check that none of the files were written to the repository of the calculation node, since they were communicated # through the ``local_copy_list``. - assert node.list_object_names() == [] + assert node.base.repository.list_object_names() == [] # Now check that all contents were successfully written to the sandbox written_hierarchy = serialize_file_hierarchy(pathlib.Path(fixture_sandbox.abspath)) diff --git a/tests/engine/processes/calcjobs/test_calc_job.py b/tests/engine/processes/calcjobs/test_calc_job.py index da818baecc..707ada511b 100644 --- a/tests/engine/processes/calcjobs/test_calc_job.py +++ b/tests/engine/processes/calcjobs/test_calc_job.py @@ -380,7 +380,7 @@ def test_run_local_code(self): uploaded_files = os.listdir(node.dry_run_info['folder']) # Since the repository will only contain files on the top-level due to `Code.set_files` we only check those - for filename in self.local_code.list_object_names(): + for filename in self.local_code.base.repository.list_object_names(): assert filename in uploaded_files @pytest.mark.usefixtures('chdir_tmp_path') @@ -443,9 +443,9 @@ def test_provenance_exclude_list(self): # Verify that the folder (representing the node's repository) indeed do not contain the input files. Note, # however, that the directory hierarchy should be there, albeit empty - assert 'base' in node.list_object_names() - assert sorted(['b']) == sorted(node.list_object_names(os.path.join('base'))) - assert ['two'] == node.list_object_names(os.path.join('base', 'b')) + assert 'base' in node.base.repository.list_object_names() + assert sorted(['b']) == sorted(node.base.repository.list_object_names(os.path.join('base'))) + assert ['two'] == node.base.repository.list_object_names(os.path.join('base', 'b')) def test_parse_no_retrieved_folder(self): """Test the `CalcJob.parse` method when there is no retrieved folder.""" @@ -575,8 +575,8 @@ def test_parse_not_implemented(generate_process): filename_stderr = process.node.get_option('scheduler_stderr') filename_stdout = process.node.get_option('scheduler_stdout') - retrieved.put_object_from_filelike(io.BytesIO(b'\n'), filename_stderr) - retrieved.put_object_from_filelike(io.BytesIO(b'\n'), filename_stdout) + retrieved.base.repository.put_object_from_filelike(io.BytesIO(b'\n'), filename_stderr) + retrieved.base.repository.put_object_from_filelike(io.BytesIO(b'\n'), filename_stdout) retrieved.store() process.parse() @@ -609,8 +609,8 @@ def test_parse_scheduler_excepted(generate_process, monkeypatch): filename_stderr = process.node.get_option('scheduler_stderr') filename_stdout = process.node.get_option('scheduler_stdout') - retrieved.put_object_from_filelike(io.BytesIO(b'\n'), filename_stderr) - retrieved.put_object_from_filelike(io.BytesIO(b'\n'), filename_stdout) + retrieved.base.repository.put_object_from_filelike(io.BytesIO(b'\n'), filename_stderr) + retrieved.base.repository.put_object_from_filelike(io.BytesIO(b'\n'), filename_stdout) retrieved.store() msg = 'crash' diff --git a/tests/orm/nodes/data/test_folder.py b/tests/orm/nodes/data/test_folder.py index fdfb61b8f7..7391586169 100644 --- a/tests/orm/nodes/data/test_folder.py +++ b/tests/orm/nodes/data/test_folder.py @@ -23,4 +23,4 @@ def test_constructor_tree(tmp_path): for filename, content in tree.items(): tmp_path.joinpath(filename).write_text(content, encoding='utf8') node = FolderData(tree=str(tmp_path)) - assert sorted(node.list_object_names()) == sorted(tree.keys()) + assert sorted(node.base.repository.list_object_names()) == sorted(tree.keys()) diff --git a/tests/orm/nodes/data/test_singlefile.py b/tests/orm/nodes/data/test_singlefile.py index c33be71e5b..383d369482 100644 --- a/tests/orm/nodes/data/test_singlefile.py +++ b/tests/orm/nodes/data/test_singlefile.py @@ -31,7 +31,7 @@ def inner(node, content_reference, filename, open_mode='r'): with node.open(mode=open_mode) as handle: assert handle.read() == content_reference - assert node.list_object_names() == [filename] + assert node.base.repository.list_object_names() == [filename] return inner diff --git a/tests/orm/nodes/test_calcjob.py b/tests/orm/nodes/test_calcjob.py index 89b0a32386..ae34376ce4 100644 --- a/tests/orm/nodes/test_calcjob.py +++ b/tests/orm/nodes/test_calcjob.py @@ -55,8 +55,10 @@ def test_get_scheduler_stdout(self): retrieved = FolderData() if with_file: - retrieved._repository.put_object_from_filelike(io.BytesIO(stdout.encode('utf-8')), option_value) # pylint: disable=protected-access - retrieved._update_repository_metadata() # pylint: disable=protected-access + retrieved.base.repository._repository.put_object_from_filelike( # pylint: disable=protected-access + io.BytesIO(stdout.encode('utf-8')), option_value + ) + retrieved.base.repository._update_repository_metadata() # pylint: disable=protected-access if with_option: node.set_option(option_key, option_value) node.store() @@ -81,8 +83,10 @@ def test_get_scheduler_stderr(self): retrieved = FolderData() if with_file: - retrieved._repository.put_object_from_filelike(io.BytesIO(stderr.encode('utf-8')), option_value) # pylint: disable=protected-access - retrieved._update_repository_metadata() # pylint: disable=protected-access + retrieved.base.repository._repository.put_object_from_filelike( # pylint: disable=protected-access + io.BytesIO(stderr.encode('utf-8')), option_value + ) + retrieved.base.repository._update_repository_metadata() # pylint: disable=protected-access if with_option: node.set_option(option_key, option_value) node.store() diff --git a/tests/orm/nodes/test_node.py b/tests/orm/nodes/test_node.py index 85574b8c19..5b74267520 100644 --- a/tests/orm/nodes/test_node.py +++ b/tests/orm/nodes/test_node.py @@ -53,9 +53,10 @@ def test_instantiate_with_computer(self): def test_repository_garbage_collection(self): """Verify that the repository sandbox folder is cleaned after the node instance is garbage collected.""" node = Data() - dirpath = node._repository.backend.sandbox.abspath # pylint: disable=protected-access + dirpath = node.base.repository._repository.backend.sandbox.abspath # pylint: disable=protected-access assert os.path.isdir(dirpath) + del node.base.repository del node assert not os.path.isdir(dirpath) @@ -73,19 +74,19 @@ def test_computer_user_immutability(self): def test_repository_metadata(): """Test the basic properties for `repository_metadata`.""" node = Data() - assert node.repository_metadata == {} + assert node.base.repository.metadata == {} # Even after storing the metadata should be empty, since it contains no files node.store() - assert node.repository_metadata == {} + assert node.base.repository.metadata == {} node = Data() + # setting any incorrect metadata on an unstored node should be reverted after storing repository_metadata = {'key': 'value'} - node.repository_metadata = repository_metadata - assert node.repository_metadata == repository_metadata - + node.backend_entity.repository_metadata = repository_metadata + assert node.base.repository.metadata == repository_metadata node.store() - assert node.repository_metadata != repository_metadata + assert node.base.repository.metadata == {} @staticmethod @pytest.mark.parametrize( @@ -966,7 +967,7 @@ def test_store_from_cache(self): os.makedirs(dir_path) with open(os.path.join(dir_path, 'file'), 'w', encoding='utf8') as file: file.write('content') - data.put_object_from_tree(tmpdir) + data.base.repository.put_object_from_tree(tmpdir) data.store() @@ -1010,13 +1011,13 @@ def test_uuid_equality_fallback(self): def test_iter_repo_keys(): """Test the ``iter_repo_keys`` method.""" data1 = Data() - data1.put_object_from_filelike(BytesIO(b'value1'), 'key1') - data1.put_object_from_filelike(BytesIO(b'value1'), 'key2') - data1.put_object_from_filelike(BytesIO(b'value3'), 'folder/key3') + data1.base.repository.put_object_from_filelike(BytesIO(b'value1'), 'key1') + data1.base.repository.put_object_from_filelike(BytesIO(b'value1'), 'key2') + data1.base.repository.put_object_from_filelike(BytesIO(b'value3'), 'folder/key3') data1.store() data2 = Data() - data2.put_object_from_filelike(BytesIO(b'value1'), 'key1') - data2.put_object_from_filelike(BytesIO(b'value4'), 'key2') + data2.base.repository.put_object_from_filelike(BytesIO(b'value1'), 'key1') + data2.base.repository.put_object_from_filelike(BytesIO(b'value4'), 'key2') data2.store() assert set(Data.objects.iter_repo_keys()) == { '31cd97ebe10a80abe1b3f401824fc2040fb8b03aafd0d37acf6504777eddee11', diff --git a/tests/orm/nodes/test_repository.py b/tests/orm/nodes/test_repository.py index c876d2f48d..a65f46b626 100644 --- a/tests/orm/nodes/test_repository.py +++ b/tests/orm/nodes/test_repository.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- # pylint: disable=redefined-outer-name,protected-access,no-member """Tests for the :mod:`aiida.orm.nodes.repository` module.""" -import io import pathlib import pytest from aiida.common import exceptions +from aiida.common.warnings import AiidaDeprecationWarning from aiida.engine import ProcessState from aiida.manage.caching import enable_caching from aiida.orm import CalcJobNode, Data, load_node @@ -19,7 +19,7 @@ def cacheable_node(): """Return a node that can be cached from.""" node = CalcJobNode(process_type='aiida.calculations:core.arithmetic.add') node.set_process_state(ProcessState.FINISHED) - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') node.store() assert node.is_valid_cache @@ -30,65 +30,65 @@ def cacheable_node(): def test_initialization(): """Test that the repository instance is lazily constructed.""" node = Data() - assert node.repository_metadata == {} - assert node._repository_instance is None + assert node.base.repository.metadata == {} + assert node.base.repository._repository_instance is None # Initialize just by calling the property - assert isinstance(node._repository.backend, SandboxRepositoryBackend) + assert isinstance(node.base.repository._repository.backend, SandboxRepositoryBackend) @pytest.mark.usefixtures('aiida_profile_clean') def test_unstored(): """Test the repository for unstored nodes.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') - assert isinstance(node._repository.backend, SandboxRepositoryBackend) - assert node.repository_metadata == {} + assert isinstance(node.base.repository._repository.backend, SandboxRepositoryBackend) + assert node.base.repository.metadata == {} @pytest.mark.usefixtures('aiida_profile_clean') def test_store(): """Test the repository after storing.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') - assert node.list_object_names() == ['relative'] - assert node.list_object_names('relative') == ['path'] + node.base.repository.put_object_from_bytes(b'content', 'relative/path') + assert node.base.repository.list_object_names() == ['relative'] + assert node.base.repository.list_object_names('relative') == ['path'] - hash_unstored = node._repository.hash() - metadata = node.repository_serialize() + hash_unstored = node.base.repository.hash() + metadata = node.base.repository.serialize() node.store() - assert isinstance(node._repository.backend, DiskObjectStoreRepositoryBackend) - assert node.repository_serialize() != metadata - assert node._repository.hash() == hash_unstored + assert isinstance(node.base.repository._repository.backend, DiskObjectStoreRepositoryBackend) + assert node.base.repository.serialize() != metadata + assert node.base.repository.hash() == hash_unstored @pytest.mark.usefixtures('aiida_profile_clean') def test_load(): """Test the repository after loading.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') node.store() - hash_stored = node._repository.hash() - metadata = node.repository_serialize() + hash_stored = node.base.repository.hash() + metadata = node.base.repository.serialize() loaded = load_node(node.uuid) - assert isinstance(node._repository.backend, DiskObjectStoreRepositoryBackend) - assert node.repository_serialize() == metadata - assert loaded._repository.hash() == hash_stored + assert isinstance(node.base.repository._repository.backend, DiskObjectStoreRepositoryBackend) + assert node.base.repository.serialize() == metadata + assert loaded.base.repository.hash() == hash_stored @pytest.mark.usefixtures('aiida_profile_clean') def test_load_updated(): """Test the repository after loading.""" node = CalcJobNode() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') node.store() loaded = load_node(node.uuid) - assert loaded.get_object_content('relative/path', mode='rb') == b'content' + assert loaded.base.repository.get_object_content('relative/path', mode='rb') == b'content' @pytest.mark.usefixtures('aiida_profile_clean') @@ -97,93 +97,93 @@ def test_caching(cacheable_node): with enable_caching(): cached = CalcJobNode(process_type='aiida.calculations:core.core.arithmetic.add') - cached.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + cached.base.repository.put_object_from_bytes(b'content', 'relative/path') cached.store() assert cached.is_created_from_cache assert cached.get_cache_source() == cacheable_node.uuid - assert cacheable_node.repository_metadata == cached.repository_metadata - assert cacheable_node._repository.hash() == cached._repository.hash() + assert cacheable_node.base.repository.metadata == cached.base.repository.metadata + assert cacheable_node.base.repository.hash() == cached.base.repository.hash() @pytest.mark.usefixtures('aiida_profile_clean') def test_clone(): """Test the repository after a node is cloned from a stored node.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') node.store() clone = node.clone() - assert clone.list_object_names('relative') == ['path'] - assert clone.get_object_content('relative/path', mode='rb') == b'content' + assert clone.base.repository.list_object_names('relative') == ['path'] + assert clone.base.repository.get_object_content('relative/path', mode='rb') == b'content' clone.store() - assert clone.list_object_names('relative') == ['path'] - assert clone.get_object_content('relative/path', mode='rb') == b'content' - assert clone.repository_metadata == node.repository_metadata - assert clone._repository.hash() == node._repository.hash() + assert clone.base.repository.list_object_names('relative') == ['path'] + assert clone.base.repository.get_object_content('relative/path', mode='rb') == b'content' + assert clone.base.repository.metadata == node.base.repository.metadata + assert clone.base.repository.hash() == node.base.repository.hash() @pytest.mark.usefixtures('aiida_profile_clean') def test_clone_unstored(): """Test the repository after a node is cloned from an unstored node.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') clone = node.clone() - assert clone.list_object_names('relative') == ['path'] - assert clone.get_object_content('relative/path', mode='rb') == b'content' + assert clone.base.repository.list_object_names('relative') == ['path'] + assert clone.base.repository.get_object_content('relative/path', mode='rb') == b'content' clone.store() - assert clone.list_object_names('relative') == ['path'] - assert clone.get_object_content('relative/path', mode='rb') == b'content' + assert clone.base.repository.list_object_names('relative') == ['path'] + assert clone.base.repository.get_object_content('relative/path', mode='rb') == b'content' @pytest.mark.usefixtures('aiida_profile_clean') def test_sealed(): """Test the repository interface for a calculation node before and after it is sealed.""" node = CalcJobNode() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') node.store() node.seal() with pytest.raises(exceptions.ModificationNotAllowed): - node.put_object_from_filelike(io.BytesIO(b'content'), 'path') + node.base.repository.put_object_from_bytes(b'content', 'path') @pytest.mark.usefixtures('aiida_profile_clean') def test_get_object_raises(): - """Test the ``NodeRepositoryMixin.get_object`` method when it is supposed to raise.""" + """Test the ``NodeRepository.get_object`` method when it is supposed to raise.""" node = Data() with pytest.raises(TypeError, match=r'path `.*` is not a relative path.'): - node.get_object('/absolute/path') + node.base.repository.get_object('/absolute/path') with pytest.raises(FileNotFoundError, match=r'object with path `.*` does not exist.'): - node.get_object('non_existing_folder/file_a') + node.base.repository.get_object('non_existing_folder/file_a') with pytest.raises(FileNotFoundError, match=r'object with path `.*` does not exist.'): - node.get_object('non_existant') + node.base.repository.get_object('non_existant') @pytest.mark.usefixtures('aiida_profile_clean') def test_get_object(): - """Test the ``NodeRepositoryMixin.get_object`` method.""" + """Test the ``NodeRepository.get_object`` method.""" node = CalcJobNode() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/file_b') + node.base.repository.put_object_from_bytes(b'content', 'relative/file_b') - file_object = node.get_object(None) + file_object = node.base.repository.get_object(None) assert isinstance(file_object, File) assert file_object.file_type == FileType.DIRECTORY assert file_object.is_file() is False assert file_object.is_dir() is True - file_object = node.get_object('relative') + file_object = node.base.repository.get_object('relative') assert isinstance(file_object, File) assert file_object.file_type == FileType.DIRECTORY assert file_object.name == 'relative' - file_object = node.get_object('relative/file_b') + file_object = node.base.repository.get_object('relative/file_b') assert isinstance(file_object, File) assert file_object.file_type == FileType.FILE assert file_object.name == 'file_b' @@ -193,12 +193,12 @@ def test_get_object(): @pytest.mark.usefixtures('aiida_profile_clean') def test_walk(): - """Test the ``NodeRepositoryMixin.walk`` method.""" + """Test the ``NodeRepository.walk`` method.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') results = [] - for root, dirnames, filenames in node.walk(): + for root, dirnames, filenames in node.base.repository.walk(): results.append((root, sorted(dirnames), sorted(filenames))) assert sorted(results) == [ @@ -210,7 +210,7 @@ def test_walk(): node.store() results = [] - for root, dirnames, filenames in node.walk(): + for root, dirnames, filenames in node.base.repository.walk(): results.append((root, sorted(dirnames), sorted(filenames))) assert sorted(results) == [ @@ -221,23 +221,33 @@ def test_walk(): @pytest.mark.usefixtures('aiida_profile_clean') def test_glob(): - """Test the ``NodeRepositoryMixin.glob`` method.""" + """Test the ``NodeRepository.glob`` method.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') - assert {path.as_posix() for path in node.glob()} == {'relative', 'relative/path'} + assert {path.as_posix() for path in node.base.repository.glob()} == {'relative', 'relative/path'} @pytest.mark.usefixtures('aiida_profile_clean') def test_copy_tree(tmp_path): """Test the ``Repository.copy_tree`` method.""" node = Data() - node.put_object_from_filelike(io.BytesIO(b'content'), 'relative/path') + node.base.repository.put_object_from_bytes(b'content', 'relative/path') - node.copy_tree(tmp_path) + node.base.repository.copy_tree(tmp_path) dirpath = pathlib.Path(tmp_path / 'relative') filepath = dirpath / 'path' assert dirpath.is_dir() assert filepath.is_file() - with node.open('relative/path', 'rb') as handle: + with node.base.repository.open('relative/path', 'rb') as handle: assert filepath.read_bytes() == handle.read() + + +@pytest.mark.usefixtures('aiida_profile_clean') +def test_deprecated_methods(monkeypatch): + """Test calling (deprecated) methods, directly from the `Node` instance still works.""" + node = Data() + monkeypatch.setenv('AIIDA_WARN_v3', 'true') + for method in node._deprecated_repo_methods: + with pytest.warns(AiidaDeprecationWarning): + getattr(node, method) diff --git a/tests/parsers/test_parser.py b/tests/parsers/test_parser.py index d42ebb2231..e7b4f0745d 100644 --- a/tests/parsers/test_parser.py +++ b/tests/parsers/test_parser.py @@ -111,7 +111,7 @@ def test_parse_from_node(self): node.store() retrieved = orm.FolderData() - retrieved.put_object_from_filelike(io.StringIO(f'{summed}'), output_filename) + retrieved.base.repository.put_object_from_filelike(io.StringIO(f'{summed}'), output_filename) retrieved.store() retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') diff --git a/tests/restapi/test_routes.py b/tests/restapi/test_routes.py index 220a6a0372..36fb05b5f0 100644 --- a/tests/restapi/test_routes.py +++ b/tests/restapi/test_routes.py @@ -78,7 +78,9 @@ def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable calc.add_incoming(structure, link_type=LinkType.INPUT_CALC, link_label='link_structure') calc.add_incoming(parameter1, link_type=LinkType.INPUT_CALC, link_label='link_parameter') - calc.put_object_from_filelike(io.BytesIO(b'The input file\nof the CalcJob node'), 'calcjob_inputs/aiida.in') + calc.base.repository.put_object_from_filelike( + io.BytesIO(b'The input file\nof the CalcJob node'), 'calcjob_inputs/aiida.in' + ) calc.store() # create log message for calcjob @@ -102,7 +104,7 @@ def init_profile(self, aiida_profile_clean, aiida_localhost): # pylint: disable retrieved_outputs = orm.FolderData() stream = io.BytesIO(b'The output file\nof the CalcJob node') - retrieved_outputs.put_object_from_filelike(stream, 'calcjob_outputs/aiida.out') + retrieved_outputs.base.repository.put_object_from_filelike(stream, 'calcjob_outputs/aiida.out') retrieved_outputs.store() retrieved_outputs.add_incoming(calc, link_type=LinkType.CREATE, link_label='retrieved') @@ -1098,7 +1100,7 @@ def test_repo(self): url = f"{self.get_url_prefix()}/nodes/{str(node_uuid)}/repo/contents?filename=\"calcjob_inputs/aiida.in\"" with self.app.test_client() as client: response_obj = client.get(url) - input_file = load_node(node_uuid).get_object_content('calcjob_inputs/aiida.in', mode='rb') + input_file = load_node(node_uuid).base.repository.get_object_content('calcjob_inputs/aiida.in', mode='rb') assert response_obj.data == input_file def test_process_report(self): diff --git a/tests/storage/psql_dos/test_backend.py b/tests/storage/psql_dos/test_backend.py index 3dda02aa94..f5c357a70a 100644 --- a/tests/storage/psql_dos/test_backend.py +++ b/tests/storage/psql_dos/test_backend.py @@ -48,7 +48,7 @@ def test_get_unreferenced_keyset(): # Error catching: put a file, get the keys from the aiida db, manually delete the keys # in the repository datanode = orm.FolderData() - datanode.put_object_from_filelike(BytesIO(b'File content'), 'file.txt') + datanode.base.repository.put_object_from_filelike(BytesIO(b'File content'), 'file.txt') datanode.store() aiida_backend = get_manager().get_profile_storage() diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index fbe159d2e1..000f7207f2 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -131,7 +131,7 @@ def test_reload_cifdata(self): the_uuid = a.uuid - assert a.list_object_names() == [basename] + assert a.base.repository.list_object_names() == [basename] with a.open() as fhandle: assert fhandle.read() == file_content @@ -146,13 +146,13 @@ def test_reload_cifdata(self): with a.open() as fhandle: assert fhandle.read() == file_content - assert a.list_object_names() == [basename] + assert a.base.repository.list_object_names() == [basename] b = load_node(the_uuid) # I check the retrieved object assert isinstance(b, CifData) - assert b.list_object_names() == [basename] + assert b.base.repository.list_object_names() == [basename] with b.open() as fhandle: assert fhandle.read() == file_content diff --git a/tests/test_generic.py b/tests/test_generic.py index c0e98792ee..024a35b39b 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -29,7 +29,7 @@ def test_code_local(aiida_profile_clean, aiida_localhost): with tempfile.NamedTemporaryFile(mode='w+') as fhandle: fhandle.write('#/bin/bash\n\necho test run\n') fhandle.flush() - code.put_object_from_filelike(fhandle, 'test.sh') + code.base.repository.put_object_from_filelike(fhandle, 'test.sh') code.store() assert code.can_run_on(aiida_localhost) @@ -64,14 +64,14 @@ def test_code_remote(aiida_profile_clean, aiida_localhost): with tempfile.NamedTemporaryFile(mode='w+') as fhandle: fhandle.write('#/bin/bash\n\necho test run\n') fhandle.flush() - code.put_object_from_filelike(fhandle, 'test.sh') + code.base.repository.put_object_from_filelike(fhandle, 'test.sh') with pytest.raises(ValidationError): # There are files inside code.store() # If there are no files, I can store - code.delete_object('test.sh') + code.base.repository.delete_object('test.sh') code.store() assert code.get_remote_computer().pk == aiida_localhost.pk # pylint: disable=no-member diff --git a/tests/test_nodes.py b/tests/test_nodes.py index a0110aa6ec..0d88e595fb 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -105,21 +105,21 @@ def test_node_uuid_hashing_for_querybuidler(self): def create_folderdata_with_empty_file(): node = orm.FolderData() with tempfile.NamedTemporaryFile() as handle: - node.put_object_from_filelike(handle, 'path/name') + node.base.repository.put_object_from_filelike(handle, 'path/name') return node @staticmethod def create_folderdata_with_empty_folder(): dirpath = tempfile.mkdtemp() node = orm.FolderData() - node.put_object_from_tree(dirpath, 'path/name') + node.base.repository.put_object_from_tree(dirpath, 'path/name') return node def test_folder_file_different(self): f1 = self.create_folderdata_with_empty_file().store() f2 = self.create_folderdata_with_empty_folder().store() - assert f1.list_object_names('path') == f2.list_object_names('path') + assert f1.base.repository.list_object_names('path') == f2.base.repository.list_object_names('path') assert f1.get_hash() != f2.get_hash() def test_updatable_attributes(self): @@ -492,44 +492,44 @@ def test_files(self): with tempfile.NamedTemporaryFile('w+') as handle: handle.write(file_content) handle.flush() - a.put_object_from_file(handle.name, 'file1.txt') - a.put_object_from_file(handle.name, 'file2.txt') + a.base.repository.put_object_from_file(handle.name, 'file1.txt') + a.base.repository.put_object_from_file(handle.name, 'file2.txt') - assert set(a.list_object_names()) == set(['file1.txt', 'file2.txt']) - with a.open('file1.txt') as fhandle: + assert set(a.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt']) + with a.base.repository.open('file1.txt') as fhandle: assert fhandle.read() == file_content - with a.open('file2.txt') as fhandle: + with a.base.repository.open('file2.txt') as fhandle: assert fhandle.read() == file_content b = a.clone() assert a.uuid != b.uuid # Check that the content is there - assert set(b.list_object_names()) == set(['file1.txt', 'file2.txt']) - with b.open('file1.txt') as handle: + assert set(b.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt']) + with b.base.repository.open('file1.txt') as handle: assert handle.read() == file_content - with b.open('file2.txt') as handle: + with b.base.repository.open('file2.txt') as handle: assert handle.read() == file_content # I overwrite a file and create a new one in the clone only with tempfile.NamedTemporaryFile(mode='w+') as handle: handle.write(file_content_different) handle.flush() - b.put_object_from_file(handle.name, 'file2.txt') - b.put_object_from_file(handle.name, 'file3.txt') + b.base.repository.put_object_from_file(handle.name, 'file2.txt') + b.base.repository.put_object_from_file(handle.name, 'file3.txt') # I check the new content, and that the old one has not changed - assert set(a.list_object_names()) == set(['file1.txt', 'file2.txt']) - with a.open('file1.txt') as handle: + assert set(a.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt']) + with a.base.repository.open('file1.txt') as handle: assert handle.read() == file_content - with a.open('file2.txt') as handle: + with a.base.repository.open('file2.txt') as handle: assert handle.read() == file_content - assert set(b.list_object_names()) == set(['file1.txt', 'file2.txt', 'file3.txt']) - with b.open('file1.txt') as handle: + assert set(b.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt', 'file3.txt']) + with b.base.repository.open('file1.txt') as handle: assert handle.read() == file_content - with b.open('file2.txt') as handle: + with b.base.repository.open('file2.txt') as handle: assert handle.read() == file_content_different - with b.open('file3.txt') as handle: + with b.base.repository.open('file3.txt') as handle: assert handle.read() == file_content_different # This should in principle change the location of the files, @@ -542,21 +542,21 @@ def test_files(self): with tempfile.NamedTemporaryFile(mode='w+') as handle: handle.write(file_content_different) handle.flush() - c.put_object_from_file(handle.name, 'file1.txt') - c.put_object_from_file(handle.name, 'file4.txt') + c.base.repository.put_object_from_file(handle.name, 'file1.txt') + c.base.repository.put_object_from_file(handle.name, 'file4.txt') - assert set(a.list_object_names()) == set(['file1.txt', 'file2.txt']) - with a.open('file1.txt') as handle: + assert set(a.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt']) + with a.base.repository.open('file1.txt') as handle: assert handle.read() == file_content - with a.open('file2.txt') as handle: + with a.base.repository.open('file2.txt') as handle: assert handle.read() == file_content - assert set(c.list_object_names()) == set(['file1.txt', 'file2.txt', 'file4.txt']) - with c.open('file1.txt') as handle: + assert set(c.base.repository.list_object_names()) == set(['file1.txt', 'file2.txt', 'file4.txt']) + with c.base.repository.open('file1.txt') as handle: assert handle.read() == file_content_different - with c.open('file2.txt') as handle: + with c.base.repository.open('file2.txt') as handle: assert handle.read() == file_content - with c.open('file4.txt') as handle: + with c.base.repository.open('file4.txt') as handle: assert handle.read() == file_content_different @pytest.mark.skip('relies on deleting folders from the repo which is not yet implemented') @@ -594,61 +594,61 @@ def test_folders(self): os.mkdir(os.path.join(tree_1, 'dir1', 'dir2', 'dir3')) # add the tree to the node - a.put_object_from_tree(tree_1, 'tree_1') + a.base.repository.put_object_from_tree(tree_1, 'tree_1') # verify if the node has the structure I expect - assert set(a.list_object_names()) == set(['tree_1']) - assert set(a.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) - assert set(a.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) - with a.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(a.base.repository.list_object_names()) == set(['tree_1']) + assert set(a.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) + assert set(a.base.repository.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) + with a.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content - with a.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with a.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # try to exit from the folder with pytest.raises(FileNotFoundError): - a.list_object_names('..') + a.base.repository.list_object_names('..') # clone into a new node b = a.clone() assert a.uuid != b.uuid # Check that the content is there - assert set(b.list_object_names()) == set(['tree_1']) - assert set(b.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) - assert set(b.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) - with b.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(b.base.repository.list_object_names()) == set(['tree_1']) + assert set(b.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) + assert set(b.base.repository.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) + with b.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content - with b.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with b.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # I overwrite a file and create a new one in the copy only dir3 = os.path.join(directory, 'dir3') os.mkdir(dir3) - b.put_object_from_tree(dir3, os.path.join('tree_1', 'dir3')) + b.base.repository.put_object_from_tree(dir3, os.path.join('tree_1', 'dir3')) # no absolute path here with pytest.raises(TypeError): - b.put_object_from_tree('dir3', os.path.join('tree_1', 'dir3')) + b.base.repository.put_object_from_tree('dir3', os.path.join('tree_1', 'dir3')) stream = io.StringIO(file_content_different) - b.put_object_from_filelike(stream, 'file3.txt') + b.base.repository.put_object_from_filelike(stream, 'file3.txt') # I check the new content, and that the old one has not changed old - assert set(a.list_object_names()) == set(['tree_1']) - assert set(a.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) - assert set(a.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) - with a.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(a.base.repository.list_object_names()) == set(['tree_1']) + assert set(a.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) + assert set(a.base.repository.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) + with a.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content - with a.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with a.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # new - assert set(b.list_object_names()) == set(['tree_1', 'file3.txt']) - assert set(b.list_object_names('tree_1')) == set(['file1.txt', 'dir1', 'dir3']) - assert set(b.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) - with b.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(b.base.repository.list_object_names()) == set(['tree_1', 'file3.txt']) + assert set(b.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1', 'dir3']) + assert set(b.base.repository.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) + with b.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content - with b.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with b.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # This should in principle change the location of the files, so I recheck @@ -659,26 +659,27 @@ def test_folders(self): # I overwrite a file, create a new one and remove a directory # in the copy only stream = io.StringIO(file_content_different) - c.put_object_from_filelike(stream, os.path.join('tree_1', 'file1.txt')) - c.put_object_from_filelike(stream, os.path.join('tree_1', 'dir1', 'file4.txt')) - c.delete_object(os.path.join('tree_1', 'dir1', 'dir2')) + c.base.repository.put_object_from_filelike(stream, os.path.join('tree_1', 'file1.txt')) + c.base.repository.put_object_from_filelike(stream, os.path.join('tree_1', 'dir1', 'file4.txt')) + c.base.repository.delete_object(os.path.join('tree_1', 'dir1', 'dir2')) # check old - assert set(a.list_object_names()) == set(['tree_1']) - assert set(a.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) - assert set(a.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) - with a.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(a.base.repository.list_object_names()) == set(['tree_1']) + assert set(a.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) + assert set(a.base.repository.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['dir2', 'file2.txt']) + with a.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content - with a.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with a.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # check new - assert set(c.list_object_names()) == set(['tree_1']) - assert set(c.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) - assert set(c.list_object_names(os.path.join('tree_1', 'dir1'))) == set(['file2.txt', 'file4.txt']) - with c.open(os.path.join('tree_1', 'file1.txt')) as fhandle: + assert set(c.base.repository.list_object_names()) == set(['tree_1']) + assert set(c.base.repository.list_object_names('tree_1')) == set(['file1.txt', 'dir1']) + assert set(c.base.repository.list_object_names(os.path.join('tree_1', + 'dir1'))) == set(['file2.txt', 'file4.txt']) + with c.base.repository.open(os.path.join('tree_1', 'file1.txt')) as fhandle: assert fhandle.read() == file_content_different - with c.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: + with c.base.repository.open(os.path.join('tree_1', 'dir1', 'file2.txt')) as fhandle: assert fhandle.read() == file_content # garbage cleaning diff --git a/tests/tools/archive/migration/test_legacy_migrations.py b/tests/tools/archive/migration/test_legacy_migrations.py index 9479117920..ea13e26e13 100644 --- a/tests/tools/archive/migration/test_legacy_migrations.py +++ b/tests/tools/archive/migration/test_legacy_migrations.py @@ -69,10 +69,12 @@ def test_full_migration(tmp_path, core_archive, archive_name): # test all node repositories were migrated uuids = reader.querybuilder().append(orm.Node, project='uuid').all(flat=True) assert set(uuids) == set(NODE_REPOS) - assert {uuid: {p.as_posix() for p in reader.get(orm.Node, uuid=uuid).glob()} for uuid in uuids} == NODE_REPOS + assert { + uuid: {p.as_posix() for p in reader.get(orm.Node, uuid=uuid).base.repository.glob()} for uuid in uuids + } == NODE_REPOS # test we can read a node repository file node = reader.get(orm.Node, uuid='3b429fd4-601c-4473-add5-7cbb76cf38cb') - content = node.get_object_content('_aiidasubmit.sh').encode('utf8') + content = node.base.repository.get_object_content('_aiidasubmit.sh').encode('utf8') assert content.startswith(b'#!/bin/bash') diff --git a/tests/tools/archive/test_repository.py b/tests/tools/archive/test_repository.py index a3501226ed..f67a292991 100644 --- a/tests/tools/archive/test_repository.py +++ b/tests/tools/archive/test_repository.py @@ -8,7 +8,6 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Test the export for nodes with files in the repository.""" -import io import os import pytest @@ -21,11 +20,11 @@ def test_export_repository(aiida_profile, tmp_path): """Test exporting a node with files in the repository.""" node = orm.Data() - node.put_object_from_filelike(io.BytesIO(b'file_a'), 'file_a') - node.put_object_from_filelike(io.BytesIO(b'file_b'), 'relative/file_b') + node.base.repository.put_object_from_bytes(b'file_a', 'file_a') + node.base.repository.put_object_from_bytes(b'file_b', 'relative/file_b') node.store() node_uuid = node.uuid - repository_metadata = node.repository_metadata + repository_metadata = node.base.repository.metadata filepath = os.path.join(tmp_path / 'export.aiida') create_archive([node], filename=filepath) @@ -34,6 +33,6 @@ def test_export_repository(aiida_profile, tmp_path): import_archive(filepath) loaded = orm.load_node(uuid=node_uuid) - assert loaded.repository_metadata == repository_metadata - assert loaded.get_object_content('file_a', mode='rb') == b'file_a' - assert loaded.get_object_content('relative/file_b', mode='rb') == b'file_b' + assert loaded.base.repository.metadata == repository_metadata + assert loaded.base.repository.get_object_content('file_a', mode='rb') == b'file_a' + assert loaded.base.repository.get_object_content('relative/file_b', mode='rb') == b'file_b'