diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index ae2a7bb6..e5f8239e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -26,6 +26,5 @@ jobs: - name: Run tests run: | - source .venv/bin/activate - python tests/test_mepo_commands.py -v + rye test -v timeout-minutes: 5 diff --git a/etc/mepo-completion.bash b/etc/mepo-completion.bash index 8abe8ab3..faa49870 100644 --- a/etc/mepo-completion.bash +++ b/etc/mepo-completion.bash @@ -2,18 +2,17 @@ # complete -W "init clone status checkout branch diff where whereis history" mepo +# SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SCRIPT_DIR=$(dirname $(realpath $0)) + _get_mepo_commands() { local mepo_cmd_list="" - if [[ "$OSTYPE" == "darwin"* ]] - then - local mepodir=$(dirname $(readlink $(which mepo))) - else - local mepodir=$(dirname $(readlink -f $(which mepo))) - fi - for mydir in $(ls -d ${mepodir}/mepo.d/command/*/); do - if [[ $mydir != *"__pycache__"* ]]; then - mepo_cmd_list+=" $(basename $mydir)" - fi + local mepo_dir=$(python3 $SCRIPT_DIR/mepo-path.py) + for pyfile in $(ls ${mepo_dir}/command/*.py*); do + command=${pyfile##*/} # remove path + command=${command%.*} # remove extension + command=$(echo $command | cut -d _ -f 1) + mepo_cmd_list+=" $command" done echo ${mepo_cmd_list} } diff --git a/etc/mepo-path.py b/etc/mepo-path.py new file mode 100644 index 00000000..04488203 --- /dev/null +++ b/etc/mepo-path.py @@ -0,0 +1,3 @@ +import os, mepo + +print(os.path.dirname(mepo.__file__)) diff --git a/pyproject.toml b/pyproject.toml index 308021b8..1ca07a02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mepo" -version = "2.0.0" +version = "2.1.0" description = "A tool for managing (m)ultiple r(epo)s" authors = [{name="GMAO SI Team", email="siteam@gmao.gsfc.nasa.gov"}] dependencies = [ @@ -23,6 +23,7 @@ dev-dependencies = [ "flake8>=7.0.0", "pre-commit>=3.7.1", "mdutils>=1.6.0", + "pytest>=8.2.1", ] [build-system] diff --git a/src/mepo/command/update-state.py b/src/mepo/command/update-state.py index 58fcc000..94bc7c7e 100644 --- a/src/mepo/command/update-state.py +++ b/src/mepo/command/update-state.py @@ -1,6 +1,9 @@ """Permanently convert mepo1 state to mepo2 state""" +from urllib.parse import urljoin + from ..state import MepoState +from ..git import get_current_remote_url def run(_): @@ -9,6 +12,9 @@ def run(_): # mepo2 style does not exist allcomps = MepoState.read_state() MepoState.mepo1_patch_undo() + for comp in allcomps: + if comp.remote.startswith("../"): + comp.remote = urljoin(get_current_remote_url() + "/", comp.remote) # Write new state MepoState.write_state(allcomps) print("\nConverted mepo1 state to mepo2\n") diff --git a/src/mepo/component.py b/src/mepo/component.py index ef732ba1..7d9753b8 100644 --- a/src/mepo/component.py +++ b/src/mepo/component.py @@ -1,15 +1,18 @@ import os import shlex -from urllib.parse import urlparse +from dataclasses import dataclass +from urllib.parse import urljoin +from .git import get_current_remote_url from .utilities import shellcmd from .utilities.version import MepoVersion -# This will be used to store the "final nodes" from each subrepo -original_final_node_list = [] +# This will be used to store the "last nodes" from each subrepo +last_node_list = [] +@dataclass(eq=True) class MepoComponent(object): __slots__ = [ @@ -24,16 +27,27 @@ class MepoComponent(object): "ignore_submodules", ] - def __init__(self): - self.name = None - self.local = None - self.remote = None - self.version = None - self.sparse = None - self.develop = None - self.recurse_submodules = None - self.fixture = None - self.ignore_submodules = None + def __init__( + self, + name=None, + local=None, + remote=None, + version=None, + sparse=None, + develop=None, + recurse_submodules=None, + fixture=None, + ignore_submodules=None, + ): + self.name = name + self.local = local + self.remote = remote + self.version = version + self.sparse = sparse + self.develop = develop + self.recurse_submodules = recurse_submodules + self.fixture = fixture + self.ignore_submodules = ignore_submodules def __repr__(self): # Older mepo clones will not have ignore_submodules in comp, so @@ -103,54 +117,23 @@ def __set_original_version(self, comp_details): def registry_to_component(self, comp_name, comp_details, comp_style): self.name = comp_name self.fixture = comp_details.get("fixture", False) - # local/remote - start + # local/remote if self.fixture: self.local = "." - repo_url = get_current_remote_url() - p = urlparse(repo_url) - last_url_node = p.path.rsplit("/")[-1] - self.remote = "../" + last_url_node + self.remote = get_current_remote_url() else: - # Assume the flag for repostories is commercial-at - repo_flag = "@" - - # To make it easier to loop over the local path, split into a list - local_list = splitall(comp_details["local"]) - - # The last node of the path is what we will decorate - last_node = local_list[-1] - - # Add that final node to a list - original_final_node_list.append(last_node) - - # Now we need to decorate all the final nodes since we can have - # nested repos with mepo - for item in original_final_node_list: - try: - # Find the index of every "final node" in a local path - # for nesting - index = local_list.index(item) - - # Decorate all final nodes - local_list[index] = decorate_node(item, repo_flag, comp_style) - except ValueError: - pass - - # Now pull the list of nodes back into a path - self.local = os.path.join(*local_list) - # print(f'final self.local: {self.local}') - + self.local = stylize_local_path(comp_details["local"], comp_style) self.remote = comp_details["remote"] - # local/remote - end - self.sparse = comp_details.get("sparse", None) # sparse is optional - self.develop = comp_details.get("develop", None) # develop is optional - self.recurse_submodules = comp_details.get( - "recurse_submodules", None - ) # recurse_submodules is optional - self.ignore_submodules = comp_details.get( - "ignore_submodules", None - ) # ignore_submodules is optional + if self.remote.startswith("../"): + self.remote = urljoin(get_current_remote_url() + "/", self.remote) + # Optionals (None, if missing) + self.sparse = comp_details.get("sparse", None) + self.develop = comp_details.get("develop", None) + self.recurse_submodules = comp_details.get("recurse_submodules", None) + self.ignore_submodules = comp_details.get("ignore_submodules", None) + # version self.__set_original_version(comp_details) + return self def to_registry_format(self): @@ -204,27 +187,34 @@ def serialize(self): return d -def get_current_remote_url(): - cmd = "git remote get-url origin" - output = shellcmd.run(shlex.split(cmd), output=True).strip() - return output +def stylize_local_path(local_path, style): + repo_flag = "@" # Assumed flag for repos + local_list = splitall(local_path) + last_node = local_list[-1] + last_node_list.append(last_node) # maintain a list of last nodes + # Decorate ALL last nodes since we can have nested repos + for item in last_node_list: + try: + index = local_list.index(item) + local_list[index] = decorate_node(item, repo_flag, style) + except ValueError: + pass + return os.path.join(*local_list) def decorate_node(item, flag, style): - # If we do not pass in a style... if not style: - # Just use what's in components.yaml - return item - # else use the style + return item # use what's in the registry file + item = item.replace(flag, "") # remove existing flag + if style == "naked": + output = item + elif style == "prefix": + output = flag + item + elif style == "postfix": + output = item + flag else: - item = item.replace(flag, "") - if style == "naked": - output = item - elif style == "prefix": - output = flag + item - elif style == "postfix": - output = item + flag - return output + raise Exception(f"Invalid style: {style}") + return output # From https://learning.oreilly.com/library/view/python-cookbook/0596001673/ch04s16.html diff --git a/src/mepo/git.py b/src/mepo/git.py index 4518a549..5975ab17 100644 --- a/src/mepo/git.py +++ b/src/mepo/git.py @@ -5,7 +5,6 @@ from urllib.parse import urljoin -from .state import MepoState from .utilities import shellcmd from .utilities import colors from .utilities.exceptions import RepoAlreadyClonedError @@ -21,33 +20,26 @@ def get_editor(): return result.stdout.rstrip().decode("utf-8") # byte to utf-8 +def get_current_remote_url(): + cmd = "git remote get-url origin" + output = shellcmd.run(shlex.split(cmd), output=True).strip() + return output + + class GitRepository: """ Class to consolidate git commands """ - __slots__ = ["__local", "__full_local_path", "__remote", "__git"] - - def __init__(self, remote_url, local_path): - self.__local = local_path - - if remote_url.startswith(".."): - rel_remote = os.path.basename(remote_url) - fixture_url = get_current_remote_url() - self.__remote = urljoin(fixture_url, rel_remote) - else: - self.__remote = remote_url + __slots__ = ["__local_path_abs", "__remote", "__git"] - root_dir = MepoState.get_root_dir() - full_local_path = os.path.normpath(os.path.join(root_dir, local_path)) - self.__full_local_path = full_local_path - self.__git = 'git -C "{}"'.format(self.__full_local_path) + def __init__(self, remote_url, local_path_abs): + self.__local_path_abs = local_path_abs + self.__remote = remote_url + self.__git = 'git -C "{}"'.format(self.__local_path_abs) def get_local_path(self): - return self.__local - - def get_full_local_path(self): - return self.__full_local_path + return self.__local_path_abs def get_remote_url(self): return self.__remote @@ -63,15 +55,15 @@ def clone(self, version, recurse, type, comp_name, partial=None): if recurse: cmd1 += "--recurse-submodules " - cmd1 += "--quiet {} {}".format(self.__remote, self.__full_local_path) + cmd1 += "--quiet {} {}".format(self.__remote, self.__local_path_abs) try: shellcmd.run(shlex.split(cmd1)) except sp.CalledProcessError: raise RepoAlreadyClonedError(f"Error! Repo [{comp_name}] already cloned") - cmd2 = "git -C {} checkout {}".format(self.__full_local_path, version) + cmd2 = "git -C {} checkout {}".format(self.__local_path_abs, version) shellcmd.run(shlex.split(cmd2)) - cmd3 = "git -C {} checkout --detach".format(self.__full_local_path) + cmd3 = "git -C {} checkout --detach".format(self.__local_path_abs) shellcmd.run(shlex.split(cmd3)) # NOTE: The above looks odd because of a quirk of git. You can't do @@ -88,7 +80,7 @@ def checkout(self, version, detach=False): shellcmd.run(shlex.split(cmd2)) def sparsify(self, sparse_config): - dst = os.path.join(self.__full_local_path, ".git", "info", "sparse-checkout") + dst = os.path.join(self.__local_path_abs, ".git", "info", "sparse-checkout") os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copy(sparse_config, dst) cmd1 = self.__git + " config core.sparseCheckout true" @@ -144,7 +136,7 @@ def show_stash(self, patch): return output.rstrip() def run_diff(self, args=None, ignore_submodules=False): - cmd = "git -C {}".format(self.__full_local_path) + cmd = "git -C {}".format(self.__local_path_abs) if args.ignore_permissions: cmd += " -c core.fileMode=false" cmd += " diff --color" @@ -183,7 +175,7 @@ def create_tag(self, tag_name, annotate, message, tf_file=None): cmd = [ "git", "-C", - self.__full_local_path, + self.__local_path_abs, "tag", "-a", "-F", @@ -194,7 +186,7 @@ def create_tag(self, tag_name, annotate, message, tf_file=None): cmd = [ "git", "-C", - self.__full_local_path, + self.__local_path_abs, "tag", "-a", "-m", @@ -204,7 +196,7 @@ def create_tag(self, tag_name, annotate, message, tf_file=None): else: raise Exception("This should not happen") else: - cmd = ["git", "-C", self.__full_local_path, "tag", tag_name] + cmd = ["git", "-C", self.__local_path_abs, "tag", tag_name] shellcmd.run(cmd) def delete_branch(self, branch_name, force): @@ -241,7 +233,7 @@ def verify_branch_or_tag(self, ref_name): return status, ref_type def check_status(self, ignore_permissions=False, ignore_submodules=False): - cmd = "git -C {}".format(self.__full_local_path) + cmd = "git -C {}".format(self.__local_path_abs) if ignore_permissions: cmd += " -c core.fileMode=false" cmd += " status --porcelain=v2" @@ -488,9 +480,9 @@ def unstage_file(self, myfile): def commit_files(self, message, tf_file=None): if tf_file: - cmd = ["git", "-C", self.__full_local_path, "commit", "-F", tf_file] + cmd = ["git", "-C", self.__local_path_abs, "commit", "-F", tf_file] elif message: - cmd = ["git", "-C", self.__full_local_path, "commit", "-m", message] + cmd = ["git", "-C", self.__local_path_abs, "commit", "-m", message] else: raise Exception("This should not happen") shellcmd.run(cmd) @@ -571,9 +563,3 @@ def get_version(self): name = hash_out.rstrip() tYpe = "h" return (name, tYpe, detached) - - -def get_current_remote_url(): - cmd = "git remote get-url origin" - output = shellcmd.run(shlex.split(cmd), output=True).strip() - return output diff --git a/tests/test_component.py b/tests/test_component.py new file mode 100644 index 00000000..bb5d7e60 --- /dev/null +++ b/tests/test_component.py @@ -0,0 +1,68 @@ +import os + +from mepo.component import stylize_local_path +from mepo.component import MepoComponent +from mepo.registry import Registry +from mepo.utilities.version import MepoVersion + +from _pytest.assertion import truncate + +truncate.DEFAULT_MAX_LINES = 9999 +truncate.DEFAULT_MAX_CHARS = 9999 + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + + +def get_registry(): + registry = os.path.join(TEST_DIR, "input", "components.yaml") + return Registry(registry).read_file() + + +def get_fvdycore_component(): + return MepoComponent( + name="fvdycore", + local="./src/Components/@FVdycoreCubed_GridComp/@fvdycore", + remote="https://github.com/GEOS-ESM/GFDL_atmos_cubed_sphere.git", + version=MepoVersion(name="geos/v1.3.0", type="t", detached=True), + sparse=None, + develop="geos/develop", + recurse_submodules=None, + fixture=False, + ignore_submodules=None, + ) + + +def get_fvdycore_serialized(): + return { + "name": "fvdycore", + "local": "./src/Components/@FVdycoreCubed_GridComp/@fvdycore", + "remote": "https://github.com/GEOS-ESM/GFDL_atmos_cubed_sphere.git", + "version": ["geos/v1.3.0", "t", True], + "sparse": None, + "develop": "geos/develop", + "recurse_submodules": None, + "fixture": False, + "ignore_submodules": None, + } + + +def test_stylize_local_path(): + local_path = "./src/Shared/@GMAO_Shared/@GEOS_Util" + output = stylize_local_path(local_path, None) + assert output == local_path + output = stylize_local_path(local_path, "prefix") + assert output == local_path + output = stylize_local_path(local_path, "naked") + assert output == "./src/Shared/@GMAO_Shared/GEOS_Util" + output = stylize_local_path(local_path, "postfix") + assert output == "./src/Shared/@GMAO_Shared/GEOS_Util@" + + +def test_MepoComponent(): + registry = get_registry() + complist = list() + for name, comp in registry.items(): + if name == "fvdycore": + fvdycore = MepoComponent().registry_to_component(name, comp, None) + assert fvdycore == get_fvdycore_component() + assert fvdycore.serialize() == get_fvdycore_serialized() diff --git a/tests/test_mepo_commands.py b/tests/test_mepo_commands.py index 19966da4..ac7e6bce 100644 --- a/tests/test_mepo_commands.py +++ b/tests/test_mepo_commands.py @@ -337,7 +337,7 @@ def test_reset(self): self.__class__.__mepo_clone() def test_mepo_version(self): - self.assertEqual(get_mepo_version(), "2.0.0") + self.assertEqual(get_mepo_version(), "2.1.0") def tearDown(self): pass diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 00000000..f7c47639 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,19 @@ +import os + +from mepo.registry import Registry + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + + +def get_ecbuild_details(): + return { + "local": "./@cmake/@ecbuild", + "remote": "../ecbuild.git", + "tag": "geos/v1.2.0", + } + + +def test_registry(): + registry = os.path.join(TEST_DIR, "input", "components.yaml") + a = Registry(registry).read_file() + assert a["ecbuild"] == get_ecbuild_details()