Merge pull request #20 from cfengine/index

CFE-3762: Added options to specify alternate index

cfbs/commands.py

@@ -21,6 +21,7 @@
 )
 
 from cfbs.pretty import pretty_file, pretty
+from cfbs.index import Index
 
 
 def cfbs_filename() -> str:
@@ -52,35 +53,6 @@ def put_definition(data: dict):
         f.write(pretty(data))
 
 
-index = None
-
-
-def index_url() -> str:
-    return "https://raw.githubusercontent.com/cfengine/cfbs-index/master/index.json"
-
-
-def index_path() -> str:
-    return cfbs_dir("index.json")
-
-
-def get_index(prefer_offline=False) -> dict:
-    global index
-    if not index and prefer_offline:
-        index = read_json(index_path())
-    if not index:
-        index = get_json(index_url())
-        if not index:
-            assert not prefer_offline
-            index = read_json(index_path())
-            if index:
-                print("Warning: Downloading index failed, using cache")
-    if not index:
-        sys.exit("Could not download or find module index")
-    if "modules" not in index:
-        sys.exit("Empty or invalid module index")
-    return index["modules"]
-
-
 def pretty_command(filenames: list) -> int:
     if not filenames:
         user_error("Filenames missing for cfbs pretty command")
@@ -135,19 +107,31 @@ def status_command() -> int:
     return 0
 
 
-def search_command(terms: list) -> int:
+def get_index_from_config():
+    if not os.path.isfile(cfbs_filename()):
+        return None
+    conf = get_definition()
+    if not "index" in conf:
+        return None
+    return conf["index"]
+
+
+def search_command(terms: list, index=None) -> int:
+    if not index:
+        index = get_index_from_config()
+    index = Index(index)
     found = False
     # No search term, list everything:
     if not terms:
-        for name, data in get_index().items():
+        for name, data in index.get_modules().items():
             if "alias" in data:
                 continue
             print(name)
             found = True
         return 0 if found else 1
 
     # Print all modules which match at least 1 search term:
-    for name, data in get_index().items():
+    for name, data in index.get_modules().items():
         if any((t for t in terms if t in name)):
             if "alias" in data:
                 print(f"{name} -> {data['alias']}")
@@ -157,10 +141,6 @@ def search_command(terms: list) -> int:
     return 0 if found else 1
 
 
-def module_exists(module_name):
-    return os.path.exists(module_name) or (module_name in get_index())
-
-
 def local_module_name(module_path):
     assert os.path.exists(module_path)
     module = module_path
@@ -260,27 +240,24 @@ def local_module_copy(module, counter, max_length):
     )
 
 
-def get_build_step(module):
-    return (
-        get_index()[module]
-        if not module.startswith("./")
-        else local_module_data(module)
-    )
-
-
-def add_command(to_add: list, added_by="cfbs add") -> int:
+def add_command(to_add: list, added_by="cfbs add", index=None) -> int:
     if not to_add:
         user_error("Must specify at least one module to add")
 
+    if not index:
+        index = get_index_from_config()
+
+    index = Index(index)
+
     # Translate all aliases:
     translated = []
     for module in to_add:
-        if not module_exists(module):
+        if not index.exists(module):
             user_error(f"Module '{module}' does not exist")
-        if not module in get_index() and os.path.exists(module):
+        if not module in index and os.path.exists(module):
             translated.append(local_module_name(module))
             continue
-        data = get_index()[module]
+        data = index[module]
         if "alias" in data:
             print(f'{module} is an alias for {data["alias"]}')
             module = data["alias"]
@@ -303,7 +280,7 @@ def add_command(to_add: list, added_by="cfbs add") -> int:
     assert not any((k not in added_by for k in to_add))
 
     # Print error and exit if there are unknown modules:
-    missing = [m for m in to_add if not m.startswith("./") and m not in get_index()]
+    missing = [m for m in to_add if not m.startswith("./") and m not in index]
     if missing:
         user_error(f"Module(s) could not be found: {', '.join(missing)}")
 
@@ -331,8 +308,8 @@ def add_command(to_add: list, added_by="cfbs add") -> int:
     dependencies = []
     dependencies_added_by = []
     for module in filtered:
-        assert module_exists(module)
-        data = get_build_step(module)
+        assert index.exists(module)
+        data = index.get_build_step(module)
         assert "alias" not in data
         if "dependencies" in data:
             for dep in data["dependencies"]:
@@ -345,8 +322,8 @@ def add_command(to_add: list, added_by="cfbs add") -> int:
         definition = get_definition()
 
     for module in filtered:
-        assert module_exists(module)
-        data = get_build_step(module)
+        assert index.exists(module)
+        data = index.get_build_step(module)
         new_module = {"name": module, **data, "added_by": added_by[module]}
         definition["build"].append(new_module)
         if user_requested:

cfbs/index.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+from cf_remote.paths import cfengine_dir
+
+from cfbs.utils import (
+    user_error,
+    get_json,
+    strip_left,
+    strip_right,
+    pad_right,
+    write_json,
+    read_json,
+    merge_json,
+    mkdir,
+    touch,
+    rm,
+    cp,
+    sh,
+)
+
+from cfbs.pretty import pretty_file, pretty
+
+
+class Index:
+    def __init__(self, path):
+        self.path = path
+        if not self.path:
+            self.path = "https://raw.githubusercontent.com/cfengine/cfbs-index/master/index.json"
+        self._data = None
+
+    def __contains__(self, key):
+        return key in self.get_modules()
+
+    def __getitem__(self, key):
+        return self.get_modules()[key]
+
+    def _cache_path(self) -> str:
+        return cfbs_dir("index.json")
+
+    def _get(self) -> dict:
+        path = self.path
+        if path.startswith("https://"):
+            index = get_json(path)
+            if not index:
+                index = read_json(self._cache_path())
+                if index:
+                    print("Warning: Downloading index failed, using cache")
+        else:
+            if not os.path.isfile(path):
+                sys.exit(f"Index does not exist at: '{path}'")
+            index = read_json(path)
+        if not index:
+            sys.exit("Could not download or find module index")
+        if "modules" not in index:
+            sys.exit("Empty or invalid module index")
+        return index
+
+    def get(self) -> dict:
+        if not self._data:
+            self._data = self._get()
+        return self._data
+
+    def get_modules(self) -> dict:
+        return self.get()["modules"]
+
+    def exists(self, module):
+        return os.path.exists(module) or (module in self)
+
+    def get_build_step(self, module):
+        return (
+            self.get_modules()[module]
+            if not module.startswith("./")
+            else local_module_data(module)
+        )

cfbs/main.py

@@ -35,6 +35,7 @@ def get_args():
     parser.add_argument(
         "--force", help="Force rebuild / redownload", action="store_true"
     )
+    parser.add_argument("--index", help="Specify alternate index", type=str)
 
     args = parser.parse_args()
     return args
@@ -71,7 +72,7 @@ def main() -> int:
     if args.command == "init":
         return commands.init_command()
     if args.command == "search":
-        return commands.search_command(args.args)
+        return commands.search_command(args.args, index=args.index)
     if args.command == "pretty":
         return commands.pretty_command(args.args)
 
@@ -81,7 +82,7 @@ def main() -> int:
     if args.command == "status":
         return commands.status_command()
     if args.command == "add":
-        return commands.add_command(args.args)
+        return commands.add_command(args.args, index=args.index)
     if args.command == "download":
         return commands.download_command(args.force)
     if args.command == "build":