code organization and add more documentation (#23)

Three things are done here: * move codes too basic to `utils` * refactor `version_utils` by extending the `semantic_version.Version` * add more docs
johnnychen94 · Feb 19, 2020 · 7f79246 · 7f79246
1 parent 6e39e52
commit 7f79246
Show file tree

Hide file tree

Showing 20 changed files with 403 additions and 141 deletions.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,7 @@
 JULIA_VERSIONS = "1" "1.0" "1.1" "1.2" "1.3" "1.4.0-rc1" "latest"
 unittest:
 	python -m unittest jill/tests/tests_filters.py
+	python -m unittest jill/tests/tests_versions.py
 
 download_install_test:
 	# check if upstream works

diff --git a/README.md b/README.md
@@ -60,7 +60,8 @@ Here's a list of slightly advanced usages that you may be interested in:
     - add a private upstream: make a modifed copy of [public registry](jill/config/sources.json) at:
         * Linux, MacOS and FreeBSD: `~/.config/jill/sources.json`
         * Windows: `~/AppData/Local/julias/sources.json`
-* check the not-so-elaborative documentation: `jill [COMMAND] -h` (e.g., `jill download -h`)
+
+You can find a more verbose documentation using `jill [COMMAND] -h` (e.g., `jill download -h`)
 
 ## For who are interested in setting up a new release mirror
 

diff --git a/README_zh.md b/README_zh.md
@@ -56,7 +56,8 @@
     - 从局域网私有源下载：创建一个类似的[配置文件](jill/config/sources.json)并存放在：
         * Linux, MacOS and FreeBSD: `~/.config/jill/sources.json`
         * Windows: `~/AppData/Local/julias/sources.json`
-* 不那么详尽的说明文档: `jill [COMMAND] -h` (例如`jill install -h`)
+
+更多的参数及其作用请查看帮助文档: `jill [COMMAND] -h` (例如`jill install -h`)
 
 ## 镜像源的搭建
 

diff --git a/jill/__init__.py b/jill/__init__.py
@@ -1,7 +0,0 @@
-from .download import download_package
-from .mirror import Mirror, MirrorConfig
-
-__all__ = [
-    "download_package",
-    "Mirror", "MirrorConfig"
-]

diff --git a/jill/__main__.py b/jill/__main__.py
@@ -1,8 +1,8 @@
 from .download import download_package
 from .install import install_julia
 from .mirror import mirror
-from .version_utils import update_releases
-from .source import show_upstream
+from .utils import update_releases
+from .utils import show_upstream
 import fire
 import logging
 import os

diff --git a/jill/download.py b/jill/download.py
@@ -1,9 +1,9 @@
-from .source import SourceRegistry
-from .version_utils import latest_version
-from .version_utils import is_version_released
-from .version_utils import is_full_version
-from .sys_utils import current_system, current_architecture
-from .gpg_utils import verify_gpg
+from .utils import SourceRegistry
+from .utils import latest_version
+from .utils import is_version_released
+from .utils import is_full_version
+from .utils import current_system, current_architecture
+from .utils import verify_gpg
 
 import wget
 import os
@@ -49,18 +49,52 @@ def download_package(version=None, sys=None, arch=None, *,
     """
     download julia release from nearest servers
 
+    `jill download [version] [sys] [arch]` downloads a Julia release
+    in your current folder. If you don't specify any argument, then
+    it will download the latest stable version for your current platform.
+
+    The syntax for `version` is:
+
+    * `stable`: latest stable Julia release. This is the _default_ option.
+    * `1`: latest `1.y.z` Julia release.
+    * `1.0`: latest `1.0.z` Julia release.
+    * `1.4.0-rc1`: as it is. This is the only way to install unstable release.
+    * `latest`/`nightly`: the nightly builds from source code.
+
+    For whatever reason, if you only want to download release from
+    a specific upstream (e.g., from JuliaComputing), then you can use
+    `--upstream` flag (e.g., `jill download --upstream Official`).
+
+    To see a full list of upstream servers, please use `jill upstream`.
+
+    If you're interested in downloading from an unregistered private
+    mirror, you can provide a `sources.json` file to CONFIG_PATH and use
+    `jill upstream` to check if your mirror is added. A config template
+    can be found at [1]. For how to make such mirror, please refer to
+    `jill mirror`.
+
+    CONFIG_PATH:
+      * windows: `~\\AppData\\Local\\julias\\sources.json`
+      * other: `~/.config/jill/sources.json`
+
+    [1]: https://github.com/johnnychen94/jill.py/blob/master/jill/config/sources.json # nopep8
+
     Arguments:
-      version: Option examples: 1, 1.2, 1.2.3, latest.
+      version:
+        The Julia version you want to install. See also `jill install`
       sys: Options are: "linux", "macos", "freebsd", "windows"
       arch: Options are: "i686", "x86_64", "ARMv7", "ARMv8"
       upstream:
         manually choose a download upstream. For example, set it to "Official"
         if you want to download from JuliaComputing's s3 buckets.
-      outdir: where release is downloaded to. By default it's current folder.
-      overwrite: True to overwrite existing releases. By default it's False.
-      max_try: try `max_try` times before returning a False.
+      outdir:
+        where release is downloaded to. By default it's the current folder.
+      overwrite:
+        add `--overwrite` flag to overwrite existing releases.
+      max_try:
+        try `max_try` times before returning a False. The default value is 3.
     """
-    version = str(version) if version else ''
+    version = str(version) if version else ""
     version = "latest" if version == "nightly" else version
     version = "" if version == "stable" else version
 
@@ -124,7 +158,7 @@ def download_package(version=None, sys=None, arch=None, *,
         # that are verified by the operating system during installation
         return package_path
     elif system in ["linux", "freebsd"]:
-        # additional verification using GPG
+        # need additional verification using GPG
         if not package_path:
             return package_path
 
@@ -146,7 +180,7 @@ def download_package(version=None, sys=None, arch=None, *,
             return False
 
         # GPG-verified julia release path
-        logging.info(f"success to verify {release_str} downloads")
+        logging.info(f"success to verify {release_str}")
         return package_path
     else:
         raise ValueError(f"unsupported system {sys}")
diff --git a/jill/install.py b/jill/install.py
@@ -1,10 +1,11 @@
-from .filters import f_major_version, f_minor_version, f_patch_version
+from .utils.filters import f_major_version, f_minor_version, f_patch_version
+from .utils import query_yes_no
+from .utils import current_architecture, current_system
+from .utils import latest_version
+from .utils import DmgMounter, TarMounter
+from .utils import Version
 from .download import download_package
-from .interactive_utils import query_yes_no
-from .sys_utils import current_architecture, current_system
-from .version_utils import latest_version
-from .mount_utils import DmgMounter, TarMounter
-from semantic_version import Version
+
 import os
 import getpass
 import shutil
@@ -56,6 +57,13 @@ def get_exec_version(path):
     return version
 
 
+def check_installer(installer_path, ext):
+    filename = os.path.basename(installer_path)
+    if not filename.endswith(ext):
+        msg = f"The installer {filename} should be {ext} file"
+        raise ValueError(msg)
+
+
 def last_julia_version(version=None):
     # version should follow semantic version syntax
     def sort_key(ver):
@@ -114,13 +122,13 @@ def make_symlinks(src_bin, symlink_dir, version):
         # symlink rules:
         # 1. always symlink latest
         # 2. only make new symlink if it's a newer version
-        if version != "latest":
-            if os.path.exists(linkpath) or os.path.islink(linkpath):
-                old_ver = get_exec_version(linkpath)
-                if Version(old_ver) > Version(version):
-                    continue
-                logging.info(f"removing previous symlink {linkname}")
-                os.remove(linkpath)
+        if os.path.exists(linkpath) or os.path.islink(linkpath):
+            old_ver = get_exec_version(linkpath)
+            if Version(old_ver) > Version(version):
+                # it's always false if version == "latest"
+                continue
+            logging.info(f"removing previous symlink {linkname}")
+            os.remove(linkpath)
         logging.info(f"make symlink {linkpath}")
         if current_system() == "windows":
             with open(linkpath, 'w') as f:
@@ -154,6 +162,8 @@ def install_julia_linux(package_path,
                         symlink_dir,
                         version,
                         upgrade):
+    check_installer(package_path, ".tar.gz")
+
     mver = f_minor_version(version)
     with TarMounter(package_path) as root:
         src_path = root
@@ -175,7 +185,8 @@ def install_julia_mac(package_path,
                       symlink_dir,
                       version,
                       upgrade):
-    assert os.path.splitext(package_path)[1] == ".dmg"
+    check_installer(package_path, ".dmg")
+
     with DmgMounter(package_path) as root:
         # mounted image contents:
         #   ['.VolumeIcon.icns', 'Applications', 'Julia-1.3.app']
@@ -200,19 +211,18 @@ def install_julia_windows(package_path,
                           symlink_dir,
                           version,
                           upgrade):
-    assert os.path.splitext(package_path)[1] == ".exe"
+    check_installer(package_path, ".exe")
 
     dest_path = os.path.join(install_dir,
                              f"julia-{f_minor_version(version)}")
     if os.path.exists(dest_path):
         logging.info(f"remove previous julia installation: {dest_path}")
         shutil.rmtree(dest_path)
 
-    if version == "latest":
-        ver = "999.999.999"
-    else:
-        ver = version
-    if Version(ver).next_patch() < Version("1.4.0"):
+    # build system changes for windows after 1.4
+    # https://github.com/JuliaLang/julia/blob/release-1.4/NEWS.md#build-system-changes
+    if Version(version).next_patch() < Version("1.4.0"):
+        # it's always false if version == "latest"
         subprocess.check_output([f'{package_path}',
                                  '/S', f'/D={dest_path}'])
     else:
@@ -234,16 +244,39 @@ def install_julia(version=None, *,
                   keep_downloads=False,
                   confirm=False):
     """
-    Install julia for Linux and MacOS
+    Install the Julia programming language for your current system
+
+    `jill install [version]` would satisfy most of your use cases, try it first
+    and then read description of other arguments. `version` is optional, valid
+    version syntax for it is:
+
+    * `stable`: latest stable Julia release. This is the _default_ option.
+    * `1`: latest `1.y.z` Julia release.
+    * `1.0`: latest `1.0.z` Julia release.
+    * `1.4.0-rc1`: as it is. This is the only way to install unstable release.
+    * `latest`/`nightly`: the nightly builds from source code.
+
+    For Linux/FreeBSD systems, if you run this command with `root` account,
+    then it will install Julia system-widely.
+
+    To download from a private mirror, please check `jill download -h`.
 
     Arguments:
-      version: Option examples: 1, 1.2, 1.2.3, latest.
+      version:
+        The Julia version you want to install.
       upstream:
         manually choose a download upstream. For example, set it to "Official"
         if you want to download from JuliaComputing's s3 buckets.
-      upgrade: True to also copy the root environment from older julia version.
-      keep_downloads: True to not remove downloaded releases.
-      confirm: add `--confirm` to skip interactive prompt.
+      upgrade:
+        add `--upgrade` flag also copy the root environment from an older
+        Julia version.
+      keep_downloads:
+        add `--keep_downloads` flag to not remove downloaded releases.
+      confirm: add `--confirm` flag to skip interactive prompt.
+      install_dir:
+        where you want julia packages installed.
+      symlink_dir:
+        where you want symlinks(e.g., `julia`, `julia-1`) placed.
     """
     install_dir = install_dir if install_dir else default_install_dir()
     symlink_dir = symlink_dir if symlink_dir else default_symlink_dir()
@@ -277,6 +310,7 @@ def install_julia(version=None, *,
     if system == "macos":
         installer = install_julia_mac
     elif system in ["linux", "freebsd"]:
+        # technically it's tarball installer
         installer = install_julia_linux
     elif system == "windows":
         installer = install_julia_windows
@@ -286,6 +320,7 @@ def install_julia(version=None, *,
     installer(package_path, install_dir, symlink_dir, version, upgrade)
 
     if not keep_downloads:
+        logging.info("----- After Installation ----- ")
         logging.info("remove downloaded files")
         logging.info(f"remove {package_path}")
         os.remove(package_path)

diff --git a/jill/mirror.py b/jill/mirror.py
@@ -1,27 +1,26 @@
-from .defaults import default_path_template
-from .defaults import default_filename_template
-from .defaults import default_latest_filename_template
+from .utils.defaults import default_path_template
+from .utils.defaults import default_filename_template
+from .utils.defaults import default_latest_filename_template
+from .utils import generate_info, is_valid_release
+from .utils import update_releases
+from .utils import read_releases
+from .utils import current_system
 from .download import download_package
-from .filters import generate_info, is_valid_release
-from .version_utils import update_releases
-from .version_utils import read_releases
-from .sys_utils import current_system
 
 from string import Template
-from itertools import product
-from semantic_version import Version
+
+import semantic_version
 
 import json
 import os
 import logging
 import time
 
-from typing import List
-
 
 class MirrorConfig:
     def __init__(self, configfile, outdir):
         if current_system() == "windows":
+            # Windows users (e.g., me) sometimes confuse the use of \\ and \
             outdir = outdir.replace("\\\\", "\\")
         self.configfile = os.path.abspath(os.path.expanduser(configfile))
         self.outdir = outdir
@@ -66,7 +65,8 @@ def latest_filename_template(self):
     def version(self):
         versions = list(set(map(lambda x: x[0],
                                 read_releases(stable_only=True))))
-        versions.sort(key=lambda ver: Version(ver))
+        # not using our extended Version
+        versions.sort(key=lambda ver: semantic_version.Version(ver))
         if self.require_latest:
             versions.append("latest")
         return versions
@@ -133,14 +133,24 @@ def mirror(outdir="julia_pkg", *,
            logfile="mirror.log",
            config="mirror.json"):
     """
-    periodly download/sync all Julia releases
+    Download/sync all Julia releases
+
+    If you want to modify the default mirror configuration, then provide
+    a `mirror.json` file and pass the path to `config`. By default it's at
+    the current directory. A template can be found at [1]
+
+    [1]: https://github.com/johnnychen94/jill.py/blob/master/mirror.example.json # nopep8
 
     Arguments:
-    outdir: default 'julia_pkg'.
-    period: the time between two sync operation. 0(default) to sync once.
-    upstream:
+      outdir: default 'julia_pkg'.
+      period: the time between two sync operation. 0(default) to sync once.
+      upstream:
         manually choose a download upstream. For example, set it to "Official"
         if you want to download from JuliaComputing's s3 buckets.
+      config:
+        path to mirror config file
+      logfile:
+        path to mirror log file
     """
     log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
     period = int(period)