From 7f792460263913b135bc7b3b50372e5b9f076c74 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Thu, 20 Feb 2020 04:29:24 +0800 Subject: [PATCH] code organization and add more documentation (#23) Three things are done here: * move codes too basic to `utils` * refactor `version_utils` by extending the `semantic_version.Version` * add more docs --- Makefile | 1 + README.md | 3 +- README_zh.md | 3 +- jill/__init__.py | 7 - jill/__main__.py | 4 +- jill/download.py | 60 +++++++-- jill/install.py | 85 +++++++++---- jill/mirror.py | 42 +++--- jill/tests/tests_filters.py | 32 +++-- jill/tests/tests_versions.py | 73 +++++++++++ jill/utils/__init__.py | 45 +++++++ jill/{ => utils}/defaults.py | 5 +- jill/{ => utils}/filters.py | 11 +- jill/{ => utils}/gpg_utils.py | 1 + jill/{ => utils}/interactive_utils.py | 0 jill/{ => utils}/mount_utils.py | 5 +- jill/{ => utils}/net_utils.py | 4 +- jill/{source.py => utils/source_utils.py} | 11 +- jill/{ => utils}/sys_utils.py | 4 + jill/{ => utils}/version_utils.py | 148 ++++++++++++++-------- 20 files changed, 403 insertions(+), 141 deletions(-) create mode 100644 jill/tests/tests_versions.py create mode 100644 jill/utils/__init__.py rename jill/{ => utils}/defaults.py (86%) rename jill/{ => utils}/filters.py (96%) rename jill/{ => utils}/gpg_utils.py (93%) rename jill/{ => utils}/interactive_utils.py (100%) rename jill/{ => utils}/mount_utils.py (96%) rename jill/{ => utils}/net_utils.py (100%) rename jill/{source.py => utils/source_utils.py} (96%) rename jill/{ => utils}/sys_utils.py (85%) rename jill/{ => utils}/version_utils.py (66%) diff --git a/Makefile b/Makefile index 436af3e..56a2d40 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ JULIA_VERSIONS = "1" "1.0" "1.1" "1.2" "1.3" "1.4.0-rc1" "latest" unittest: python -m unittest jill/tests/tests_filters.py + python -m unittest jill/tests/tests_versions.py download_install_test: # check if upstream works diff --git a/README.md b/README.md index 39d727e..59046b2 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ Here's a list of slightly advanced usages that you may be interested in: - add a private upstream: make a modifed copy of [public registry](jill/config/sources.json) at: * Linux, MacOS and FreeBSD: `~/.config/jill/sources.json` * Windows: `~/AppData/Local/julias/sources.json` -* check the not-so-elaborative documentation: `jill [COMMAND] -h` (e.g., `jill download -h`) + +You can find a more verbose documentation using `jill [COMMAND] -h` (e.g., `jill download -h`) ## For who are interested in setting up a new release mirror diff --git a/README_zh.md b/README_zh.md index 880a0ca..c2c4cff 100644 --- a/README_zh.md +++ b/README_zh.md @@ -56,7 +56,8 @@ - 从局域网私有源下载：创建一个类似的[配置文件](jill/config/sources.json)并存放在： * Linux, MacOS and FreeBSD: `~/.config/jill/sources.json` * Windows: `~/AppData/Local/julias/sources.json` -* 不那么详尽的说明文档: `jill [COMMAND] -h` (例如`jill install -h`) + +更多的参数及其作用请查看帮助文档: `jill [COMMAND] -h` (例如`jill install -h`) ## 镜像源的搭建 diff --git a/jill/__init__.py b/jill/__init__.py index bdc71e8..e69de29 100644 --- a/jill/__init__.py +++ b/jill/__init__.py @@ -1,7 +0,0 @@ -from .download import download_package -from .mirror import Mirror, MirrorConfig - -__all__ = [ - "download_package", - "Mirror", "MirrorConfig" -] diff --git a/jill/__main__.py b/jill/__main__.py index 559e1bc..4aa17bf 100644 --- a/jill/__main__.py +++ b/jill/__main__.py @@ -1,8 +1,8 @@ from .download import download_package from .install import install_julia from .mirror import mirror -from .version_utils import update_releases -from .source import show_upstream +from .utils import update_releases +from .utils import show_upstream import fire import logging import os diff --git a/jill/download.py b/jill/download.py index bbecf21..4c93862 100644 --- a/jill/download.py +++ b/jill/download.py @@ -1,9 +1,9 @@ -from .source import SourceRegistry -from .version_utils import latest_version -from .version_utils import is_version_released -from .version_utils import is_full_version -from .sys_utils import current_system, current_architecture -from .gpg_utils import verify_gpg +from .utils import SourceRegistry +from .utils import latest_version +from .utils import is_version_released +from .utils import is_full_version +from .utils import current_system, current_architecture +from .utils import verify_gpg import wget import os @@ -49,18 +49,52 @@ def download_package(version=None, sys=None, arch=None, *, """ download julia release from nearest servers + `jill download [version] [sys] [arch]` downloads a Julia release + in your current folder. If you don't specify any argument, then + it will download the latest stable version for your current platform. + + The syntax for `version` is: + + * `stable`: latest stable Julia release. This is the _default_ option. + * `1`: latest `1.y.z` Julia release. + * `1.0`: latest `1.0.z` Julia release. + * `1.4.0-rc1`: as it is. This is the only way to install unstable release. + * `latest`/`nightly`: the nightly builds from source code. + + For whatever reason, if you only want to download release from + a specific upstream (e.g., from JuliaComputing), then you can use + `--upstream` flag (e.g., `jill download --upstream Official`). + + To see a full list of upstream servers, please use `jill upstream`. + + If you're interested in downloading from an unregistered private + mirror, you can provide a `sources.json` file to CONFIG_PATH and use + `jill upstream` to check if your mirror is added. A config template + can be found at [1]. For how to make such mirror, please refer to + `jill mirror`. + + CONFIG_PATH: + * windows: `~\\AppData\\Local\\julias\\sources.json` + * other: `~/.config/jill/sources.json` + + [1]: https://github.com/johnnychen94/jill.py/blob/master/jill/config/sources.json # nopep8 + Arguments: - version: Option examples: 1, 1.2, 1.2.3, latest. + version: + The Julia version you want to install. See also `jill install` sys: Options are: "linux", "macos", "freebsd", "windows" arch: Options are: "i686", "x86_64", "ARMv7", "ARMv8" upstream: manually choose a download upstream. For example, set it to "Official" if you want to download from JuliaComputing's s3 buckets. - outdir: where release is downloaded to. By default it's current folder. - overwrite: True to overwrite existing releases. By default it's False. - max_try: try `max_try` times before returning a False. + outdir: + where release is downloaded to. By default it's the current folder. + overwrite: + add `--overwrite` flag to overwrite existing releases. + max_try: + try `max_try` times before returning a False. The default value is 3. """ - version = str(version) if version else '' + version = str(version) if version else "" version = "latest" if version == "nightly" else version version = "" if version == "stable" else version @@ -124,7 +158,7 @@ def download_package(version=None, sys=None, arch=None, *, # that are verified by the operating system during installation return package_path elif system in ["linux", "freebsd"]: - # additional verification using GPG + # need additional verification using GPG if not package_path: return package_path @@ -146,7 +180,7 @@ def download_package(version=None, sys=None, arch=None, *, return False # GPG-verified julia release path - logging.info(f"success to verify {release_str} downloads") + logging.info(f"success to verify {release_str}") return package_path else: raise ValueError(f"unsupported system {sys}") diff --git a/jill/install.py b/jill/install.py index bfea5aa..80085fc 100644 --- a/jill/install.py +++ b/jill/install.py @@ -1,10 +1,11 @@ -from .filters import f_major_version, f_minor_version, f_patch_version +from .utils.filters import f_major_version, f_minor_version, f_patch_version +from .utils import query_yes_no +from .utils import current_architecture, current_system +from .utils import latest_version +from .utils import DmgMounter, TarMounter +from .utils import Version from .download import download_package -from .interactive_utils import query_yes_no -from .sys_utils import current_architecture, current_system -from .version_utils import latest_version -from .mount_utils import DmgMounter, TarMounter -from semantic_version import Version + import os import getpass import shutil @@ -56,6 +57,13 @@ def get_exec_version(path): return version +def check_installer(installer_path, ext): + filename = os.path.basename(installer_path) + if not filename.endswith(ext): + msg = f"The installer {filename} should be {ext} file" + raise ValueError(msg) + + def last_julia_version(version=None): # version should follow semantic version syntax def sort_key(ver): @@ -114,13 +122,13 @@ def make_symlinks(src_bin, symlink_dir, version): # symlink rules: # 1. always symlink latest # 2. only make new symlink if it's a newer version - if version != "latest": - if os.path.exists(linkpath) or os.path.islink(linkpath): - old_ver = get_exec_version(linkpath) - if Version(old_ver) > Version(version): - continue - logging.info(f"removing previous symlink {linkname}") - os.remove(linkpath) + if os.path.exists(linkpath) or os.path.islink(linkpath): + old_ver = get_exec_version(linkpath) + if Version(old_ver) > Version(version): + # it's always false if version == "latest" + continue + logging.info(f"removing previous symlink {linkname}") + os.remove(linkpath) logging.info(f"make symlink {linkpath}") if current_system() == "windows": with open(linkpath, 'w') as f: @@ -154,6 +162,8 @@ def install_julia_linux(package_path, symlink_dir, version, upgrade): + check_installer(package_path, ".tar.gz") + mver = f_minor_version(version) with TarMounter(package_path) as root: src_path = root @@ -175,7 +185,8 @@ def install_julia_mac(package_path, symlink_dir, version, upgrade): - assert os.path.splitext(package_path)[1] == ".dmg" + check_installer(package_path, ".dmg") + with DmgMounter(package_path) as root: # mounted image contents: # ['.VolumeIcon.icns', 'Applications', 'Julia-1.3.app'] @@ -200,7 +211,7 @@ def install_julia_windows(package_path, symlink_dir, version, upgrade): - assert os.path.splitext(package_path)[1] == ".exe" + check_installer(package_path, ".exe") dest_path = os.path.join(install_dir, f"julia-{f_minor_version(version)}") @@ -208,11 +219,10 @@ def install_julia_windows(package_path, logging.info(f"remove previous julia installation: {dest_path}") shutil.rmtree(dest_path) - if version == "latest": - ver = "999.999.999" - else: - ver = version - if Version(ver).next_patch() < Version("1.4.0"): + # build system changes for windows after 1.4 + # https://github.com/JuliaLang/julia/blob/release-1.4/NEWS.md#build-system-changes + if Version(version).next_patch() < Version("1.4.0"): + # it's always false if version == "latest" subprocess.check_output([f'{package_path}', '/S', f'/D={dest_path}']) else: @@ -234,16 +244,39 @@ def install_julia(version=None, *, keep_downloads=False, confirm=False): """ - Install julia for Linux and MacOS + Install the Julia programming language for your current system + + `jill install [version]` would satisfy most of your use cases, try it first + and then read description of other arguments. `version` is optional, valid + version syntax for it is: + + * `stable`: latest stable Julia release. This is the _default_ option. + * `1`: latest `1.y.z` Julia release. + * `1.0`: latest `1.0.z` Julia release. + * `1.4.0-rc1`: as it is. This is the only way to install unstable release. + * `latest`/`nightly`: the nightly builds from source code. + + For Linux/FreeBSD systems, if you run this command with `root` account, + then it will install Julia system-widely. + + To download from a private mirror, please check `jill download -h`. Arguments: - version: Option examples: 1, 1.2, 1.2.3, latest. + version: + The Julia version you want to install. upstream: manually choose a download upstream. For example, set it to "Official" if you want to download from JuliaComputing's s3 buckets. - upgrade: True to also copy the root environment from older julia version. - keep_downloads: True to not remove downloaded releases. - confirm: add `--confirm` to skip interactive prompt. + upgrade: + add `--upgrade` flag also copy the root environment from an older + Julia version. + keep_downloads: + add `--keep_downloads` flag to not remove downloaded releases. + confirm: add `--confirm` flag to skip interactive prompt. + install_dir: + where you want julia packages installed. + symlink_dir: + where you want symlinks(e.g., `julia`, `julia-1`) placed. """ install_dir = install_dir if install_dir else default_install_dir() symlink_dir = symlink_dir if symlink_dir else default_symlink_dir() @@ -277,6 +310,7 @@ def install_julia(version=None, *, if system == "macos": installer = install_julia_mac elif system in ["linux", "freebsd"]: + # technically it's tarball installer installer = install_julia_linux elif system == "windows": installer = install_julia_windows @@ -286,6 +320,7 @@ def install_julia(version=None, *, installer(package_path, install_dir, symlink_dir, version, upgrade) if not keep_downloads: + logging.info("----- After Installation ----- ") logging.info("remove downloaded files") logging.info(f"remove {package_path}") os.remove(package_path) diff --git a/jill/mirror.py b/jill/mirror.py index e0a7cbe..a414206 100644 --- a/jill/mirror.py +++ b/jill/mirror.py @@ -1,27 +1,26 @@ -from .defaults import default_path_template -from .defaults import default_filename_template -from .defaults import default_latest_filename_template +from .utils.defaults import default_path_template +from .utils.defaults import default_filename_template +from .utils.defaults import default_latest_filename_template +from .utils import generate_info, is_valid_release +from .utils import update_releases +from .utils import read_releases +from .utils import current_system from .download import download_package -from .filters import generate_info, is_valid_release -from .version_utils import update_releases -from .version_utils import read_releases -from .sys_utils import current_system from string import Template -from itertools import product -from semantic_version import Version + +import semantic_version import json import os import logging import time -from typing import List - class MirrorConfig: def __init__(self, configfile, outdir): if current_system() == "windows": + # Windows users (e.g., me) sometimes confuse the use of \\ and \ outdir = outdir.replace("\\\\", "\\") self.configfile = os.path.abspath(os.path.expanduser(configfile)) self.outdir = outdir @@ -66,7 +65,8 @@ def latest_filename_template(self): def version(self): versions = list(set(map(lambda x: x[0], read_releases(stable_only=True)))) - versions.sort(key=lambda ver: Version(ver)) + # not using our extended Version + versions.sort(key=lambda ver: semantic_version.Version(ver)) if self.require_latest: versions.append("latest") return versions @@ -133,14 +133,24 @@ def mirror(outdir="julia_pkg", *, logfile="mirror.log", config="mirror.json"): """ - periodly download/sync all Julia releases + Download/sync all Julia releases + + If you want to modify the default mirror configuration, then provide + a `mirror.json` file and pass the path to `config`. By default it's at + the current directory. A template can be found at [1] + + [1]: https://github.com/johnnychen94/jill.py/blob/master/mirror.example.json # nopep8 Arguments: - outdir: default 'julia_pkg'. - period: the time between two sync operation. 0(default) to sync once. - upstream: + outdir: default 'julia_pkg'. + period: the time between two sync operation. 0(default) to sync once. + upstream: manually choose a download upstream. For example, set it to "Official" if you want to download from JuliaComputing's s3 buckets. + config: + path to mirror config file + logfile: + path to mirror log file """ log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' period = int(period) diff --git a/jill/tests/tests_filters.py b/jill/tests/tests_filters.py index 3c9ab5c..525bfa6 100644 --- a/jill/tests/tests_filters.py +++ b/jill/tests/tests_filters.py @@ -1,16 +1,22 @@ -from jill.filters import f_major_version, f_vmajor_version, f_Vmajor_version -from jill.filters import f_minor_version, f_vminor_version, f_Vminor_version -from jill.filters import f_patch_version, f_vpatch_version, f_Vpatch_version -from jill.filters import f_version -from jill.filters import f_system, f_System, f_SYSTEM -from jill.filters import f_sys, f_Sys, f_SYS -from jill.filters import f_os, f_Os, f_OS -from jill.filters import f_arch, f_Arch, f_ARCH -from jill.filters import f_osarch, f_Osarch, f_OSarch -from jill.filters import f_osbit -from jill.filters import f_bit -from jill.filters import f_extension -from jill.filters import generate_info +from jill.utils.filters import f_major_version +from jill.utils.filters import f_minor_version +from jill.utils.filters import f_patch_version +from jill.utils.filters import f_vmajor_version +from jill.utils.filters import f_vminor_version +from jill.utils.filters import f_vpatch_version +from jill.utils.filters import f_Vmajor_version +from jill.utils.filters import f_Vminor_version +from jill.utils.filters import f_Vpatch_version +from jill.utils.filters import f_version +from jill.utils.filters import f_system, f_System, f_SYSTEM +from jill.utils.filters import f_sys, f_Sys, f_SYS +from jill.utils.filters import f_os, f_Os, f_OS +from jill.utils.filters import f_arch, f_Arch, f_ARCH +from jill.utils.filters import f_osarch, f_Osarch, f_OSarch +from jill.utils.filters import f_osbit +from jill.utils.filters import f_bit +from jill.utils.filters import f_extension +from jill.utils.filters import generate_info import unittest diff --git a/jill/tests/tests_versions.py b/jill/tests/tests_versions.py new file mode 100644 index 0000000..cd92d53 --- /dev/null +++ b/jill/tests/tests_versions.py @@ -0,0 +1,73 @@ +from jill.utils.version_utils import latest_version +from jill.utils.version_utils import latest_major_version +from jill.utils.version_utils import latest_minor_version +from jill.utils.version_utils import latest_patch_version + +import unittest + + +class TestVersions(unittest.TestCase): + def test_latest_version(self): + self.assertEqual( + latest_version("latest", "windows", "x86_64"), + "latest") + self.assertEqual( + latest_version("0.7", "windows", "x86_64"), + "0.7.0") + self.assertEqual( + latest_version("1.1", "windows", "x86_64"), + "1.1.1") + self.assertEqual( + latest_version("1.0.5", "windows", "x86_64"), + "1.0.5") + self.assertEqual( + latest_version("1.4.0-rc1", "windows", "x86_64"), + "1.4.0-rc1") + self.assertEqual( + latest_version("999.999.999", "windows", "x86_64"), + "999.999.999") + + def test_latest_major_version(self): + self.assertEqual( + latest_major_version("latest", "windows", "x86_64"), + "latest") + self.assertEqual( + latest_major_version("1.0.5", "windows", "x86_64"), + "1.0.5") + self.assertEqual( + latest_major_version("1.4.0-rc1", "windows", "x86_64"), + "1.4.0-rc1") + self.assertEqual( + latest_major_version("999.999.999", "windows", "x86_64"), + "999.999.999") + + def test_latest_minor_version(self): + self.assertEqual( + latest_minor_version("latest", "windows", "x86_64"), + "latest") + self.assertEqual( + latest_minor_version("1.0.5", "windows", "x86_64"), + "1.0.5") + self.assertEqual( + latest_minor_version("1.4.0-rc1", "windows", "x86_64"), + "1.4.0-rc1") + self.assertEqual( + latest_minor_version("999.999.999", "windows", "x86_64"), + "999.999.999") + + def test_latest_patch_version(self): + self.assertEqual( + latest_patch_version("latest", "windows", "x86_64"), + "latest") + self.assertEqual( + latest_patch_version("1.1", "windows", "x86_64"), + "1.1.1") + self.assertEqual( + latest_patch_version("1.1.0", "windows", "x86_64"), + "1.1.1") + self.assertEqual( + latest_patch_version("1.4.0-rc1", "windows", "x86_64"), + "1.4.0-rc1") + self.assertEqual( + latest_patch_version("999.999.999", "windows", "x86_64"), + "999.999.999") diff --git a/jill/utils/__init__.py b/jill/utils/__init__.py new file mode 100644 index 0000000..f0e0ea1 --- /dev/null +++ b/jill/utils/__init__.py @@ -0,0 +1,45 @@ +from .filters import generate_info +from .filters import is_valid_release +from .gpg_utils import verify_gpg +from .interactive_utils import query_yes_no +from .mount_utils import TarMounter, DmgMounter +from .sys_utils import current_architecture, current_system +from .version_utils import Version +from .version_utils import latest_version +from .version_utils import is_version_released +from .version_utils import is_full_version +from .version_utils import update_releases +from .version_utils import read_releases +from .source_utils import SourceRegistry +from .source_utils import show_upstream + +__all__ = [ + # filters + "generate_info", + "is_valid_release", + + # gpg_utils + "verify_gpg", + + # interactive_utils + "query_yes_no", + + # mount_utils + "TarMounter", "DmgMounter", + + # sys_utils + "current_architecture", + "current_system", + + # version_utils + "Version", + "is_full_version", + "latest_version", + "update_releases", + "is_version_released", + "read_releases", + + # source_utils + "SourceRegistry", + "show_upstream" +] diff --git a/jill/defaults.py b/jill/utils/defaults.py similarity index 86% rename from jill/defaults.py rename to jill/utils/defaults.py index 45440b1..0ae13c6 100644 --- a/jill/defaults.py +++ b/jill/utils/defaults.py @@ -1,7 +1,10 @@ from .sys_utils import current_system import os -PKG_ROOT = os.path.abspath(os.path.dirname(__file__)) +# this file isn't really a python script +# just some configuration constants +PKG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), + "..")) def get_configfiles(filename): diff --git a/jill/filters.py b/jill/utils/filters.py similarity index 96% rename from jill/filters.py rename to jill/utils/filters.py index 767ee7e..ea6f5f6 100644 --- a/jill/filters.py +++ b/jill/utils/filters.py @@ -1,3 +1,6 @@ +""" +Module `filters` defines placeholders and how names are filtered. +""" from .defaults import default_filename_template from .defaults import default_latest_filename_template @@ -6,8 +9,6 @@ from typing import Mapping, Optional, Callable -__all__ = ["generate_info"] - VERSION_REGEX = re.compile( r'v?(?P\d+)\.(?P\d+)\.(?P\d+)(-(?P\w+))?') SPECIAL_VERSION_NAMES = ["latest", "nightly", "stable"] @@ -123,7 +124,11 @@ def __init__(self, self.validate = validate def __call__(self, *args, **kwargs): - assert self.validate(*args, **kwargs) + if not self.validate(*args, **kwargs): + # TODO: add error handler + msg = f"validation fails:\n" + msg += f" - args: {args}\n - kwargs: {kwargs}" + raise ValueError(msg) # directly return rst if there're no special filter rules rst = self.f(*args, **kwargs) return self.rules.get(rst, rst) diff --git a/jill/gpg_utils.py b/jill/utils/gpg_utils.py similarity index 93% rename from jill/gpg_utils.py rename to jill/utils/gpg_utils.py index f1f0a33..e1c1a21 100644 --- a/jill/gpg_utils.py +++ b/jill/utils/gpg_utils.py @@ -7,6 +7,7 @@ def _verify_gpg(gpg: GPG, datafile, signature_file): + # this requires gnupg installed in the system with open(signature_file, "rb") as fh: return gpg.verify_file(fh, data_filename=datafile) diff --git a/jill/interactive_utils.py b/jill/utils/interactive_utils.py similarity index 100% rename from jill/interactive_utils.py rename to jill/utils/interactive_utils.py diff --git a/jill/mount_utils.py b/jill/utils/mount_utils.py similarity index 96% rename from jill/mount_utils.py rename to jill/utils/mount_utils.py index 5360c80..0971c72 100644 --- a/jill/mount_utils.py +++ b/jill/utils/mount_utils.py @@ -1,8 +1,9 @@ +from tempfile import mkdtemp import os import sys import subprocess import shutil -from tempfile import mkdtemp +import time class Mounter: @@ -16,7 +17,7 @@ def __init__(self, src_path, mount_root="."): class TarMounter(Mounter): def __enter__(self): self.tempdir = mkdtemp() - # TODO: support .tar + # this only supports compressed tarball: *.tar.gz and *.tgz args = ["tar", "-zxf", self.src_path, "-C", self.tempdir] extra_args = ["--strip-components", "1"] diff --git a/jill/net_utils.py b/jill/utils/net_utils.py similarity index 100% rename from jill/net_utils.py rename to jill/utils/net_utils.py index bea045b..f2ebd27 100644 --- a/jill/net_utils.py +++ b/jill/utils/net_utils.py @@ -1,9 +1,9 @@ from urllib.parse import urlparse +from ipaddress import ip_address + import socket import requests import time -from ipaddress import ip_address - import logging from requests.exceptions import RequestException diff --git a/jill/source.py b/jill/utils/source_utils.py similarity index 96% rename from jill/source.py rename to jill/utils/source_utils.py index 1d75aec..69cc854 100644 --- a/jill/source.py +++ b/jill/utils/source_utils.py @@ -1,7 +1,11 @@ +""" +This module provides tools to read information about upstream mirror, e.g., +if it has a specific julia download. +""" from .defaults import default_scheme_ports from .defaults import SOURCE_CONFIGFILE - -from .net_utils import query_ip, port_response_time +from .net_utils import query_ip +from .net_utils import port_response_time from .net_utils import is_url_available from .filters import generate_info @@ -12,9 +16,8 @@ import requests import json import os -import logging -from typing import Optional, List +from typing import List from requests.exceptions import RequestException diff --git a/jill/sys_utils.py b/jill/utils/sys_utils.py similarity index 85% rename from jill/sys_utils.py rename to jill/utils/sys_utils.py index de3267f..78f5cec 100644 --- a/jill/sys_utils.py +++ b/jill/utils/sys_utils.py @@ -1,3 +1,7 @@ +""" +tools to detect current system and architecture so that things work with tools +in the `filters` module +""" import platform diff --git a/jill/version_utils.py b/jill/utils/version_utils.py similarity index 66% rename from jill/version_utils.py rename to jill/utils/version_utils.py index 59b171c..446c52e 100644 --- a/jill/version_utils.py +++ b/jill/utils/version_utils.py @@ -1,11 +1,13 @@ -from .source import SourceRegistry from .filters import is_valid_release -from .filters import SPECIAL_VERSION_NAMES from .filters import is_system, is_architecture from .filters import VALID_SYSTEM, VALID_ARCHITECTURE +from .filters import f_major_version, f_minor_version, f_patch_version from .defaults import RELEASE_CONFIGFILE from .sys_utils import current_system, current_architecture -from semantic_version import Version + +from .source_utils import SourceRegistry + +import semantic_version from itertools import product import csv @@ -15,7 +17,61 @@ from typing import Tuple, List +def is_full_version(version: str): + if version == "latest": + return True + try: + semantic_version.Version(version) + except ValueError: + return False + return True + + +class Version(semantic_version.Version): + """ + a thin wrapper on semantic_version.Version that + + * accepts "latest" + * accepts partial version, e.g., `1`, `1.0` + """ + + def __init__(self, version_string): + if version_string == "latest": + version_string = "999.999.999" + self.major_version = "latest" + self.minor_version = "latest" + self.patch_version = "latest" + else: + version_string = Version.get_version(version_string) + self.major_version = f_major_version(version_string) + self.minor_version = f_minor_version(version_string) + self.patch_version = f_patch_version(version_string) + super(Version, self).__init__(version_string) + + # TODO: we can actually wrap latest_version here + @staticmethod + def get_version(version: str): + """ + add trailing 0s for incomplete version string + """ + splited = str(version).split(".") + if len(splited) == 1: + major, minor, patch = splited[0], "0", "0" + elif len(splited) == 2: + major, minor = splited[0:2] + patch = "0" + elif len(splited) == 3: + major, minor, patch = splited[0:3] + else: + raise ValueError(f"Unrecognized version string {version}") + return ".".join([major, minor, patch]) + + def read_releases(stable_only=False) -> List[Tuple]: + """ + read release info from local storage + """ + # TODO: read from versions.json (#16) cfg_file = RELEASE_CONFIGFILE if not os.path.isfile(cfg_file): return [] @@ -24,15 +80,11 @@ def read_releases(stable_only=False) -> List[Tuple]: for row in csv.reader(csvfile): releases.append(tuple(row)) if stable_only: - releases = list(filter(lambda x: x[0] not in SPECIAL_VERSION_NAMES, + releases = list(filter(lambda x: x[0] != "latest", releases)) return releases -def is_full_version(version): - return len(str(version).split("-")[0].split(".")) >= 3 - - def is_version_released(version, system, architecture, update=False, upstream=None, @@ -52,6 +104,7 @@ def is_version_released(version, system, architecture, rst = False if update: # query process is time-consuming + print(upstream) registry = SourceRegistry(upstream=upstream) rst = bool(registry.query_download_url(*item, timeout=timeout, @@ -68,28 +121,9 @@ def is_version_released(version, system, architecture, return rst -def get_version(version: str): - """ - add tailing 0s for incomplete version string - """ - if version in ["latest"]: - return Version("999.999.999") - splited = str(version).split(".") - if len(splited) == 1: - major, minor, patch = splited[0], "0", "0" - elif len(splited) == 2: - major, minor = splited[0:2] - patch = "0" - elif len(splited) == 3: - major, minor, patch = splited[0:3] - else: - raise ValueError(f"Unrecognized version string {version}") - return Version(".".join([major, minor, patch])) - - def _latest_version(next_version, version, system, architecture, **kwargs) -> str: - last_version = get_version(version) + last_version = Version(version) if not is_version_released(last_version, system, architecture, **kwargs): return str(last_version) @@ -105,13 +139,17 @@ def _latest_version(next_version, version, system, architecture, def latest_patch_version(version, system, architecture, **kwargs) -> str: """ - return the latest X.Y.z version starting from input version X.Y.Z + return the latest X.Y.z version starting from input version X.Y """ - if not kwargs.get("update", False): + if version == "latest": + return version + # TODO: this is only useful for ARM, remove it (#16) + if (architecture in ["ARMv7", "ARMv8"] and + not kwargs.get("update", False)): # just query from the sorted database versions = [item for item in read_releases() if (item[2] == architecture and - get_version(version).next_minor() > get_version(item[0]))] # nopep8 + Version(version).next_minor() > Version(item[0]))] # nopep8 return versions[-1][0] return _latest_version(Version.next_patch, @@ -121,13 +159,20 @@ def latest_patch_version(version, system, architecture, **kwargs) -> str: def latest_minor_version(version, system, architecture, **kwargs) -> str: """ - return the latest X.y.z version starting from input version X.Y.Z + return the latest X.y.z version starting from input version X """ - if not kwargs.get("update", False): + # if user passes a complete version here, then we don't need to query + # from local storage, just trying to download it would be fine. + if is_full_version(version): + return version + + # TODO: this is only useful for ARM, remove it (#16) + if (architecture in ["ARMv7", "ARMv8"] and + not kwargs.get("update", False)): # just query from the sorted database versions = [item for item in read_releases() if (item[2] == architecture and - get_version(version).next_major() > get_version(item[0]))] # nopep8 + Version(version).next_major() > Version(item[0]))] # nopep8 return versions[-1][0] latest_minor = _latest_version(Version.next_minor, @@ -140,13 +185,20 @@ def latest_minor_version(version, system, architecture, **kwargs) -> str: def latest_major_version(version, system, architecture, **kwargs) -> str: """ - return the latest x.y.z version starting from input version X.Y.Z + return the latest x.y.z version """ - if not kwargs.get("update", False): + # if user passes a complete version here, then we don't need to query + # from local storage, just trying to download it would be fine. + if is_full_version(version): + return version + + # TODO: this is only useful for ARM, remove it (#16) + if (architecture in ["ARMv7", "ARMv8"] and + not kwargs.get("update", False)): # just query from the sorted database versions = [item for item in read_releases() if (item[2] == architecture and - get_version("1.0.0").next_major() > get_version(item[0]))] # nopep8 + Version("1.0.0").next_major() > Version(item[0]))] # nopep8 return versions[-1][0] latest_major = _latest_version(Version.next_major, @@ -166,31 +218,25 @@ def latest_version(version, system, architecture, **kwargs) -> str: find the latest version for partial semantic version string. Directly return `version` if it's already a complete version string. """ - if version in SPECIAL_VERSION_NAMES: - return version + # if user passes a complete version here, then we don't need to query + # from local storage, just trying to download it would be fine. if is_full_version(version): return version - f_list = [latest_minor_version, - latest_patch_version] - f_list.append(lambda ver, sys, arch, **kwargs: ver) # type: ignore - - if len(version) == 0: + if len(version.strip()) == 0: + # if empty string is provided, query the latest version since 1.0.0 return latest_major_version('1', system, architecture, **kwargs) else: + # TODO: we can also support ^ and > semantics here + f_list = [latest_minor_version, + latest_patch_version] idx = len(version.split('.')) - 1 return f_list[idx](version, system, architecture, **kwargs) -def _make_version(ver: str) -> Version: - if ver == "latest": - ver = "999.999.999" - return Version(ver) - - def sort_releases(): releases = read_releases() - releases.sort(key=lambda x: (x[1], x[2], _make_version(x[0]))) + releases.sort(key=lambda x: (x[1], x[2], Version(x[0]))) with open(RELEASE_CONFIGFILE, 'w') as csvfile: for item in releases: writer = csv.writer(csvfile)