From b14d6142b7ff43adfd50550d18a42db3b18192e0 Mon Sep 17 00:00:00 2001 From: aln8 Date: Mon, 2 Sep 2024 11:32:27 +0800 Subject: [PATCH] feat: add --download-timeout parser options --- src/rosdep2/gbpdistro_support.py | 9 +++---- src/rosdep2/main.py | 8 +++++- src/rosdep2/rep3.py | 7 ++---- src/rosdep2/sources_list.py | 18 +++++++------- test/test_rosdep_gbpdistro_support.py | 5 ++-- test/test_rosdep_main.py | 1 + test/test_rosdep_rep3.py | 7 ++++++ test/test_rosdep_sources_list.py | 35 +++++++++++++++++++++++---- 8 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/rosdep2/gbpdistro_support.py b/src/rosdep2/gbpdistro_support.py index be4936147..8303a2258 100644 --- a/src/rosdep2/gbpdistro_support.py +++ b/src/rosdep2/gbpdistro_support.py @@ -34,9 +34,6 @@ FUERTE_GBPDISTRO_URL = 'https://raw.githubusercontent.com/ros/rosdistro/' \ 'master/releases/fuerte.yaml' -# seconds to wait before aborting download of gbpdistro data -DOWNLOAD_TIMEOUT = 15.0 - def get_owner_name(url): """ @@ -181,7 +178,7 @@ def get_gbprepo_as_rosdep_data(gbpdistro): return rosdep_data -def download_gbpdistro_as_rosdep_data(gbpdistro_url, targets_url=None): +def download_gbpdistro_as_rosdep_data(gbpdistro_url, timeout, targets_url=None): """ Download gbpdistro file from web and convert format to rosdep distro data. @@ -197,9 +194,9 @@ def download_gbpdistro_as_rosdep_data(gbpdistro_url, targets_url=None): # we can convert a gbpdistro file into rosdep data by following a # couple rules # will output a warning - targets_data = download_targets_data(targets_url=targets_url) + targets_data = download_targets_data(targets_url=targets_url, timeout=timeout) try: - f = urlopen_gzip(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen_gzip(gbpdistro_url, timeout=timeout) text = f.read() f.close() gbpdistro_data = yaml.safe_load(text) diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index c059ebd42..5c46f5299 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -382,6 +382,11 @@ def _rosdep_main(args): default=[], action='append', help='Dependency types to install, can be given multiple times. ' 'Choose from {}. Default: all except doc.'.format(VALID_DEPENDENCY_TYPES)) + parser.add_option("--download-timeout", dest="download_timeout", + type=float, default=15.0, + help="Affects the 'update' verb. " + "Timeout in seconds for pull source list data. " + "Default to 15.0 seconds.") options, args = parser.parse_args(args) if options.print_version or options.print_all_versions: @@ -598,7 +603,7 @@ def configure_installer_context(installer_context, options): def command_init(options): try: - data = download_default_sources_list() + data = download_default_sources_list(options.download_timeout) except URLError as e: print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL), file=sys.stderr) print(e, file=sys.stderr) @@ -671,6 +676,7 @@ def update_error_handler(data_source, exc): error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, ros_distro=options.ros_distro, + download_timeout=options.download_timeout, quiet=options.quiet) if not options.quiet: print('updated cache in %s' % (sources_cache_dir)) diff --git a/src/rosdep2/rep3.py b/src/rosdep2/rep3.py index fb6c93e18..7f0bbf07a 100644 --- a/src/rosdep2/rep3.py +++ b/src/rosdep2/rep3.py @@ -35,11 +35,8 @@ # location of targets file for processing gbpdistro files REP3_TARGETS_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/releases/targets.yaml' -# seconds to wait before aborting download of gbpdistro data -DOWNLOAD_TIMEOUT = 15.0 - -def download_targets_data(targets_url=None): +def download_targets_data(targets_url=None, timeout=15.0): """ Download REP 3 targets file and unmarshal from YAML. DEPRECATED: this function is deprecated. List of targets should be obtained @@ -56,7 +53,7 @@ def download_targets_data(targets_url=None): if targets_url is None: targets_url = REP3_TARGETS_URL try: - f = urlopen_gzip(targets_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen_gzip(targets_url, timeout=timeout) text = f.read() f.close() targets_data = yaml.safe_load(text) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index 8c98f2e26..10a6e7830 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -61,9 +61,6 @@ # rosdep DEFAULT_SOURCES_LIST_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/sources.list.d/20-default.list' -# seconds to wait before aborting download of rosdep data -DOWNLOAD_TIMEOUT = 15.0 - SOURCES_LIST_DIR = 'sources.list.d' SOURCES_CACHE_DIR = 'sources.cache' @@ -292,13 +289,13 @@ def create_default(os_override=None): return DataSourceMatcher(tags) -def download_rosdep_data(url): +def download_rosdep_data(url, timeout): """ :raises: :exc:`DownloadFailure` If data cannot be retrieved (e.g. 404, bad YAML format, server down). """ try: - f = urlopen_gzip(url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen_gzip(url, timeout=timeout) text = f.read() f.close() data = yaml.safe_load(text) @@ -311,10 +308,11 @@ def download_rosdep_data(url): raise DownloadFailure(str(e)) -def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): +def download_default_sources_list(timeout, url=DEFAULT_SOURCES_LIST_URL): """ Download (and validate) contents of default sources list. + :param timeout: download timeout in seconds :param url: override URL of default sources list file :return: raw sources list data, ``str`` :raises: :exc:`DownloadFailure` If data cannot be @@ -323,7 +321,7 @@ def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): retrieved (e.g. 404, server down). """ try: - f = urlopen_gzip(url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen_gzip(url, timeout=timeout) except (URLError, httplib.HTTPException) as e: raise URLError(str(e) + ' (%s)' % url) data = f.read().decode() @@ -435,6 +433,7 @@ def _generate_key_from_urls(urls): def update_sources_list(sources_list_dir=None, sources_cache_dir=None, success_handler=None, error_handler=None, skip_eol_distros=False, ros_distro=None, + download_timeout=15.0, quiet=False): """ Re-downloaded data from remote sources and store in cache. Also @@ -449,6 +448,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, if a particular source fails. This hook is mainly for printing errors to console. :param skip_eol_distros: skip downloading sources for EOL distros + :param timeout: download timeout in seconds, default to 15.0 :returns: list of (`DataSource`, cache_file_path) pairs for cache files that were updated, ``[str]`` @@ -464,14 +464,14 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, for source in list(sources): try: if source.type == TYPE_YAML: - rosdep_data = download_rosdep_data(source.url) + rosdep_data = download_rosdep_data(source.url, download_timeout) elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 if not source.tags[0] in ['electric', 'fuerte']: if not quiet: print('Ignore legacy gbpdistro "%s"' % source.tags[0]) sources.remove(source) continue # do not store this entry in the cache - rosdep_data = download_gbpdistro_as_rosdep_data(source.url) + rosdep_data = download_gbpdistro_as_rosdep_data(source.url, download_timeout) retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) if success_handler is not None: success_handler(source) diff --git a/test/test_rosdep_gbpdistro_support.py b/test/test_rosdep_gbpdistro_support.py index c3cd53ee3..1536dd49c 100644 --- a/test/test_rosdep_gbpdistro_support.py +++ b/test/test_rosdep_gbpdistro_support.py @@ -77,7 +77,7 @@ def test_download_gbpdistro_as_rosdep_data(): from rosdep2.gbpdistro_support import FUERTE_GBPDISTRO_URL from rosdep2.rep3 import REP3_TARGETS_URL from rosdep2 import DownloadFailure - data = download_gbpdistro_as_rosdep_data(FUERTE_GBPDISTRO_URL) + data = download_gbpdistro_as_rosdep_data(FUERTE_GBPDISTRO_URL, 15.0) # don't go beyond this, this test is just making sure the download # plumbing is correct, not the loader. for k in ['ros', 'catkin', 'genmsg']: @@ -89,13 +89,14 @@ def test_download_gbpdistro_as_rosdep_data(): # override targets URL with bad URL download_gbpdistro_as_rosdep_data( FUERTE_GBPDISTRO_URL, + 15.0, targets_url='http://bad.ros.org/foo.yaml') assert False, 'should have raised' except DownloadFailure: pass try: # use targets URL, which should have a bad format - download_gbpdistro_as_rosdep_data(REP3_TARGETS_URL) + download_gbpdistro_as_rosdep_data(REP3_TARGETS_URL, 15.0) assert False, 'should have raised' except DownloadFailure: pass diff --git a/test/test_rosdep_main.py b/test/test_rosdep_main.py index 1a99d7c91..bf544df74 100644 --- a/test/test_rosdep_main.py +++ b/test/test_rosdep_main.py @@ -192,6 +192,7 @@ def read_stdout(cmd, capture_stderr=False): 'install', '-s', '-i', '--os', 'ubuntu:lucid', '--rosdistro', 'fuerte', + '--download-timeout', '15.0', '--from-paths', catkin_tree ] + cmd_extras) stdout, stderr = b diff --git a/test/test_rosdep_rep3.py b/test/test_rosdep_rep3.py index 68db9dcf6..f1cb009cd 100644 --- a/test/test_rosdep_rep3.py +++ b/test/test_rosdep_rep3.py @@ -67,3 +67,10 @@ def test_download_targets_data(): assert False, 'should have raised' except DownloadFailure: pass + + # try timeout + try: + download_targets_data(targets_url=REP3_TARGETS_URL, timeout=0.0001) + assert False, 'should have raised' + except DownloadFailure: + pass diff --git a/test/test_rosdep_sources_list.py b/test/test_rosdep_sources_list.py index 633d26214..16bce1bba 100644 --- a/test/test_rosdep_sources_list.py +++ b/test/test_rosdep_sources_list.py @@ -69,10 +69,16 @@ def test_url_constants(): @pytest.mark.online def test_download_default_sources_list(): from rosdep2.sources_list import download_default_sources_list - data = download_default_sources_list() + data = download_default_sources_list(15.0) assert 'http' in data, data # sanity check, all sources files have urls try: - download_default_sources_list(url='http://bad.ros.org/foo.yaml') + download_default_sources_list(15.0, url='http://bad.ros.org/foo.yaml') + assert False, 'should not have succeeded/valdiated' + except URLError: + pass + + try: + download_default_sources_list(0.0001, url='http://bad.ros.org/foo.yaml') assert False, 'should not have succeeded/valdiated' except URLError: pass @@ -248,6 +254,16 @@ def error_handler(loc, e): 'yaml %s ubuntu' % (GITHUB_URL, GITHUB_PYTHON_URL, BADHOSTNAME_URL) assert expected == index, '\n[%s]\nvs\n[%s]' % (expected, index) + # test timeout + errors = [] + + def error_handler(loc, e): + errors.append((loc, e)) + retval = update_sources_list(sources_list_dir=sources_list_dir, + sources_cache_dir=tempdir, error_handler=error_handler, download_timeout=0.0001) + print(retval) + print(errors) + @pytest.mark.online def test_load_cached_sources_list(): @@ -306,22 +322,31 @@ def test_download_rosdep_data(): from rosdep2.sources_list import download_rosdep_data from rosdep2 import DownloadFailure url = GITHUB_BASE_URL - data = download_rosdep_data(url) + data = download_rosdep_data(url, 15.0) assert 'boost' in data # sanity check # try with a bad URL try: - data = download_rosdep_data('http://badhost.willowgarage.com/rosdep.yaml') + data = download_rosdep_data('http://badhost.willowgarage.com/rosdep.yaml', 15.0) assert False, 'should have raised' except DownloadFailure as e: pass + + # test timeout + url = GITHUB_BASE_URL + try: + data = download_rosdep_data(url, 0.0001) + assert False, 'should have raised' + except DownloadFailure as e: + pass + # try to trigger both non-dict clause and YAMLError clause for url in [ 'https://code.ros.org/svn/release/trunk/distros/', 'https://code.ros.org/svn/release/trunk/distros/manifest.xml', ]: try: - data = download_rosdep_data(url) + data = download_rosdep_data(url, 15.0) assert False, 'should have raised' except DownloadFailure as e: pass