clone: Fetch the upstream remote specified in debian/upstream/metadata

This avoids a step if the workflow involves using `--upstream-vcs-tag`.

Overridable with the --upstream-remote configuration option. Set this to
the empty string to disable cloning.

Closes: #888313

gbp/config.py

@@ -187,6 +187,7 @@ class GbpOptionParser(OptionParser):
                 'track': 'True',
                 'track-missing': 'False',
                 'upstream-branch': 'upstream',
+                'upstream-remote': 'upstream',
                 'upstream-tag': 'upstream/%(version)s',
                 'upstream-tree': 'TAG',
                 'upstream-vcs-tag': '',
@@ -201,6 +202,11 @@ class GbpOptionParser(OptionParser):
         'upstream-tree':
             "Where to generate the upstream tarball from "
             "(tag or branch), default is '%(upstream-tree)s'",
+        'upstream-remote':
+            "If debian/upstream/metadata specifies an upstream git repository, "
+            "the remote name to fetch it into, default is "
+            "'%(upstream-remote)s'. Set to an empty string to disable cloning "
+            "upstream",
         'pq-from':
             "How to find the patch queue base. DEBIAN or TAG, "
             "the default is '%(pq-from)s'",

gbp/scripts/clone.py

@@ -117,6 +117,7 @@ def build_parser(name):
 
     branch_group.add_option("--all", action="store_true", dest="all", default=False,
                             help="track all branches, not only debian and upstream")
+    branch_group.add_config_file_option(option_name="upstream-remote", dest="upstream_remote")
     branch_group.add_config_file_option(option_name="upstream-branch", dest="upstream_branch")
     branch_group.add_config_file_option(option_name="debian-branch", dest="debian_branch")
     branch_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
@@ -208,6 +209,31 @@ def main(argv):
 
         repo_setup.set_user_name_and_email(options.repo_user, options.repo_email, repo)
 
+        if options.upstream_remote:
+            try:
+                with open(os.path.join(clone_to, 'debian', 'upstream', 'metadata')) as f:
+                    import yaml
+
+                    y = yaml.load(f, Loader=yaml.CSafeLoader)
+                    try:
+                        upstream_repo = y['Repository']
+
+                        try:
+                            # check if it exists as a git repo
+                            repo.fetch(repo=upstream_repo)
+
+                            repo.add_remote_repo(options.upstream_remote,
+                                                 upstream_repo,
+                                                 tags=True)
+                        except GitRepositoryError:
+                            gbp.log.warn(
+                                'Unable to fetch upstream repository %s, '
+                                'ignoring.' % upstream_repo)
+                    except KeyError:
+                        pass
+            except FileNotFoundError:
+                pass
+
         if postclone:
             Hook('Postclone', options.postclone,
                  extra_env={'GBP_GIT_DIR': repo.git_dir},

tests/component/__init__.py

@@ -178,7 +178,7 @@ def check_tags(cls, repo, tags):
 
     @classmethod
     def _check_repo_state(cls, repo, current_branch, branches, files=None,
-                          dirs=None, tags=None, clean=True):
+                          dirs=None, tags=None, clean=True, remotes=[]):
         """
         Check that repository is clean and given branches, tags, files
         and dirs exist
@@ -191,6 +191,12 @@ def _check_repo_state(cls, repo, current_branch, branches, files=None,
                                                           local_branches)
         eq_(set(local_branches), set(branches), assert_msg)
 
+        if remotes:
+            repo_remotes = repo.get_remotes().keys()
+            assert_msg = "Remotes: expected %s, found %s" % (remotes,
+                                                             repo_remotes)
+            eq_(set(repo_remotes), set(remotes), assert_msg)
+
         if files is not None or dirs is not None:
             # Get files of the working copy recursively
             local_f = set()

tests/component/deb/test_clone.py

@@ -82,3 +82,15 @@ def test_clone_github(self):
         self.assertEquals(ret, 0)
         cloned = ComponentTestGitRepository(dest)
         self._check_repo_state(cloned, 'master', ['master'])
+
+    @skipUnless(os.getenv("GBP_NETWORK_TESTS"), "network tests disabled")
+    def test_clone_upstream_remote(self):
+        """Test that cloning from github urls works"""
+        dest = os.path.join(self._tmpdir,
+                            'cloned_repo')
+        ret = clone(['arg0', "vcsgit:tepl", dest])
+        self.assertEquals(ret, 0)
+        cloned = ComponentTestGitRepository(dest)
+        self._check_repo_state(cloned, 'debian/master',
+                               ['debian/master', 'upstream/latest', 'pristine-tar'],
+                               remotes=['origin', 'upstream'])