urlscm: separate download directory

doc/manual/policies.rst

@@ -626,3 +626,19 @@ New behavior
    passed to the ``fingerprintScript``. Other environment variables are unset
    but whitelisted variables (see :ref:`configuration-config-whitelist`) are
    still available.
+
+scmOnlyExtracted
+~~~~~~~~~~~~~~~~
+
+Introduced in: 0.25
+
+This policy controls if the downloaded file of a UrlScm is stored in the source
+workspace when extractors are available for it.
+
+Old behavior
+    Downloaded original and `.extracted` canary file are part of the source
+    workspace.
+
+New behavior
+    If a extractor is available the downloaded file and the canary are stored in
+    workspace/../_downloaded.

pym/bob/builder.py

@@ -1216,6 +1216,8 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                             os.rename(scmPath, atticPath)
                             BobState().setAtticDirectoryState(atticPath, scmSpec)
                             atticPaths.add(scmPath, atticPath)
+                            downloadPath = os.path.normpath(os.path.join(prettySrcPath, "..", "_download", scmDir))
+                            shutil.rmtree(downloadPath, ignore_errors=True)
                         del oldCheckoutState[scmDir]
                         BobState().setDirectoryState(prettySrcPath, oldCheckoutState)
 

pym/bob/input.py

@@ -2959,6 +2959,7 @@ class RecipeSet:
                 schema.Optional('gitCommitOnBranch') : bool,
                 schema.Optional('fixImportScmVariant') : bool,
                 schema.Optional('defaultFileMode') : bool,
+                schema.Optional('scmOnlyExtracted') : bool,
             },
             error="Invalid policy specified! Are you using an appropriate version of Bob?"
         ),
@@ -3046,6 +3047,11 @@ def __init__(self):
                 InfoOnce("defaultFileMode policy not set. File mode of URL SCMs not set for locally copied files.",
                     help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#defaultfilemode for more information.")
             ),
+            "scmOnlyExtracted": (
+                "0.24.1.dev105",
+                InfoOnce("scmOnlyExtracted policy is not set. Downloaded original is part of the source workspace",
+                    help="See http://bob-build-tool.readthedocs.io/en/latest/manual/policies.html#scmOnlyExtracted for more information.")
+            ),
         }
         self.__buildHooks = {}
         self.__sandboxOpts = {}

pym/bob/intermediate.py

@@ -549,6 +549,7 @@ def fromRecipeSet(cls, recipeSet):
             'gitCommitOnBranch' : recipeSet.getPolicy('gitCommitOnBranch'),
             'fixImportScmVariant' : recipeSet.getPolicy('fixImportScmVariant'),
             'defaultFileMode' : recipeSet.getPolicy('defaultFileMode'),
+            'scmOnlyExtracted' : recipeSet.getPolicy('scmOnlyExtracted'),
         }
         self.__data['archiveSpec'] = recipeSet.archiveSpec()
         self.__data['envWhiteList'] = sorted(recipeSet.envWhiteList())

pym/bob/scm/__init__.py

@@ -62,6 +62,7 @@ def getScm(spec, overrides=[], recipeSet=None):
             recipeSet and recipeSet.getPolicy('scmIgnoreUser'),
             recipeSet.getPreMirrors() if recipeSet else [],
             recipeSet.getFallbackMirrors() if recipeSet else [],
-            recipeSet and recipeSet.getPolicy('defaultFileMode'))
+            recipeSet and recipeSet.getPolicy('defaultFileMode'),
+            recipeSet and recipeSet.getPolicy('scmOnlyExtracted'))
     else:
         raise ParseError("Unknown SCM '{}'".format(scm))

pym/bob/scm/url.py

@@ -223,7 +223,8 @@ class UrlScm(Scm):
     }
 
     def __init__(self, spec, overrides=[], stripUser=None,
-                 preMirrors=[], fallbackMirrors=[], defaultFileMode=None):
+                 preMirrors=[], fallbackMirrors=[], defaultFileMode=None,
+                 onlyExtracted=True):
         super().__init__(spec, overrides)
         self.__url = spec["url"]
         self.__digestSha1 = spec.get("digestSHA1")
@@ -262,6 +263,7 @@ def __init__(self, spec, overrides=[], stripUser=None,
         self.__fallbackMirrorsUrls = spec.get("fallbackMirrors")
         self.__fallbackMirrorsUpload = spec.get("__fallbackMirrorsUpload")
         self.__fileMode = spec.get("fileMode", 0o600 if defaultFileMode else None)
+        self.__onlyExtracted = spec.get("onlyExtracted", onlyExtracted)
 
     def getProperties(self, isJenkins, pretty=False):
         ret = super().getProperties(isJenkins)
@@ -282,6 +284,7 @@ def getProperties(self, isJenkins, pretty=False):
             'fallbackMirrors' : self.__getFallbackMirrorsUrls(),
             '__fallbackMirrorsUpload' : self.__getFallbackMirrorsUpload(),
             'fileMode' : dumpMode(self.__fileMode) if pretty else self.__fileMode,
+            'onlyExtracted' : self.__onlyExtracted,
         })
         return ret
 
@@ -536,9 +539,14 @@ async def switch(self, invoker, oldScm):
         return True
 
     async def invoke(self, invoker):
-        os.makedirs(invoker.joinPath(self.__dir), exist_ok=True)
-        workspaceFile = os.path.join(self.__dir, self.__fn)
-        destination = invoker.joinPath(self.__dir, self.__fn)
+        extractors = self.__getExtractors()
+        downloadDestination = ""
+        if extractors and self.__onlyExtracted:
+            downloadDestination = os.path.join("..", "_download", self.__dir)
+
+        os.makedirs(invoker.joinPath(self.__dir, downloadDestination), exist_ok=True)
+        destination = invoker.joinPath(self.__dir, downloadDestination, self.__fn)
+        workspaceFile = os.path.join(self.__dir, downloadDestination, self.__fn)
 
         # Download only if necessary
         if not self.isDeterministic() or not os.path.isfile(destination):
@@ -587,8 +595,7 @@ async def invoke(self, invoker):
                     await self._put(invoker, workspaceFile, destination, url)
 
         # Run optional extractors
-        extractors = self.__getExtractors()
-        canary = invoker.joinPath(self.__dir, "." + self.__fn + ".extracted")
+        canary = invoker.joinPath(self.__dir, downloadDestination, "." + self.__fn + ".extracted")
         if extractors and isYounger(destination, canary):
             for cmd in extractors:
                 if shutil.which(cmd[0]) is None: continue
@@ -672,7 +679,10 @@ def __getExtractors(self):
                 strip = [extractor[3].format(self.__strip)]
             else:
                 strip = []
-            ret.append([extractor[1]] + [a.format(self.__fn) for a in extractor[2]] + strip)
+            fileToExtract = self.__fn
+            if self.__onlyExtracted:
+                fileToExtract = os.path.join("..", "_download", self.__dir, self.__fn)
+            ret.append([extractor[1]] + [a.format(fileToExtract) for a in extractor[2]] + strip)
 
         if not ret:
             raise BuildError("Extractor does not support 'stripComponents'!")