Merge pull request #1046 from msarahan/1.21_test_fixes

Revert MSVC to use vcvarsall; improve detection of needed VCS install

.travis.yml

@@ -27,7 +27,7 @@ install:
 
 script:
   - flake8 .
-  - py.test --cov conda_build --cov-report xml tests
+  - py.test -v --cov conda_build --cov-report xml tests
   - conda build --help
 
 notifications:

appveyor.yml

@@ -78,7 +78,7 @@ build: false
 test_script:
   - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%"
   - set PATH
-  - py.test --cov conda_build --cov-report xml tests
+  - py.test -v --cov conda_build --cov-report xml tests
 
 on_success:
   - pip install codecov

conda_build/build.py

@@ -29,7 +29,7 @@
 from conda.resolve import Resolve, MatchSpec, NoPackagesFound
 
 from conda_build import __version__
-from conda_build import environ, source, tarcheck
+from conda_build import environ, source, tarcheck, external
 from conda_build.config import config
 from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path
 from conda_build.scripts import create_entry_points, prepend_bin_path
@@ -472,12 +472,23 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
                 #    have the appropriate VCS available in the environment.  People
                 #    are not used to explicitly listing it in recipes, though.
                 #    We add it for them here, but warn them about it.
-                vcs_source = m.uses_vcs()
+                vcs_source = m.uses_vcs_in_build()
                 if vcs_source and vcs_source not in specs:
-                    specs.append(vcs_source)
-                    log.warn("Your recipe depends on {} at build time (for templates), "
-                            "but you have not listed it as a build dependency.  Doing so for"
-                            " this build.")
+                    vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
+                    has_vcs_available = os.path.isfile(external.find_executable(vcs_executable))
+                    if not has_vcs_available:
+                        if (vcs_source != "mercurial" or
+                                not any(spec.startswith('python') and "3." in spec
+                                        for spec in specs)):
+                            specs.append(vcs_source)
+
+                            log.warn("Your recipe depends on {} at build time (for templates), "
+                                    "but you have not listed it as a build dependency.  Doing "
+                                    "so for this build.")
+                        else:
+                            raise ValueError("Your recipe uses mercurial in build, but mercurial"
+                                            " does not yet support Python 3.  Please handle all of "
+                                            "your mercurial actions outside of your build script.")
             # Display the name only
             # Version number could be missing due to dependency on source info.
             create_env(config.build_prefix, specs)

conda_build/main_build.py

@@ -345,6 +345,10 @@ def execute(args, parser):
                     pkg = line.lstrip('  - ').split(' -> ')[-1]
                     pkg = pkg.strip().split(' ')[0]
                     if pkg in skip_names:
+                        sys.stderr.write("Warning: package conflict - you may have unresolved "
+                                         "dependencies. Try to conda install each of your "
+                                         "dependencies to figure out which has unresolved "
+                                         "dependencies.")
                         continue
                     recipe_glob = glob(pkg + '-[v0-9][0-9.]*')
                     if os.path.exists(pkg):

conda_build/metadata.py

@@ -29,6 +29,8 @@
 from conda_build.config import config
 from conda_build.utils import comma_join
 
+on_win = (sys.platform == 'win32')
+
 
 def ns_cfg():
     # Remember to update the docs of any of this changes
@@ -695,7 +697,7 @@ def __repr__(self):
         '''
         return self.__str__()
 
-    def uses_vcs(self):
+    def uses_vcs_in_meta(self):
         """returns true if recipe contains metadata associated with version control systems.
         If this metadata is present, a download/copy will be forced in parse_or_try_download.
         """
@@ -719,3 +721,18 @@ def uses_vcs(self):
                         vcs = "mercurial"
                     return vcs
         return None
+
+    def uses_vcs_in_build(self):
+        build_script = "bld.bat" if on_win else "build.sh"
+        build_script = os.path.join(os.path.dirname(self.meta_path), build_script)
+        if os.path.isfile(build_script):
+            vcs_types = ["git", "svn", "hg"]
+            with open(self.meta_path) as f:
+                build_script = f.read()
+                for vcs in vcs_types:
+                    matches = re.findall(r"{}(?:\.exe)?".format(vcs), build_script)
+                    if len(matches) > 0:
+                        if vcs == "hg":
+                            vcs = "mercurial"
+                        return vcs
+        return None

conda_build/render.py

@@ -13,7 +13,6 @@
 import tempfile
 import os
 from os.path import isdir, isfile, abspath
-import re
 import subprocess
 
 import yaml
@@ -78,7 +77,7 @@ def bldpkg_path(m):
 def parse_or_try_download(metadata, no_download_source, verbose,
                           force_download=False, dirty=False):
 
-    if (force_download or (not no_download_source and metadata.uses_vcs())):
+    if (force_download or (not no_download_source and metadata.uses_vcs_in_meta())):
         # this try/catch is for when the tool to download source is actually in
         #    meta.yaml, and not previously installed in builder env.
         try:
@@ -93,6 +92,8 @@ def parse_or_try_download(metadata, no_download_source, verbose,
             print("Error was: ")
             print(error)
             need_source_download = True
+    elif not metadata.get_section('source'):
+        need_source_download = False
     else:
         # we have not downloaded source in the render phase.  Download it in
         #     the build phase

conda_build/windows.py

@@ -1,15 +1,14 @@
 from __future__ import absolute_import, division, print_function
 
 import os
-import re
 import sys
 import shutil
-import subprocess
 from os.path import dirname, isdir, isfile, join
 
 # Leverage the hard work done by setuptools/distutils to find vcvarsall using
 # either the registry or the VS**COMNTOOLS environment variable
-from distutils.msvc9compiler import query_vcvarsall
+from distutils.msvc9compiler import find_vcvarsall as distutils_find_vcvarsall
+from distutils.msvc9compiler import Reg, WINSDK_BASE
 
 import conda.config as cc
 
@@ -21,6 +20,20 @@
 
 assert sys.platform == 'win32'
 
+# Set up a load of paths that can be imported from the tests
+if 'ProgramFiles(x86)' in os.environ:
+    PROGRAM_FILES_PATH = os.environ['ProgramFiles(x86)']
+else:
+    PROGRAM_FILES_PATH = os.environ['ProgramFiles']
+
+WIN_SDK_71_PATH = Reg.get_value(os.path.join(WINSDK_BASE, 'v7.1'),
+                                'installationfolder')
+WIN_SDK_71_BAT_PATH = os.path.join(WIN_SDK_71_PATH, 'Bin', 'SetEnv.cmd')
+# Get the Visual Studio 2008 path (not the Visual C++ for Python path)
+# and get the 'vcvars64.bat' from inside the bin (in the directory above
+# that returned by distutils_find_vcvarsall)
+VCVARS64_VS9_BAT_PATH = os.path.join(os.path.dirname(distutils_find_vcvarsall(9)),
+                                     'bin', 'vcvars64.bat')
 VS_VERSION_STRING = {
     '8.0': 'Visual Studio 8 2005',
     '9.0': 'Visual Studio 9 2008',
@@ -62,23 +75,39 @@ def fix_staged_scripts():
         os.remove(join(scripts_dir, fn))
 
 
+def build_vcvarsall_vs_path(version):
+    """
+    Given the Visual Studio version, returns the default path to the
+    Microsoft Visual Studio vcvarsall.bat file.
+    Expected versions are of the form {9, 10, 12, 14}
+    """
+    vstools = "VS{0}0COMNTOOLS".format(version)
+    if vstools in os.environ:
+        return os.path.join(os.environ[vstools], '..\\..\\VC\\vcvarsall.bat')
+    else:
+        # prefer looking at env var; fall back to program files defaults
+        return os.path.join(PROGRAM_FILES_PATH,
+                            'Microsoft Visual Studio {}'.format(version), 'VC',
+                            'vcvarsall.bat')
+
+
 def msvc_env_cmd(bits, override=None):
     arch_selector = 'x86' if bits == 32 else 'amd64'
 
-    compiler_vars = {}
+    msvc_env_lines = []
 
     version = None
-    if override:
+    if override is not None:
         version = override
         # The DISTUTILS_USE_SDK variable tells distutils to not try and validate
         # the MSVC compiler. For < 3.5 this still forcibly looks for 'cl.exe'.
         # For > 3.5 it literally just skips the validation logic.
         # See distutils _msvccompiler.py and msvc9compiler.py / msvccompiler.py
         # for more information.
-        compiler_vars.update({"DISTUTILS_USE_SDK": 1,
-                              # This is also required to hit the 'don't validate' logic on < 3.5.
-                              # For > 3.5 this is ignored.
-                              "MSSdk": 1})
+        msvc_env_lines.append('set DISTUTILS_USE_SDK=1')
+        # This is also required to hit the 'don't validate' logic on < 3.5.
+        # For > 3.5 this is ignored.
+        msvc_env_lines.append('set MSSdk=1')
 
     if not version:
         if config.PY3K and config.use_MSVC2015:
@@ -88,20 +117,60 @@ def msvc_env_cmd(bits, override=None):
         else:
             version = '9.0'
 
-    compiler_vars.update({
-        "VS_VERSION": version,
-        "VS_MAJOR": version.split('.')[0],
-        "VS_YEAR": VS_VERSION_STRING[version][-4:],
-        "CMAKE_GENERATOR": VS_VERSION_STRING[version] + {64: ' Win64', 32: ''}[bits],
-        # tell msys2 to ignore path conversions for issue-causing windows-style flags in build
-        #   See https://github.com/conda-forge/icu-feedstock/pull/5
-        "MSYS2_ARG_CONV_EXCL": "/AI;/AL;/OUT;/out;%MSYS2_ARG_CONV_EXCL%",
-        "MSYS2_ENV_CONV_EXCL": "CL;%MSYS2_ENV_CONV_EXCL%",
-        })
-
-    captured_vars = query_vcvarsall(float(version), arch_selector)
-    compiler_vars.update(captured_vars)
-    return compiler_vars
+    vcvarsall_vs_path = build_vcvarsall_vs_path(version)
+
+    def build_vcvarsall_cmd(cmd, arch=arch_selector):
+        # Default argument `arch_selector` is defined above
+        return 'call "{cmd}" {arch}'.format(cmd=cmd, arch=arch)
+
+    msvc_env_lines.append('set "VS_VERSION={}"'.format(version))
+    msvc_env_lines.append('set "VS_MAJOR={}"'.format(version.split('.')[0]))
+    msvc_env_lines.append('set "VS_YEAR={}"'.format(VS_VERSION_STRING[version][-4:]))
+    msvc_env_lines.append('set "CMAKE_GENERATOR={}"'.format(VS_VERSION_STRING[version] +
+                                                            {64: ' Win64', 32: ''}[bits]))
+    # tell msys2 to ignore path conversions for issue-causing windows-style flags in build
+    #   See https://github.com/conda-forge/icu-feedstock/pull/5
+    msvc_env_lines.append('set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out;%MSYS2_ARG_CONV_EXCL%"')
+    msvc_env_lines.append('set "MSYS2_ENV_CONV_EXCL=CL"')
+    if version == '10.0':
+        win_sdk_arch = '/Release /x86' if bits == 32 else '/Release /x64'
+        win_sdk_cmd = build_vcvarsall_cmd(WIN_SDK_71_BAT_PATH, arch=win_sdk_arch)
+
+        # There are two methods of building Python 3.3 and 3.4 extensions (both
+        # of which required Visual Studio 2010 - as explained in the Python wiki
+        # https://wiki.python.org/moin/WindowsCompilers)
+        # 1) Use the Windows SDK 7.1
+        # 2) Use Visual Studio 2010 (any edition)
+        # However, VS2010 never shipped with a 64-bit compiler, so in this case
+        # **only** option (1) applies. For this reason, we always try and
+        # activate the Windows SDK first. Unfortunately, unsuccessfully setting
+        # up the environment does **not EXIT 1** and therefore we must fall
+        # back to attempting to set up VS2010.
+        # DelayedExpansion is required for the SetEnv.cmd
+        msvc_env_lines.append('Setlocal EnableDelayedExpansion')
+        msvc_env_lines.append(win_sdk_cmd)
+        # If the WindowsSDKDir environment variable has not been successfully
+        # set then try activating VS2010
+        msvc_env_lines.append('if not "%WindowsSDKDir%" == "{}" ( {} )'.format(
+            WIN_SDK_71_PATH, build_vcvarsall_cmd(vcvarsall_vs_path)))
+    elif version == '9.0':
+        error1 = 'if errorlevel 1 {}'
+
+        # Setuptools captures the logic of preferring the Microsoft Visual C++
+        # Compiler for Python 2.7 - falls back to VS2008 if necessary
+        msvc_env_lines.append(build_vcvarsall_cmd(vcvarsall_vs_path))
+        # The Visual Studio 2008 Express edition does not properly contain
+        # the amd64 build files, so we call the vcvars64.bat manually,
+        # rather than using the vcvarsall.bat which would try and call the
+        # missing bat file.
+        if arch_selector == 'amd64':
+            msvc_env_lines.append(error1.format(
+                build_vcvarsall_cmd(VCVARS64_VS9_BAT_PATH)))
+    else:
+        # Visual Studio 14 or otherwise
+        msvc_env_lines.append(build_vcvarsall_cmd(vcvarsall_vs_path))
+
+    return '\n'.join(msvc_env_lines) + '\n'
 
 
 def kill_processes(process_names=["msbuild.exe"]):
@@ -126,18 +195,6 @@ def kill_processes(process_names=["msbuild.exe"]):
         except:
             continue
 
-def _merge_dicts(d1, d2):
-    """Merges d2's contents into d1.  Unlike update, this keeps all entries of both, by performing
-    unions of values."""
-    for key, value in d1.items():
-        if key in d2:
-            combined = set(value.split(';'))
-            combined.update(set(d2[key].split(';')))
-            d1[key] = ";".join(combined)
-            # delete it.  We'll merge remaining vars at the end.
-            del d2[key]
-    d1.update(d2)
-    return d1
 
 def build(m, bld_bat, dirty=False, activate=True):
     env = environ.get_dict(m, dirty=dirty)
@@ -154,25 +211,17 @@ def build(m, bld_bat, dirty=False, activate=True):
         with open(join(src_dir, 'bld.bat'), 'w') as fo:
             # more debuggable with echo on
             fo.write('@echo on\n')
-
-            compiler_vars = msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None))
-            # ensure that all values are uppercase, for sake of merge.
-            env = {key.upper(): value for key, value in env.items()}
-            compiler_vars = {key.upper(): value for key, value in compiler_vars.items()}
-
-            # this is a union of all values from env and from compiler vars.  env should take priority.
-            env = _merge_dicts(env, compiler_vars)
-
             for key, value in env.items():
                 fo.write('set "{key}={value}"\n'.format(key=key, value=value))
             fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
             fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
-            fo.write('\n')
+            fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)))
+            if activate:
+                fo.write("call activate _build\n")
             fo.write("REM ===== end generated header =====\n")
             fo.write(data)
 
-
-        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
+        cmd = [os.environ['COMSPEC'], '/c', 'bld.bat']
         _check_call(cmd, cwd=src_dir)
         kill_processes()
         fix_staged_scripts()

tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat

@@ -1,5 +1,8 @@
 mkdir %PREFIX%\etc\conda\activate.d
+:: output something so it's more obvious when scripts are running
+echo "echo setting TEST_VAR" > %PREFIX%\etc\conda\activate.d\test.bat
 echo set TEST_VAR=1 > %PREFIX%\etc\conda\activate.d\test.bat
 
 mkdir %PREFIX%\etc\conda\deactivate.d
+echo "echo setting TEST_VAR" > %PREFIX%\etc\conda\deactivate.d\test.bat
 echo set TEST_VAR= > %PREFIX%\etc\conda\deactivate.d\test.bat

tests/test-recipes/metadata/_dirty_skip_section/bld.bat

@@ -1,3 +1,4 @@
 :: ensure that the DIRTY environment variable is available for logic in build scripts
+echo DIRTY environment variable should be "1".  Is currently: "%DIRTY%"
 IF "%DIRTY%" == "1" exit 0
 exit 1

tests/test-recipes/metadata/has_prefix_files/run_test.py

@@ -20,13 +20,6 @@ def main():
     print(data)
     assert prefix in data
 
-    with open(join(prefix, 'binary-has-prefix'), 'rb') as f:
-        data = f.read()
-
-    print('binary-has-prefix')
-    print(data)
-    assert prefix.encode('utf-8') in data
-
     if sys.platform == 'win32':
         forward_slash_prefix = prefix.replace('\\', '/')
         with open(join(prefix, 'forward-slash-prefix')) as f:
@@ -35,6 +28,12 @@ def main():
         print('forward-slash-prefix')
         print(data)
         assert forward_slash_prefix in data
+    else:
+        with open(join(prefix, 'binary-has-prefix'), 'rb') as f:
+            data = f.read()
+        print('binary-has-prefix')
+        print(data)
+        assert prefix.encode('utf-8') in data
 
 if __name__ == '__main__':
     main()

tests/test-recipes/metadata/set_env_var_activate_build/bld.bat

@@ -1 +1,2 @@
+echo TEST_VAR is "%TEST_VAR%" (should be "1")
 if "%TEST_VAR%" == "" exit 1