From ff77d81354f32706a357ab2d351033a9be5a2cac Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 23 Jun 2016 21:22:51 -0500 Subject: [PATCH 1/5] handle empty source better; fix some win tests --- conda_build/render.py | 2 ++ .../metadata/_dirty_skip_section/bld.bat | 1 + .../metadata/has_prefix_files/run_test.py | 13 ++++++------- .../metadata/set_env_var_activate_build/bld.bat | 1 + 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index f048990abe..7c664adbf9 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -93,6 +93,8 @@ def parse_or_try_download(metadata, no_download_source, verbose, print("Error was: ") print(error) need_source_download = True + elif not metadata.get_section('source'): + need_source_download = False else: # we have not downloaded source in the render phase. Download it in # the build phase diff --git a/tests/test-recipes/metadata/_dirty_skip_section/bld.bat b/tests/test-recipes/metadata/_dirty_skip_section/bld.bat index 2100201c67..c870f4bf18 100644 --- a/tests/test-recipes/metadata/_dirty_skip_section/bld.bat +++ b/tests/test-recipes/metadata/_dirty_skip_section/bld.bat @@ -1,3 +1,4 @@ :: ensure that the DIRTY environment variable is available for logic in build scripts +echo DIRTY environment variable should be "1". Is currently: "%DIRTY%" IF "%DIRTY%" == "1" exit 0 exit 1 \ No newline at end of file diff --git a/tests/test-recipes/metadata/has_prefix_files/run_test.py b/tests/test-recipes/metadata/has_prefix_files/run_test.py index fb50fba2a0..3dbda3d572 100644 --- a/tests/test-recipes/metadata/has_prefix_files/run_test.py +++ b/tests/test-recipes/metadata/has_prefix_files/run_test.py @@ -20,13 +20,6 @@ def main(): print(data) assert prefix in data - with open(join(prefix, 'binary-has-prefix'), 'rb') as f: - data = f.read() - - print('binary-has-prefix') - print(data) - assert prefix.encode('utf-8') in data - if sys.platform == 'win32': forward_slash_prefix = prefix.replace('\\', '/') with open(join(prefix, 'forward-slash-prefix')) as f: @@ -35,6 +28,12 @@ def main(): print('forward-slash-prefix') print(data) assert forward_slash_prefix in data + else: + with open(join(prefix, 'binary-has-prefix'), 'rb') as f: + data = f.read() + print('binary-has-prefix') + print(data) + assert prefix.encode('utf-8') in data if __name__ == '__main__': main() diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat b/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat index 358ee87d83..8f6830ed69 100644 --- a/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat +++ b/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat @@ -1 +1,2 @@ +echo TEST_VAR is "%TEST_VAR%" (should be "1") if "%TEST_VAR%" == "" exit 1 \ No newline at end of file From 7c20b8578b88c3271b747b6b9edbfb8b35c6f332 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 23 Jun 2016 08:07:04 -0500 Subject: [PATCH 2/5] revert MSVC disutils changes --- conda_build/render.py | 1 - conda_build/windows.py | 140 ++++++++++++++++++++++++++++------------- 2 files changed, 95 insertions(+), 46 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index 7c664adbf9..1354a57c82 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -13,7 +13,6 @@ import tempfile import os from os.path import isdir, isfile, abspath -import re import subprocess import yaml diff --git a/conda_build/windows.py b/conda_build/windows.py index a992b8a86d..6370f67a1a 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -1,15 +1,14 @@ from __future__ import absolute_import, division, print_function import os -import re import sys import shutil -import subprocess from os.path import dirname, isdir, isfile, join # Leverage the hard work done by setuptools/distutils to find vcvarsall using # either the registry or the VS**COMNTOOLS environment variable -from distutils.msvc9compiler import query_vcvarsall +from distutils.msvc9compiler import find_vcvarsall as distutils_find_vcvarsall +from distutils.msvc9compiler import Reg, WINSDK_BASE import conda.config as cc @@ -21,6 +20,20 @@ assert sys.platform == 'win32' +# Set up a load of paths that can be imported from the tests +if 'ProgramFiles(x86)' in os.environ: + PROGRAM_FILES_PATH = os.environ['ProgramFiles(x86)'] +else: + PROGRAM_FILES_PATH = os.environ['ProgramFiles'] + +WIN_SDK_71_PATH = Reg.get_value(os.path.join(WINSDK_BASE, 'v7.1'), + 'installationfolder') +WIN_SDK_71_BAT_PATH = os.path.join(WIN_SDK_71_PATH, 'Bin', 'SetEnv.cmd') +# Get the Visual Studio 2008 path (not the Visual C++ for Python path) +# and get the 'vcvars64.bat' from inside the bin (in the directory above +# that returned by distutils_find_vcvarsall) +VCVARS64_VS9_BAT_PATH = os.path.join(os.path.dirname(distutils_find_vcvarsall(9)), + 'bin', 'vcvars64.bat') VS_VERSION_STRING = { '8.0': 'Visual Studio 8 2005', '9.0': 'Visual Studio 9 2008', @@ -62,23 +75,39 @@ def fix_staged_scripts(): os.remove(join(scripts_dir, fn)) +def build_vcvarsall_vs_path(version): + """ + Given the Visual Studio version, returns the default path to the + Microsoft Visual Studio vcvarsall.bat file. + Expected versions are of the form {9, 10, 12, 14} + """ + vstools = "VS{0}0COMNTOOLS".format(version) + if vstools in os.environ: + return os.path.join(os.environ[vstools], '..\\..\\VC\\vcvarsall.bat') + else: + # prefer looking at env var; fall back to program files defaults + return os.path.join(PROGRAM_FILES_PATH, + 'Microsoft Visual Studio {}'.format(version), 'VC', + 'vcvarsall.bat') + + def msvc_env_cmd(bits, override=None): arch_selector = 'x86' if bits == 32 else 'amd64' - compiler_vars = {} + msvc_env_lines = [] version = None - if override: + if override is not None: version = override # The DISTUTILS_USE_SDK variable tells distutils to not try and validate # the MSVC compiler. For < 3.5 this still forcibly looks for 'cl.exe'. # For > 3.5 it literally just skips the validation logic. # See distutils _msvccompiler.py and msvc9compiler.py / msvccompiler.py # for more information. - compiler_vars.update({"DISTUTILS_USE_SDK": 1, - # This is also required to hit the 'don't validate' logic on < 3.5. - # For > 3.5 this is ignored. - "MSSdk": 1}) + msvc_env_lines.append('set DISTUTILS_USE_SDK=1') + # This is also required to hit the 'don't validate' logic on < 3.5. + # For > 3.5 this is ignored. + msvc_env_lines.append('set MSSdk=1') if not version: if config.PY3K and config.use_MSVC2015: @@ -88,20 +117,60 @@ def msvc_env_cmd(bits, override=None): else: version = '9.0' - compiler_vars.update({ - "VS_VERSION": version, - "VS_MAJOR": version.split('.')[0], - "VS_YEAR": VS_VERSION_STRING[version][-4:], - "CMAKE_GENERATOR": VS_VERSION_STRING[version] + {64: ' Win64', 32: ''}[bits], - # tell msys2 to ignore path conversions for issue-causing windows-style flags in build - # See https://github.com/conda-forge/icu-feedstock/pull/5 - "MSYS2_ARG_CONV_EXCL": "/AI;/AL;/OUT;/out;%MSYS2_ARG_CONV_EXCL%", - "MSYS2_ENV_CONV_EXCL": "CL;%MSYS2_ENV_CONV_EXCL%", - }) - - captured_vars = query_vcvarsall(float(version), arch_selector) - compiler_vars.update(captured_vars) - return compiler_vars + vcvarsall_vs_path = build_vcvarsall_vs_path(version) + + def build_vcvarsall_cmd(cmd, arch=arch_selector): + # Default argument `arch_selector` is defined above + return 'call "{cmd}" {arch}'.format(cmd=cmd, arch=arch) + + msvc_env_lines.append('set "VS_VERSION={}"'.format(version)) + msvc_env_lines.append('set "VS_MAJOR={}"'.format(version.split('.')[0])) + msvc_env_lines.append('set "VS_YEAR={}"'.format(VS_VERSION_STRING[version][-4:])) + msvc_env_lines.append('set "CMAKE_GENERATOR={}"'.format(VS_VERSION_STRING[version] + + {64: ' Win64', 32: ''}[bits])) + # tell msys2 to ignore path conversions for issue-causing windows-style flags in build + # See https://github.com/conda-forge/icu-feedstock/pull/5 + msvc_env_lines.append('set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out;%MSYS2_ARG_CONV_EXCL%"') + msvc_env_lines.append('set "MSYS2_ENV_CONV_EXCL=CL"') + if version == '10.0': + win_sdk_arch = '/Release /x86' if bits == 32 else '/Release /x64' + win_sdk_cmd = build_vcvarsall_cmd(WIN_SDK_71_BAT_PATH, arch=win_sdk_arch) + + # There are two methods of building Python 3.3 and 3.4 extensions (both + # of which required Visual Studio 2010 - as explained in the Python wiki + # https://wiki.python.org/moin/WindowsCompilers) + # 1) Use the Windows SDK 7.1 + # 2) Use Visual Studio 2010 (any edition) + # However, VS2010 never shipped with a 64-bit compiler, so in this case + # **only** option (1) applies. For this reason, we always try and + # activate the Windows SDK first. Unfortunately, unsuccessfully setting + # up the environment does **not EXIT 1** and therefore we must fall + # back to attempting to set up VS2010. + # DelayedExpansion is required for the SetEnv.cmd + msvc_env_lines.append('Setlocal EnableDelayedExpansion') + msvc_env_lines.append(win_sdk_cmd) + # If the WindowsSDKDir environment variable has not been successfully + # set then try activating VS2010 + msvc_env_lines.append('if not "%WindowsSDKDir%" == "{}" ( {} )'.format( + WIN_SDK_71_PATH, build_vcvarsall_cmd(vcvarsall_vs_path))) + elif version == '9.0': + error1 = 'if errorlevel 1 {}' + + # Setuptools captures the logic of preferring the Microsoft Visual C++ + # Compiler for Python 2.7 - falls back to VS2008 if necessary + msvc_env_lines.append(build_vcvarsall_cmd(vcvarsall_vs_path)) + # The Visual Studio 2008 Express edition does not properly contain + # the amd64 build files, so we call the vcvars64.bat manually, + # rather than using the vcvarsall.bat which would try and call the + # missing bat file. + if arch_selector == 'amd64': + msvc_env_lines.append(error1.format( + build_vcvarsall_cmd(VCVARS64_VS9_BAT_PATH))) + else: + # Visual Studio 14 or otherwise + msvc_env_lines.append(build_vcvarsall_cmd(vcvarsall_vs_path)) + + return '\n'.join(msvc_env_lines) + '\n' def kill_processes(process_names=["msbuild.exe"]): @@ -126,18 +195,6 @@ def kill_processes(process_names=["msbuild.exe"]): except: continue -def _merge_dicts(d1, d2): - """Merges d2's contents into d1. Unlike update, this keeps all entries of both, by performing - unions of values.""" - for key, value in d1.items(): - if key in d2: - combined = set(value.split(';')) - combined.update(set(d2[key].split(';'))) - d1[key] = ";".join(combined) - # delete it. We'll merge remaining vars at the end. - del d2[key] - d1.update(d2) - return d1 def build(m, bld_bat, dirty=False, activate=True): env = environ.get_dict(m, dirty=dirty) @@ -154,24 +211,17 @@ def build(m, bld_bat, dirty=False, activate=True): with open(join(src_dir, 'bld.bat'), 'w') as fo: # more debuggable with echo on fo.write('@echo on\n') - - compiler_vars = msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)) - # ensure that all values are uppercase, for sake of merge. - env = {key.upper(): value for key, value in env.items()} - compiler_vars = {key.upper(): value for key, value in compiler_vars.items()} - - # this is a union of all values from env and from compiler vars. env should take priority. - env = _merge_dicts(env, compiler_vars) - for key, value in env.items(): fo.write('set "{key}={value}"\n'.format(key=key, value=value)) fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"])) fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"])) + fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None))) + if activate: + fo.write("call activate _build\n") fo.write('\n') fo.write("REM ===== end generated header =====\n") fo.write(data) - cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat'] _check_call(cmd, cwd=src_dir) kill_processes() From 62ecffdefa2215a2cf77a44e44f1aa0c98844989 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 23 Jun 2016 21:49:11 -0500 Subject: [PATCH 3/5] add warning message about unsatisfiable packages --- conda_build/main_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 78aecf239f..91cf2bc30b 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -345,6 +345,10 @@ def execute(args, parser): pkg = line.lstrip(' - ').split(' -> ')[-1] pkg = pkg.strip().split(' ')[0] if pkg in skip_names: + sys.stderr.write("Warning: package conflict - you may have unresolved " + "dependencies. Try to conda install each of your " + "dependencies to figure out which has unresolved " + "dependencies.") continue recipe_glob = glob(pkg + '-[v0-9][0-9.]*') if os.path.exists(pkg): From 2d889ab07c6c9898b19b6c4a5a71ed82ee6f6d9f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 23 Jun 2016 22:09:41 -0500 Subject: [PATCH 4/5] add -v to pytest so we see which tests hang --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dfa9bc5791..22428fed2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ install: script: - flake8 . - - py.test --cov conda_build --cov-report xml tests + - py.test -v --cov conda_build --cov-report xml tests - conda build --help notifications: diff --git a/appveyor.yml b/appveyor.yml index 712381ec52..3944e7d92b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -78,7 +78,7 @@ build: false test_script: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - set PATH - - py.test --cov conda_build --cov-report xml tests + - py.test -v --cov conda_build --cov-report xml tests on_success: - pip install codecov From aeda5b8bba57df9ad3111ce8619adc9cf59d3ec4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 24 Jun 2016 13:06:16 -0500 Subject: [PATCH 5/5] add notion of uses_vcs in meta and in build; fix win calling --- conda_build/build.py | 23 ++++++++++++++----- conda_build/metadata.py | 19 ++++++++++++++- conda_build/render.py | 2 +- conda_build/windows.py | 3 +-- .../bld.bat | 3 +++ 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index dbf1bda9dd..2f13770b8f 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -29,7 +29,7 @@ from conda.resolve import Resolve, MatchSpec, NoPackagesFound from conda_build import __version__ -from conda_build import environ, source, tarcheck +from conda_build import environ, source, tarcheck, external from conda_build.config import config from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path from conda_build.scripts import create_entry_points, prepend_bin_path @@ -472,12 +472,23 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, # have the appropriate VCS available in the environment. People # are not used to explicitly listing it in recipes, though. # We add it for them here, but warn them about it. - vcs_source = m.uses_vcs() + vcs_source = m.uses_vcs_in_build() if vcs_source and vcs_source not in specs: - specs.append(vcs_source) - log.warn("Your recipe depends on {} at build time (for templates), " - "but you have not listed it as a build dependency. Doing so for" - " this build.") + vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source + has_vcs_available = os.path.isfile(external.find_executable(vcs_executable)) + if not has_vcs_available: + if (vcs_source != "mercurial" or + not any(spec.startswith('python') and "3." in spec + for spec in specs)): + specs.append(vcs_source) + + log.warn("Your recipe depends on {} at build time (for templates), " + "but you have not listed it as a build dependency. Doing " + "so for this build.") + else: + raise ValueError("Your recipe uses mercurial in build, but mercurial" + " does not yet support Python 3. Please handle all of " + "your mercurial actions outside of your build script.") # Display the name only # Version number could be missing due to dependency on source info. create_env(config.build_prefix, specs) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b302624e03..56eb7f011a 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -29,6 +29,8 @@ from conda_build.config import config from conda_build.utils import comma_join +on_win = (sys.platform == 'win32') + def ns_cfg(): # Remember to update the docs of any of this changes @@ -695,7 +697,7 @@ def __repr__(self): ''' return self.__str__() - def uses_vcs(self): + def uses_vcs_in_meta(self): """returns true if recipe contains metadata associated with version control systems. If this metadata is present, a download/copy will be forced in parse_or_try_download. """ @@ -719,3 +721,18 @@ def uses_vcs(self): vcs = "mercurial" return vcs return None + + def uses_vcs_in_build(self): + build_script = "bld.bat" if on_win else "build.sh" + build_script = os.path.join(os.path.dirname(self.meta_path), build_script) + if os.path.isfile(build_script): + vcs_types = ["git", "svn", "hg"] + with open(self.meta_path) as f: + build_script = f.read() + for vcs in vcs_types: + matches = re.findall(r"{}(?:\.exe)?".format(vcs), build_script) + if len(matches) > 0: + if vcs == "hg": + vcs = "mercurial" + return vcs + return None diff --git a/conda_build/render.py b/conda_build/render.py index 1354a57c82..04afaa9b4b 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -77,7 +77,7 @@ def bldpkg_path(m): def parse_or_try_download(metadata, no_download_source, verbose, force_download=False, dirty=False): - if (force_download or (not no_download_source and metadata.uses_vcs())): + if (force_download or (not no_download_source and metadata.uses_vcs_in_meta())): # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: diff --git a/conda_build/windows.py b/conda_build/windows.py index 6370f67a1a..d224999dc9 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -218,11 +218,10 @@ def build(m, bld_bat, dirty=False, activate=True): fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None))) if activate: fo.write("call activate _build\n") - fo.write('\n') fo.write("REM ===== end generated header =====\n") fo.write(data) - cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat'] + cmd = [os.environ['COMSPEC'], '/c', 'bld.bat'] _check_call(cmd, cwd=src_dir) kill_processes() fix_staged_scripts() diff --git a/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat index 1a2f8ad626..cc6eed0c09 100644 --- a/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat +++ b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat @@ -1,5 +1,8 @@ mkdir %PREFIX%\etc\conda\activate.d +:: output something so it's more obvious when scripts are running +echo "echo setting TEST_VAR" > %PREFIX%\etc\conda\activate.d\test.bat echo set TEST_VAR=1 > %PREFIX%\etc\conda\activate.d\test.bat mkdir %PREFIX%\etc\conda\deactivate.d +echo "echo setting TEST_VAR" > %PREFIX%\etc\conda\deactivate.d\test.bat echo set TEST_VAR= > %PREFIX%\etc\conda\deactivate.d\test.bat