[WIP] Cleanup xcode detect etc

.travis.yml

@@ -63,6 +63,17 @@ matrix:
       <<: *osx-setup-steps
       script: pipenv run -v test -f make,ninja
 
+    - name: "macOS: XCode10.2 test with make and ninja on Python 3.7"
+      os: osx
+      osx_image: xcode10.2  # provides Python 3.7.2
+      language: cpp  # 'language: python' is not yet supported on macOS
+      install:
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ninja
+        - pip3 install --upgrade pip pipenv
+        - pipenv install --dev
+        - pipenv install PyObjC  # This is not in the Pipfile because it breaks other platforms https://github.com/pypa/pipenv/issues/3187
+      script: pipenv run -v test -f make,ninja
+
     - name: "lint with Python 2.7"
       python: '2.7'
       <<: *linux-setup-steps

Pipfile

@@ -5,7 +5,8 @@ name = "pypi"
 
 [dev-packages]
 flake8 = "*"
-pywin32 = { version = "*", sys_platform = "=='win32'" }
+pywin32 = {version = "*",sys_platform = "=='win32'"}
+pprintpp = "*"
 
 [packages]
 

gyp/MSVS/MSVSSettings.py

@@ -196,10 +196,11 @@ class _Enumeration(_Type):
   def __init__(self, label_list, new=None):
     _Type.__init__(self)
     self._label_list = label_list
-    self._msbuild_values = set(value for value in label_list
-                               if value is not None)
+    self._label_list_new = label_list
+    self._msbuild_values = set(value for value in label_list if value is not None)
     if new is not None:
       self._msbuild_values.update(new)
+      self._label_list_new += new
 
   def ValidateMSVS(self, value):
     # Try to convert.  It will raise an exception if not valid.
@@ -211,10 +212,10 @@ def ValidateMSBuild(self, value):
 
   def ConvertToMSBuild(self, value):
     index = int(value)
-    if index < 0 or index >= len(self._label_list):
-      raise ValueError('index value (%d) not in expected range [0, %d)' %
-                       (index, len(self._label_list)))
-    label = self._label_list[index]
+    # Conversion is done by the new list (including new values)
+    if index < 0 or index >= len(self._label_list_new):
+      raise ValueError('index value (%d) not in expected range [0, %d)' % (index, len(self._label_list_new)))
+    label = self._label_list_new[index]
     if label is None:
       raise ValueError('converted value for %s not specified.' % value)
     return label
@@ -460,27 +461,21 @@ def ConvertToMSBuildSettings(msvs_settings, stderr=sys.stderr):
   """
   msbuild_settings = {}
   for msvs_tool_name, msvs_tool_settings in msvs_settings.items():
-    if msvs_tool_name in _msvs_to_msbuild_converters:
-      msvs_tool = _msvs_to_msbuild_converters[msvs_tool_name]
-      for msvs_setting, msvs_value in msvs_tool_settings.items():
-        if msvs_setting in msvs_tool:
-          # Invoke the translation function.
-          try:
-            msvs_tool[msvs_setting](msvs_value, msbuild_settings)
-          except ValueError as e:
-            print(('Warning: while converting %s/%s to MSBuild, '
-                              '%s' % (msvs_tool_name, msvs_setting, e)),
-                              file=stderr)
-        else:
-          _ValidateExclusionSetting(msvs_setting,
-                                    msvs_tool,
-                                    ('Warning: unrecognized setting %s/%s '
-                                     'while converting to MSBuild.' %
-                                     (msvs_tool_name, msvs_setting)),
-                                    stderr)
-    else:
-      print(('Warning: unrecognized tool %s while converting to '
-                        'MSBuild.' % msvs_tool_name), file=stderr)
+    msvs_tool = _msvs_to_msbuild_converters.get(msvs_tool_name)
+    if msvs_tool is None:
+      print('Warning: unrecognized tool %s while converting to MSBuild.' % msvs_tool_name, file=stderr)
+      continue
+    for msvs_setting, msvs_value in msvs_tool_settings.items():
+      setting_converter = msvs_tool.get(msvs_setting)
+      if setting_converter is None:
+        exclusion_message = 'Warning: unrecognized setting %s/%s while converting to MSBuild.' % (msvs_tool_name, msvs_setting)
+        _ValidateExclusionSetting(msvs_setting, msvs_tool, exclusion_message, stderr)
+        continue
+      # Invoke the translation function.
+      try:
+        setting_converter(msvs_value, msbuild_settings)
+      except ValueError as e:
+        print('Warning: while converting %s/%s to MSBuild, %s' % (msvs_tool_name, msvs_setting, e), file=stderr)
   return msbuild_settings
 
 
@@ -507,35 +502,31 @@ def ValidateMSBuildSettings(settings, stderr=sys.stderr):
 
 
 def _ValidateSettings(validators, settings, stderr):
-  """Validates that the settings are valid for MSBuild or MSVS.
+  """
+  Validates that the settings are valid for MSBuild or MSVS.
 
   We currently only validate the names of the settings, not their values.
 
   Args:
       validators: A dictionary of tools and their validators.
-      settings: A dictionary.  The key is the tool name.  The values are
-          themselves dictionaries of settings and their values.
+      settings: A dictionary.  The key is the tool name, values are dictionaries of settings and their values.
       stderr: The stream receiving the error messages.
   """
   for tool_name in settings:
-    if tool_name in validators:
-      tool_validators = validators[tool_name]
-      for setting, value in settings[tool_name].items():
-        if setting in tool_validators:
-          try:
-            tool_validators[setting](value)
-          except ValueError as e:
-            print(('Warning: for %s/%s, %s' %
-                              (tool_name, setting, e)), file=stderr)
-        else:
-          _ValidateExclusionSetting(setting,
-                                    tool_validators,
-                                    ('Warning: unrecognized setting %s/%s' %
-                                     (tool_name, setting)),
-                                    stderr)
-
-    else:
-      print(('Warning: unrecognized tool %s' % tool_name), file=stderr)
+    tool_validators = validators.get(tool_name)
+    if tool_validators is None:
+      print('Warning: unrecognized tool %s' % tool_name, file=stderr)
+      continue
+    for setting, value in settings[tool_name].items():
+      validator = tool_validators.get(setting)
+      if validator is None:
+        exclusion_message = 'Warning: unrecognized setting %s/%s' % (tool_name, setting)
+        _ValidateExclusionSetting(setting, tool_validators, exclusion_message, stderr)
+        continue
+      try:
+        validator(value)
+      except ValueError as e:
+        print('Warning: for %s/%s, %s' % (tool_name, setting, e), file=stderr)
 
 
 # MSVS and MBuild names of the tools.
@@ -879,8 +870,7 @@ def _ValidateSettings(validators, settings, stderr):
                        'PromptImmediately',  # /ERRORREPORT:PROMPT
                        'QueueForNextLogin'],  # /ERRORREPORT:QUEUE
                       new=['SendErrorReport']))  # /ERRORREPORT:SEND
-_Renamed(_link, 'IgnoreDefaultLibraryNames', 'IgnoreSpecificDefaultLibraries',
-         _file_list)  # /NODEFAULTLIB
+_Renamed(_link, 'IgnoreDefaultLibraryNames', 'IgnoreSpecificDefaultLibraries', _file_list)  # /NODEFAULTLIB
 _Renamed(_link, 'ResourceOnlyDLL', 'NoEntryPoint', _boolean)  # /NOENTRY
 _Renamed(_link, 'SwapRunFromNet', 'SwapRunFromNET', _boolean)  # /SWAPRUN:NET
 

gyp/XCodeDetect.py

@@ -1,65 +1,83 @@
+"""Simplify access to Xcode information."""
 import subprocess
-import sys
+
+from gyp.common import memoize, DebugOutput, DEBUG_GENERAL
 
 
 def run(*cmd_args):
-  return subprocess.check_output(cmd_args, stderr=subprocess.PIPE).decode('utf-8')
+  return subprocess.check_output(cmd_args, stderr=subprocess.PIPE).decode('utf-8').strip()
+
+
+@memoize
+def Version():
+  version = ''
+  try:
+    lines = run('xcodebuild', '-version').splitlines()
+    version = ''.join(lines[0].split()[-1].split('.'))
+    version = (version + '0' * (3 - len(version))).zfill(4)
+  except subprocess.CalledProcessError:
+    pass
+  try:
+    lines = run('pkgutil', '--pkg-info=com.apple.pkg.CLTools_Executables').splitlines()
+    for l in lines:
+      n, v = l.split(': ', 1)
+      if n != 'version':
+        continue
+      parts = v.split('.', 4)
+      version = '%s%s%s%s' % tuple(parts[0:4])
+      break
+  except subprocess.CalledProcessError:
+    pass
+  return version
+
+
+@memoize
+def GetSdkVersionInfoItem(sdk, item):
+  # xcodebuild requires Xcode and can't run on Command Line Tools-only systems from 10.7 onward.
+  # Since the CLT has no SDK paths anyway, returning None is the most sensible route and should still do the right thing.
+  try:
+    return run('xcrun', '--sdk', sdk, item)
+  except subprocess.CalledProcessError as e:
+    DebugOutput(DEBUG_GENERAL, 'cmd=%s\nstatus=%s\noutput=%s' % (' '.join(e.cmd), e.returncode, e.output))
+    return None
 
 
-class XCodeDetect(object):
-  """Simplify access to Xcode information."""
-  _cache = {}
+@memoize
+def SDKVersion():
+  try:
+    out = run('xcrun', '--show-sdk-version')
+  except subprocess.CalledProcessError:
+    try:
+      out = run('xcodebuild', '-version', '-sdk', '', 'SDKVersion')
+    except subprocess.CalledProcessError:
+      return None
+  version = out.strip()
+  return version
 
-  @staticmethod
-  def Version():
-    if 'Version' not in XCodeDetect._cache:
-      version = ''
-      try:
-        lines = run('xcodebuild', '-version').splitlines()
-        version = ''.join(lines[0].decode('utf-8').split()[-1].split('.'))
-        version = (version + '0' * (3 - len(version))).zfill(4)
-      except subprocess.CalledProcessError:
-        pass
-      try:
-        lines = run('pkgutil', '--pkg-info=com.apple.pkg.CLTools_Executables').splitlines()
-        for l in lines:
-          n, v = l.split(': ', 1)
-          if n != 'version':
-            continue
-          parts = v.split('.',4)
-          version = '%s%s%s%s' % tuple(parts[0:4])
-          break
-      except subprocess.CalledProcessError:
-        pass
-      XCodeDetect._cache['Version'] = version
-    return XCodeDetect._cache['Version']
 
+@memoize
+def IPhoneSDKPath():
+  try:
+    path = GetSdkVersionInfoItem('iphoneos', '--show-sdk-path')
+  except subprocess.CalledProcessError:
+    return None
+  return path
 
-  @staticmethod
-  def SDKVersion():
-    if 'SDKVersion' not in XCodeDetect._cache:
-      out = ''
-      try:
-        out = run('xcrun', '--show-sdk-version')
-      except subprocess.CalledProcessError:
-        pass
-      try:
-        out = run('xcodebuild', '-version', '-sdk', '', 'SDKVersion')
-      except subprocess.CalledProcessError:
-        pass
-      XCodeDetect._cache['SDKVersion'] = out.strip()
-    return XCodeDetect._cache['SDKVersion']
 
+@memoize
+def GetIOSCodeSignIdentityKey(identity):
+  if not identity:
+    return None
+  output = run('security', 'find-identity', '-p', 'codesigning', '-v')
+  # TODO(refack):could also be '0 valid identities found'
+  output_lines = output.splitlines()
+  match_lines = [line for line in output_lines if identity in line]
+  if len(match_lines) != 1:
+    DebugOutput(DEBUG_GENERAL, 'Not exactly one codesigning fingerprints for identity: %s\n output=\n%s' % (identity, output))
+  fingerprint = match_lines[0].split()[1]
+  return fingerprint
 
-  @staticmethod
-  def HasIPhoneSDK():
-    if not sys.platform == 'darwin':
-      return False
 
-    if 'HasIPhoneSDK' not in XCodeDetect._cache:
-      try:
-        out = run('xcrun', '--sdk', 'iphoneos', '--show-sdk-path')
-      except subprocess.CalledProcessError:
-        out = 1
-      XCodeDetect._cache['HasIPhoneSDK'] = out == 0
-    return XCodeDetect._cache['HasIPhoneSDK']
+@memoize
+def BuildMachineOSBuild():
+  return run('sw_vers', '-buildVersion')