Merge pull request #11 from Autodesk/0.7.1fixes

0.7.1fixes

README.md

@@ -5,6 +5,8 @@ A high-level python interface for running computational jobs using a variety of
 ## Installation
 
 Normal installation:
+`pyccc` works with Python 2.6 and 3.5+. You'll probably want a local version of [docker](https://www.docker.com/get-docker) running.
+
 ```shell
 pip install pyccc
 ```

pyccc/python.py

@@ -42,6 +42,14 @@ def exports(o):
 DEFAULT_INTERPRETER = 'python%s.%s' % sys.version_info[:2]
 PICKLE_PROTOCOL = 2  # required for 2/3 compatibile pickle objects
 
+if sys.version_info.major == 2:
+    PYVERSION = 2
+    import __builtin__ as BUILTINS
+else:
+    assert sys.version_info.major == 3
+    PYVERSION = 3
+    import builtins as BUILTINS
+
 
 @exports
 class PythonCall(object):
@@ -51,12 +59,14 @@ def __init__(self, function, *args, **kwargs):
         self.kwargs = kwargs
 
         try:
-            temp = function.__self__.__class__
+            cls = function.__self__.__class__
         except AttributeError:
             self.is_instancemethod = False
         else:
-            self.is_instancemethod = True
-
+            if function.__self__ == BUILTINS or function.__self__ is None:
+                self.is_instancemethod = False
+            else:
+                self.is_instancemethod = True
 
 @exports
 class PythonJob(job.Job):
@@ -111,6 +121,7 @@ def _get_python_files(self):
         python_files['function.pkl'] = pyccc.BytesContainer(
                 pickle.dumps(remote_function, protocol=PICKLE_PROTOCOL),
                 name='function.pkl')
+        self._remote_function = remote_function
 
         sourcefile = StringContainer(self._get_source(),
                                      name='source.py')
@@ -238,7 +249,7 @@ class PackagedFunction(native.object):
     """
     def __init__(self, function_call):
         func = function_call.function
-        self.is_imethod = hasattr(func, '__self__')
+        self.is_imethod = function_call.is_instancemethod
         if self.is_imethod:
             self.obj = func.__self__
             self.imethod_name = func.__name__

pyccc/source_inspections.py

@@ -15,6 +15,7 @@
 Source code inspections for sending python code to workers
 """
 from __future__ import print_function, unicode_literals, absolute_import, division
+
 from future import standard_library, builtins
 standard_library.install_aliases()
 from future.builtins import *
@@ -23,7 +24,6 @@
 import linecache
 import re
 import string
-from collections import namedtuple
 
 __author__ = 'aaronvirshup'
 
@@ -48,7 +48,7 @@ def get_global_vars(func):
     for name, value in closure['global'].items():
         if inspect.ismodule(value):  # TODO: deal FUNCTIONS from closure
             globalvars['modules'][name] = value.__name__
-        elif inspect.isfunction(value):
+        elif inspect.isfunction(value) or inspect.ismethod(value):
             globalvars['functions'][name] = value
         else:
             globalvars['vars'][name] = value
@@ -69,6 +69,9 @@ def getsource(classorfunc):
     Returns:
         str: source code (without any decorators)
     """
+    if _isbuiltin(classorfunc):
+        return ''
+
     try:
         source = inspect.getsource(classorfunc)
     except TypeError:  # raised if defined in __main__ - use fallback to get the source instead
@@ -195,28 +198,33 @@ def getclosurevars(func):
     if inspect.ismethod(func):
         func = func.__func__
 
-    if not inspect.isfunction(func):
+    elif not inspect.isroutine(func):
         raise TypeError("'{!r}' is not a Python function".format(func))
 
-    code = func.__code__
+    # AMVMOD: deal with python 2 builtins that don't define these
+    code = getattr(func, '__code__', None)
+    closure = getattr(func, '__closure__', None)
+    co_names = getattr(code, 'co_names', ())
+    glb = getattr(func, '__globals__', {})
+
     # Nonlocal references are named in co_freevars and resolved
     # by looking them up in __closure__ by positional index
-    if func.__closure__ is None:
+    if closure is None:
         nonlocal_vars = {}
     else:
         nonlocal_vars = {var: cell.cell_contents
                          for var, cell in zip(code.co_freevars, func.__closure__)}
 
     # Global and builtin references are named in co_names and resolved
     # by looking them up in __globals__ or __builtins__
-    global_ns = func.__globals__
+    global_ns = glb
     builtin_ns = global_ns.get("__builtins__", builtins.__dict__)
     if inspect.ismodule(builtin_ns):
         builtin_ns = builtin_ns.__dict__
     global_vars = {}
     builtin_vars = {}
     unbound_names = set()
-    for name in code.co_names:
+    for name in co_names:
         if name in ("None", "True", "False"):
             # Because these used to be builtins instead of keywords, they
             # may still show up as name references. We ignore them.
@@ -233,3 +241,12 @@ def getclosurevars(func):
             'global': global_vars,
             'builtin': builtin_vars,
             'unbound': unbound_names}
+
+
+def _isbuiltin(obj):
+    if inspect.isbuiltin(obj):
+        return True
+    elif obj.__module__ in ('builtins', '__builtin__'):
+        return True
+    else:
+        return False

pyccc/static/run_job.py

@@ -34,6 +34,14 @@
 RENAMETABLE = {'pyccc.python': 'source',
                '__main__': 'source'}
 
+if sys.version_info.major == 2:
+    PYVERSION = 2
+    import __builtin__ as BUILTINS
+else:
+    assert sys.version_info.major == 3
+    PYVERSION = 3
+    import builtins as BUILTINS
+
 
 def main():
     os.environ['IS_PYCCC_JOB'] = '1'
@@ -58,7 +66,10 @@ def load_job():
         funcpkg = MappedUnpickler(pf).load()
 
     if hasattr(funcpkg, 'func_name'):
-        func = getattr(source, funcpkg.func_name)
+        try:
+            func = getattr(source, funcpkg.func_name)
+        except AttributeError:
+            func = getattr(BUILTINS, funcpkg.func_name)
     else:
         func = None
 
@@ -103,17 +114,24 @@ def find_class(self, module, name):
             # can't use ``super`` here (not 2/3 compatible)
             klass = pickle.Unpickler.find_class(self, modname, name)
 
-        except ImportError:
-            if hasattr(source, name):
-                newmod = types.ModuleType(modname)
-                sys.modules[modname] = newmod
-                setattr(newmod, name, getattr(source, name))
-            klass = self.find_class(modname, name)
-
-        klass.__module__ = module
+        except (ImportError, RuntimeError):
+            definition = getattr(source, name)
+            newmod = _makemod(modname)
+            sys.modules[modname] = newmod
+            setattr(newmod, name, definition)
+            klass = pickle.Unpickler.find_class(self, newmod.__name__, name)
+            klass.__module__ = module
         return klass
 
 
+def _makemod(name):
+    fields = name.split('.')
+    for i in range(0, len(fields)):
+        tempname = str('.'.join(fields[0:i]))
+        sys.modules[tempname] = types.ModuleType(tempname)
+    return sys.modules[tempname]
+
+
 if __name__ == '__main__':
     main()
 

pyccc/tests/engine_fixtures.py

@@ -0,0 +1,40 @@
+import pytest
+import pyccc
+
+__all__ = 'typedfixture fixture_types subprocess_engine public_ccc_engine local_docker_engine'.split()
+
+fixture_types = {}
+
+
+def typedfixture(*types, **kwargs):
+    """This is a decorator that lets us associate fixtures with one or more arbitrary types.
+    This makes it easy, later, to run the same test on all fixtures of a given type"""
+
+    def fixture_wrapper(func):
+        for t in types:
+            fixture_types.setdefault(t, []).append(func.__name__)
+        return pytest.fixture(**kwargs)(func)
+
+    return fixture_wrapper
+
+
+###################
+# Fixtures        #
+###################
+
+@typedfixture('engine')
+def subprocess_engine():
+    return pyccc.Subprocess()
+
+
+@typedfixture('engine')
+def public_ccc_engine():
+    if not pytest.config.getoption("--testccc"):
+        pytest.skip("need --testccc option to run")
+    else:
+        return pyccc.CloudComputeCannon('cloudcomputecannon.bionano.autodesk.com:9000')
+
+
+@typedfixture('engine')
+def local_docker_engine():
+    return pyccc.Docker()