From d02b1a77d1293c4afb6557f48b58f2f74742d8f3 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 21 Nov 2022 23:41:53 +0200 Subject: [PATCH] [Core] Refactor serializer logic --- CHANGELOG.md | 3 ++- examples/call_async_cython.py | 8 ++---- lithops/job/serialize.py | 51 +++++++++++++++++------------------ 3 files changed, 28 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 977b27ee6..8ee2910f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,10 @@ ### Added - [Google Cloud Functions] Added Python 3.10 runtime compatibility -- [Core] Allow to automatically transfer .so (cythonized .py) files if referneced in the code +- [Core] Allow to automatically transfer .so (cythonized .py) files if referenced in the code ### Changed +- [Core] Improved cython coverage - [IBM VPC] Make 'image_id' mandatory in config - [IBM VPC] Infer zone_name from subnet - [Knative] Reduced service name length diff --git a/examples/call_async_cython.py b/examples/call_async_cython.py index a4164472c..e5a01fcd6 100755 --- a/examples/call_async_cython.py +++ b/examples/call_async_cython.py @@ -10,11 +10,7 @@ from function import my_c_function -def my_function(x): - return my_c_function(x) - - if __name__ == '__main__': - fexec = lithops.FunctionExecutor() - fexec.call_async(my_function, 3) + fexec = lithops.FunctionExecutor(log_level='DEBUG') + fexec.call_async(my_c_function, 3) print(fexec.get_result()) diff --git a/lithops/job/serialize.py b/lithops/job/serialize.py index 21f2de99b..d7eb5ab48 100644 --- a/lithops/job/serialize.py +++ b/lithops/job/serialize.py @@ -75,7 +75,8 @@ def __call__(self, list_of_objs, include_modules, exclude_modules): mod_paths.add(origin) else: self._modulemgr.add(module_name) - direct_modules.add(origin if origin not in ['built-in', None] else module_name) + + direct_modules.add(origin if origin != 'built-in' else module_name) logger.debug("Referenced modules: {}".format(None if not direct_modules else ", ".join(direct_modules))) @@ -94,8 +95,8 @@ def __call__(self, list_of_objs, include_modules, exclude_modules): else: mod_paths = mod_paths.union(tent_mod_paths) - logger.debug("Modules to transmit: {}" - .format(None if not mod_paths else ", ".join(mod_paths))) + logger.debug("Modules to transmit: {}".format(None if + not mod_paths else ", ".join(mod_paths))) return (strs, mod_paths) @@ -107,40 +108,36 @@ def _module_inspect(self, obj): seen = set() mods = set() - if inspect.isfunction(obj) or (inspect.ismethod(obj) and inspect.isfunction(obj.__func__)): + if inspect.isfunction(obj) or (inspect.ismethod(obj) and + inspect.isfunction(obj.__func__)): # The obj is the user's function worklist.append(obj) - elif type(obj) == dict: - # the obj is the user's iterdata - to_anayze = list(obj.values()) - for param in to_anayze: - if type(param).__module__ != "__builtin__": - if inspect.isfunction(param): - # it is a user defined function - worklist.append(param) - else: - # it is a user defined class - members = inspect.getmembers(param) - for k, v in members: - if inspect.ismethod(v) and inspect.isfunction(v.__func__): - worklist.append(v) - elif type(obj).__name__ == 'cython_function_or_method': - members = inspect.getmembers(obj) - for k, v in members: + for k, v in inspect.getmembers(obj): if k == '__globals__': mods.add(v['__file__']) + elif type(obj) == dict: + # the obj is the user's iterdata + for param in obj.values(): + if type(param).__module__ == "__builtin__": + continue + elif inspect.isfunction(param): + # it is a user defined function + worklist.append(param) + else: + # it is a user defined class + for k, v in inspect.getmembers(param): + if inspect.isfunction(v) or (inspect.ismethod(v) and + inspect.isfunction(v.__func__)): + worklist.append(v) else: # The obj is the user's function but in form of a class - members = inspect.getmembers(obj) found_methods = [] - for k, v in members: - if inspect.isfunction(v): - found_methods.append(k) - worklist.append(v) - elif inspect.ismethod(v) and inspect.isfunction(v.__func__): + for k, v in inspect.getmembers(obj): + if inspect.isfunction(v) or (inspect.ismethod(v) and + inspect.isfunction(v.__func__)): found_methods.append(k) worklist.append(v) if "__call__" not in found_methods: