Handling a classmethod etching problem with copy_reg

Question

I encountered an etching error when working with multiprocessing:

from multiprocessing import Pool def test_func(x): return x**2 class Test: @classmethod def func(cls, x): return x**2 def mp_run(n, func, args): return Pool(n).map(func, args) if __name__ == '__main__': args = range(1,6) print mp_run(5, test_func, args) # [1, 4, 9, 16, 25] print mp_run(5, Test.func, args) """ Exception in thread Thread-3: Traceback (most recent call last): File "/usr/lib64/python2.6/threading.py", line 532, in __bootstrap_inner self.run() File "/usr/lib64/python2.6/threading.py", line 484, in run self.__target(*self.__args, **self.__kwargs) File "/usr/lib64/python2.6/multiprocessing/pool.py", line 225, in _handle_tasks put(task) PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup __builtin__.instancemethod failed """ 

And I found a useful thread here , the solution is perfect for these self-style methods, but I had a problem applying the recipe to @classmethod:

  def _pickle_method(method): func_name = method.im_func.__name__ obj = method.im_self cls = method.im_class return _unpickle_method, (func_name, obj, cls) def _unpickle_method(func_name, obj, cls): try: for cls in cls.mro(): try: func = cls.__dict__[func_name] except KeyError: pass else: break except AttributeError: func = cls.__dict__[func_name] return func.__get__(obj, cls) copy_reg.pickle(MethodType, _pickle_method, _unpickle_method) new_func = pickle.loads(pickle.dumps(Test.func)) """ Traceback (most recent call last): File "test3.py", line 45, in <module> new_func = pickle.loads(pickle.dumps(Test.func)) File "/usr/lib64/python2.6/pickle.py", line 1366, in dumps Pickler(file, protocol).dump(obj) File "/usr/lib64/python2.6/pickle.py", line 224, in dump self.save(obj) File "/usr/lib64/python2.6/pickle.py", line 331, in save self.save_reduce(obj=obj, *rv) File "/usr/lib64/python2.6/pickle.py", line 401, in save_reduce save(args) File "/usr/lib64/python2.6/pickle.py", line 286, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python2.6/pickle.py", line 562, in save_tuple save(element) File "/usr/lib64/python2.6/pickle.py", line 286, in save f(self, obj) # Call unbound method with explicit self File "/usr/lib64/python2.6/pickle.py", line 748, in save_global (obj, module, name)) pickle.PicklingError: Can't pickle <type 'classobj'>: it not found as __builtin__.classobj """ 

Is there a way to change multiple lines to make it work for classmethod?

Answer 1

I changed the recipe so that it works with classmethod. Here is the code.

 import copy_reg import types def _pickle_method(method): func_name = method.im_func.__name__ obj = method.im_self cls = method.im_class if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names cls_name = cls.__name__.lstrip('_') func_name = '_%s%s' % (cls_name, func_name) return _unpickle_method, (func_name, obj, cls) def _unpickle_method(func_name, obj, cls): if obj and func_name in obj.__dict__: cls, obj = obj, None # if func_name is classmethod for cls in cls.__mro__: try: func = cls.__dict__[func_name] except KeyError: pass else: break return func.__get__(obj, cls) copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method) 

Answer 2

The following solution now also correctly processes class methods. Please let me know if something is missing.

 def _pickle_method(method): """ Pickle methods properly, including class methods. """ func_name = method.im_func.__name__ obj = method.im_self cls = method.im_class if isinstance(cls, type): # handle classmethods differently cls = obj obj = None if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names cls_name = cls.__name__.lstrip('_') func_name = '_%s%s' % (cls_name, func_name) return _unpickle_method, (func_name, obj, cls) def _unpickle_method(func_name, obj, cls): """ Unpickle methods properly, including class methods. """ if obj is None: return cls.__dict__[func_name].__get__(obj, cls) for cls in cls.__mro__: try: func = cls.__dict__[func_name] except KeyError: pass else: break return func.__get__(obj, cls) 

Answer 3

Instead of returning the actual class object from _pickle_method , return a string that can be used to import it when scattering, and then do it in _unpickle_method