ipyparallel: Can't pickly memoryview objects

Florian Lindner <[email protected]> Thu, 16 Mar 2017 10:07:46 +0100
Newsgroups gmane.comp.python.ipython.user
Message-ID <[email protected]>
Hello,

I have this small sample program:

import ipyparallel as ipp, numpy as np

class X:
    def __init__(self, x, xs):
        self.A = np.full((10, 10), x)
        np.fill_diagonal(self.A, x*10)

    def norm(self):
        return np.linalg.norm(self.A)


def main():
    rc = ipp.Client()
    lview = rc.load_balanced_view()
    dview = rc[:]
    dview.push({"X" : X})
    dview.execute("import numpy as np")

    xs = np.linspace(1, 100, 10)
    sqxs = []

    sqxs = lview.map(lambda x: X(x, xs), xs)
    sqxs.wait()

    for i in sqxs:
        print(i.norm())

if __name__ == "__main__":
    main()


trying it to run gives TypeError: can't pickle memoryview objects. Probably because I provide the xs array to X.

How can I work around this situation?

Thanks,
Florian

Complete traceback:

Traceback (most recent call last):
  File "ipython_parallel.py", line 30, in <module>
    main()
  File "ipython_parallel.py", line 23, in main
    sqxs = lview.map(lambda x: X(x, xs), xs)
  File "<decorator-gen-136>", line 2, in map
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 50, in sync_results
    ret = f(self, *args, **kwargs)
  File "<decorator-gen-135>", line 2, in map
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 35, in save_ids
    ret = f(self, *args, **kwargs)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 1109, in map
    return pf.map(*sequences)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/remotefunction.py", line 285, in map
    return self(*sequences, __ipp_mapping=True)
  File "<decorator-gen-118>", line 2, in __call__
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/remotefunction.py", line 76, in sync_view_results
    return f(self, *args, **kwargs)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/remotefunction.py", line 259, in __call__
    ar = view.apply(f, *args)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 211, in apply
    return self._really_apply(f, args, kwargs)
  File "<decorator-gen-134>", line 2, in _really_apply
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 47, in sync_results
    return f(self, *args, **kwargs)
  File "<decorator-gen-133>", line 2, in _really_apply
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 35, in save_ids
    ret = f(self, *args, **kwargs)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/view.py", line 1037, in _really_apply
    metadata=metadata)
  File "/usr/lib/python3.6/site-packages/ipyparallel/client/client.py", line 1395, in send_apply_request
    item_threshold=self.session.item_threshold,
  File "/usr/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py", line 166, in pack_apply_message
    serialize_object(arg, buffer_threshold, item_threshold) for arg in args))
  File "/usr/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py", line 166, in <genexpr>
    serialize_object(arg, buffer_threshold, item_threshold) for arg in args))
  File "/usr/lib/python3.6/site-packages/ipyparallel/serialize/serialize.py", line 112, in serialize_object
    buffers.insert(0, pickle.dumps(cobj, PICKLE_PROTOCOL))
TypeError: can't pickle memoryview objects