Merge pull request #219 in LCL/wolframclientforpython from bugfix/381526-memoryview to master

* commit '6e245c5ba0058a3e69ff655fe1268aec804ae137':
  adding tests
  using dispatch to implement force_bytes and force_text
  adding comment for py2
  adding a custom rule for memoryview
  fixing six
  need to add encoding for py2

Author: crazyeng

wolframclient/deserializers/wxf/wxfparser.py

@@ -10,6 +10,28 @@
 )
 from wolframclient.serializers.wxfencoder.streaming import ExactSizeReader, ZipCompressedReader
 from wolframclient.utils import six
+from wolframclient.utils.dispatch import Dispatch
+
+wxf_input_to_buffer = Dispatch()
+
+@wxf_input_to_buffer.dispatch((six.binary_type, six.buffer_types))
+def encode_buffer(wxf_input):
+    return six.BytesIO(wxf_input)
+
+if six.PY2:
+    @wxf_input_to_buffer.dispatch(memoryview, replace_existing = True)
+    def encode_buffer(wxf_input):
+        return six.BytesIO(wxf_input.tobytes())    
+
+@wxf_input_to_buffer.dispatch(object)
+def encode_default(wxf_input):
+    if hasattr(wxf_input, 'read'):
+        return wxf_input
+    raise TypeError(
+        "Class %s neither implements a read method nor is a binary type."
+        % wxf_input.__class__.__name__
+    )
+
 
 
 class WXFParser(object):
@@ -58,15 +80,8 @@ def __init__(self, wxf_input):
         """WXF parser returning Python object from a WXF encoded byte sequence.
         """
         self.context = SerializationContext()
-        if isinstance(wxf_input, (six.binary_type, six.buffer_types)):
-            self.reader = six.BytesIO(wxf_input)
-        elif hasattr(wxf_input, "read"):
-            self.reader = wxf_input
-        else:
-            raise TypeError(
-                "Class %s neither implements a read method nor is a binary type."
-                % wxf_input.__class__.__name__
-            )
+        self.reader  = wxf_input_to_buffer(wxf_input)
+
         version, compress = self.parse_header()
         if compress == True:
             self.reader = ZipCompressedReader(self.reader)

wolframclient/tests/core_functions.py

@@ -1,7 +1,9 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, unicode_literals
 
 from wolframclient.utils import six
 from wolframclient.utils.dispatch import Dispatch
+from wolframclient.utils.encoding import force_bytes, force_text
 from wolframclient.utils.functional import composition, flatten, iterate, partition, riffle
 from wolframclient.utils.tests import TestCase as BaseTestCase
 
@@ -38,6 +40,24 @@ def test_partition(self):
 
         self.assertEqual(list(partition([], 3)), [])
 
+    def test_force_encoding(self):
+
+        self.assertEqual(force_text("aà"), "aà")
+        self.assertEqual(force_text(abs), "<built-in function abs>")
+        self.assertEqual(force_text(b"a\xc3\xa0"), "aà")
+
+        self.assertEqual(force_text(memoryview(b"abc")), "abc")
+        self.assertEqual(force_text(bytearray(b"abc")), "abc")
+
+        self.assertEqual(force_bytes(b"abc"), b"abc")
+        self.assertEqual(force_bytes(abs), b"<built-in function abs>")
+        self.assertEqual(force_bytes("aà"), b"a\xc3\xa0")
+
+        self.assertEqual(force_text(force_bytes("aà")), "aà")
+
+        self.assertEqual(force_bytes(memoryview(b"abc")), b"abc")
+        self.assertEqual(force_bytes(bytearray(b"abc")), b"abc")
+
     def test_dispatch(self):
 
         normalizer = Dispatch()

wolframclient/tests/deserializers/wxf_deserialize.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from __future__ import absolute_import, print_function, unicode_literals
 
 import decimal

wolframclient/tests/serializers/wxf_serialization.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from __future__ import absolute_import, print_function, unicode_literals
 
 import decimal

wolframclient/utils/encoding.py

@@ -1,63 +1,87 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 from wolframclient.utils import six
+from wolframclient.utils.dispatch import Dispatch
 from wolframclient.utils.functional import map
 
+force_text = Dispatch()
 
-def force_text(s, encoding="utf-8", errors="strict"):
-    """
-    Similar to smart_text, except that lazy instances are resolved to
-    strings, rather than kept as lazy objects.
-
-    If strings_only is True, don't convert (some) non-string-like objects.
-    """
-    # Handle the common case first for performance reasons.
-    if isinstance(s, six.text_type):
-        return s
-    if not isinstance(s, six.string_types):
-        if six.PY3:
-            if isinstance(s, bytes):
-                s = six.text_type(s, encoding, errors)
-            else:
-                s = six.text_type(s)
-        elif hasattr(s, "__unicode__"):
-            s = six.text_type(s)
+
+@force_text.dispatch(six.text_type)
+def encode(s, encoding="utf-8", errors="strict"):
+    return s
+
+
+@force_text.dispatch(six.binary_type, replace_existing=True)
+def encode(s, encoding="utf-8", errors="strict"):
+    return s.decode(encoding, errors)
+
+
+if not six.PY2:
+
+    @force_text.dispatch(object)
+    def encode(s, encoding="utf-8", errors="strict"):
+        return six.text_type(s)
+
+
+else:
+
+    @force_text.dispatch(object)
+    def encode(s, encoding="utf-8", errors="strict"):
+        if hasattr(s, "__unicode__"):
+            return six.text_type(s)
         else:
-            s = six.text_type(bytes(s), encoding, errors)
-    else:
-        # Note: We use .decode() here, instead of six.text_type(s, encoding,
-        # errors), so that if s is a SafeBytes, it ends up being a
-        # SafeText at the end.
-        s = s.decode(encoding, errors)
+            return six.text_type(bytes(s), encoding, errors)
+
+
+force_bytes = Dispatch()
+
+
+@force_bytes.dispatch(six.string_types)
+def encode(s, encoding="utf-8", errors="strict"):
+    return s.encode(encoding, errors)
+
+
+@force_bytes.dispatch(six.binary_type, replace_existing=True)
+def encode(s, encoding="utf-8", errors="strict"):
     return s
 
 
-def force_bytes(s, encoding="utf-8", errors="strict"):
-    """
-    If strings_only is True, don't convert (some) non-string-like objects.
-    """
-    # Handle the common case first for performance reasons.
-    if isinstance(s, bytes):
-        return s
-
-    if isinstance(s, six.buffer_types):
-        return bytes(s)
-
-    if not isinstance(s, six.string_types):
-        try:
-            if six.PY3:
-                return six.text_type(s).encode(encoding)
-            else:
-                return bytes(s)
-        except UnicodeEncodeError:
-            if isinstance(s, Exception):
-                # An Exception subclass containing non-ASCII data that doesn't
-                # know how to print itself properly. We shouldn't raise a
-                # further exception.
-                return b" ".join(force_bytes(arg, encoding, errors=errors) for arg in s)
-            return six.text_type(s).encode(encoding, errors)
-    else:
-        return s.encode(encoding, errors)
+@force_bytes.dispatch(six.buffer_types, replace_existing=True)
+def encode(s, encoding="utf-8", errors="strict"):
+    return six.binary_type(s)
+
+
+if six.PY2:
+
+    @force_bytes.dispatch(memoryview, replace_existing=True)
+    def encode(s, encoding="utf-8", errors="strict"):
+        return s.tobytes()
+
+
+if not six.PY2:
+
+    def encode_default(s, encoding="utf-8", errors="strict"):
+        return six.text_type(s).encode(encoding)
+
+
+else:
+
+    def encode_default(s, encoding="utf-8", errors="strict"):
+        return six.binary_type(s)
+
+
+@force_bytes.dispatch(object)
+def encode(s, encoding="utf-8", errors="strict"):
+    try:
+        return encode_default(s, encoding, errors)
+    except UnicodeEncodeError:
+        if isinstance(s, Exception):
+            # An Exception subclass containing non-ASCII data that doesn't
+            # know how to print itself properly. We shouldn't raise a
+            # further exception.
+            return b" ".join(force_bytes(arg, encoding, errors=errors) for arg in s)
+        return six.text_type(s).encode(encoding, errors)
 
 
 def safe_force_text(obj):

wolframclient/utils/six.py

@@ -34,7 +34,6 @@
     StringIO = io.StringIO
     BytesIO = io.BytesIO
 
-    memoryview = memoryview
     buffer_types = (bytes, bytearray, memoryview)
 
 else:
@@ -49,13 +48,6 @@
 
     StringIO = BytesIO = StringIO.StringIO
 
-    # memoryview and buffer are not strictly equivalent, but should be fine for
-    # django core usage (mainly BinaryField). However, Jython doesn't support
-    # buffer (see http://bugs.jython.org/issue1521), so we have to be careful.
-    if JYTHON:
-        memoryview = memoryview
-    else:
-        memoryview = buffer
     buffer_types = (bytearray, memoryview, buffer)
 
 iterable_types = [