Merge pull request #236 in LCL/wolframclientforpython from feature/python-array to release/B12_1_1

* commit '10b6b4619d995b2510ec4bc86c4145d97cad9ab0': (21 commits)
  refactor
  Adding error message when PA type is invalid.
  refactor
  Add error message for invalid array type
  verbosity = 0 is not overriden anymore
  Set default verbosity in test if not set
  adding true false test
  adding packed array support
  renaming to numeric array
  using last
  testing shape
  working implementation of python array
  moving python array
  code refactor
  adding STRUCT_MAPPING constant
  adding python array class and test suite
  remove get
  adding ast api
  removing unused protected_types
  removing extra function
  ...

Author: Raj Balaebail

wolframclient/cli/commands/test.py

@@ -70,6 +70,9 @@ def handle(self, *args, **opts):
 
         # verbosity > 1 print test name
         verbosity = opts.get("verbosity")
+        # for consistency with parser, set default to 2. This is only possible when calling test from setup.py
+        if verbosity is None:
+            verbosity = 2
         xml_path = opts.get("xml_output_dir")
         # if opts.get('produce_xml'):
         if xml_path is not None or opts.get("produce_xml"):

wolframclient/deserializers/wxf/wxfparser.py

@@ -141,27 +141,27 @@ def token_for_string(self, token):
 
     def token_for_integer8(self, token):
         self.context.add_part()
-        token.data = constants.StructInt8LE.unpack(self.reader.read(1))[0]
+        token.data = constants.STRUCT_MAPPING.Integer8.unpack(self.reader.read(1))[0]
         return token
 
     def token_for_integer16(self, token):
         self.context.add_part()
-        token.data = constants.StructInt16LE.unpack(self.reader.read(2))[0]
+        token.data = constants.STRUCT_MAPPING.Integer16.unpack(self.reader.read(2))[0]
         return token
 
     def token_for_integer32(self, token):
         self.context.add_part()
-        token.data = constants.StructInt32LE.unpack(self.reader.read(4))[0]
+        token.data = constants.STRUCT_MAPPING.Integer32.unpack(self.reader.read(4))[0]
         return token
 
     def token_for_integer64(self, token):
         self.context.add_part()
-        token.data = constants.StructInt64LE.unpack(self.reader.read(8))[0]
+        token.data = constants.STRUCT_MAPPING.Integer64.unpack(self.reader.read(8))[0]
         return token
 
     def token_for_real64(self, token):
         self.context.add_part()
-        token.data = constants.StructDouble.unpack(self.reader.read(8))[0]
+        token.data = constants.STRUCT_MAPPING.Real64.unpack(self.reader.read(8))[0]
         return token
 
     def token_for_function(self, token):

wolframclient/language/array.py

@@ -0,0 +1,45 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+from wolframclient.exception import WolframLanguageException
+from wolframclient.serializers.wxfencoder import constants
+from wolframclient.utils.encoding import concatenate_bytes
+
+try:
+    from collections.abc import Sequence
+except ImportError:
+    from collections import Sequence
+
+
+class NumericArray(Sequence):
+    def __init__(self, array, type, shape=None):
+
+        self.array = array
+        self.shape = shape or (len(array),)
+        self.type = type
+        self._valid_type_or_fail(type)
+        self.struct = constants.STRUCT_MAPPING[type]
+
+    def _valid_type_or_fail(self, type):
+        if type not in constants.STRUCT_MAPPING:
+            raise WolframLanguageException(
+                "Type %s is not one of the supported array types: %s."
+                % (type, ", ".join(constants.STRUCT_MAPPING.keys()))
+            )
+
+    def tobytes(self):
+        return concatenate_bytes(self.struct.pack(el) for el in self.array)
+
+    def __getitem__(self, k):
+        return self.array[k]
+
+    def __len__(self):
+        return len(self.array)
+
+
+class PackedArray(NumericArray):
+    def _valid_type_or_fail(self, type):
+        if type not in constants.VALID_PACKED_ARRAY_LABEL_TYPES:
+            raise WolframLanguageException(
+                "Type %s is not one of the supported packed array types: %s."
+                % (type, ", ".join(constants.VALID_PACKED_ARRAY_LABEL_TYPES_TUPLE))
+            )

wolframclient/serializers/encoders/builtin.py

@@ -2,6 +2,7 @@
 
 import math
 
+from wolframclient.language.array import NumericArray, PackedArray
 from wolframclient.language.expression import WLFunction, WLInputExpression, WLSymbol
 from wolframclient.serializers.serializable import WLSerializable
 from wolframclient.serializers.utils import safe_len
@@ -173,3 +174,13 @@ def encode_serializable(serializer, o):
 @encoder.dispatch(Association)
 def encode_association(serializer, o):
     return _to_key_value(serializer.serialize_association, serializer, o)
+
+
+@encoder.dispatch(NumericArray)
+def encode_array(serializer, o):
+    return serializer.serialize_numeric_array(o.tobytes(), o.shape, o.type)
+
+
+@encoder.dispatch(PackedArray)
+def encode_array(serializer, o):
+    return serializer.serialize_packed_array(o.tobytes(), o.shape, o.type)

wolframclient/serializers/wxfencoder/constants.py

@@ -91,13 +91,29 @@ def _bytes(value):
     )
 )
 
-StructInt8LE = struct.Struct(b"<b")
-StructUInt8LE = struct.Struct(b"<B")
-StructInt16LE = struct.Struct(b"<h")
-StructUInt16LE = struct.Struct(b"<H")
-StructInt32LE = struct.Struct(b"<i")
-StructUInt32LE = struct.Struct(b"<I")
-StructInt64LE = struct.Struct(b"<q")
-StructUInt64LE = struct.Struct(b"<Q")
-StructFloat = struct.Struct(b"<f")
-StructDouble = struct.Struct(b"<d")
+VALID_PACKED_ARRAY_LABEL_TYPES_TUPLE = (
+    "Integer8",
+    "Integer16",
+    "Integer32",
+    "Integer64",
+    "Real32",
+    "Real64",
+    "ComplexReal32",
+    "ComplexReal64",
+)
+VALID_PACKED_ARRAY_LABEL_TYPES = frozenset(VALID_PACKED_ARRAY_LABEL_TYPES_TUPLE)
+
+STRUCT_MAPPING = Settings(
+    Integer8=struct.Struct(b"<b"),
+    UnsignedInteger8=struct.Struct(b"<B"),
+    Integer16=struct.Struct(b"<h"),
+    UnsignedInteger16=struct.Struct(b"<H"),
+    Integer32=struct.Struct(b"<i"),
+    UnsignedInteger32=struct.Struct(b"<I"),
+    Integer64=struct.Struct(b"<q"),
+    UnsignedInteger64=struct.Struct(b"<Q"),
+    Real32=struct.Struct(b"<f"),
+    Real64=struct.Struct(b"<d"),
+    ComplexReal32=struct.Struct(b"<f"),
+    ComplexReal64=struct.Struct(b"<d"),
+)

wolframclient/serializers/wxfencoder/utils.py

@@ -3,21 +3,13 @@
 from wolframclient.exception import WolframLanguageException
 from wolframclient.serializers.wxfencoder.constants import (
     ARRAY_TYPES,
+    STRUCT_MAPPING,
     VALID_PACKED_ARRAY_TYPES,
     WXF_CONSTANTS,
-    StructDouble,
-    StructFloat,
-    StructInt8LE,
-    StructInt16LE,
-    StructInt32LE,
-    StructInt64LE,
-    StructUInt8LE,
-    StructUInt16LE,
-    StructUInt32LE,
-    StructUInt64LE,
 )
 from wolframclient.utils import six
 from wolframclient.utils.datastructures import Settings
+from wolframclient.utils.functional import last
 
 if six.JYTHON:
     import jarray
@@ -71,21 +63,26 @@ def integer_size(value):
         raise ValueError("Value %i is not a machine-sized integer." % value)
 
 
-_packing = {1: StructInt8LE, 2: StructInt16LE, 4: StructInt32LE, 8: StructInt64LE}
+_packing = {
+    1: STRUCT_MAPPING.Integer8,
+    2: STRUCT_MAPPING.Integer16,
+    4: STRUCT_MAPPING.Integer32,
+    8: STRUCT_MAPPING.Integer64,
+}
 
 if six.JYTHON:
 
     def integer_to_bytes(value, int_size):
         buffer = jarray.zeros(8, "c")
-        _packing.get(int_size).pack_into(buffer, 0, value)
+        _packing[int_size].pack_into(buffer, 0, value)
         return buffer[:int_size].tostring()
 
 
 elif six.PY2:
 
     def integer_to_bytes(value, int_size):
         buffer = bytearray(8)
-        _packing.get(int_size).pack_into(buffer, 0, value)
+        _packing[int_size].pack_into(buffer, 0, value)
         return buffer[:int_size]
 
 
@@ -97,17 +94,17 @@ def integer_to_bytes(value, int_size):
 
 if six.JYTHON:
 
-    def float_to_bytes(value):
+    def float_to_bytes(value, pack_into=STRUCT_MAPPING.Real64.pack_into):
         buffer = jarray.zeros(8, "c")
-        StructDouble.pack_into(buffer, 0, value)
+        pack_into(buffer, 0, value)
         return buffer.tostring()
 
 
 else:
 
-    def float_to_bytes(value):
+    def float_to_bytes(value, pack_into=STRUCT_MAPPING.Real64.pack_into):
         buffer = bytearray(8)
-        StructDouble.pack_into(buffer, 0, value)
+        pack_into(buffer, 0, value)
         return buffer
 
 
@@ -146,20 +143,7 @@ def array_to_list(data, dimensions, wl_type):
 
 
 if hasattr(memoryview, "cast"):
-    unpack_mapping = Settings(
-        Integer8="b",
-        UnsignedInteger8="B",
-        Integer16="h",
-        UnsignedInteger16="H",
-        Integer32="i",
-        UnsignedInteger32="I",
-        Integer64="q",
-        UnsignedInteger64="Q",
-        Real32="f",
-        Real64="d",
-        ComplexReal32="f",
-        ComplexReal64="d",
-    )
+    unpack_mapping = Settings((k, last(v.format)) for k, v in STRUCT_MAPPING.items())
 
     def _to_complex(array, max_depth, curr_depth):
         # recursivelly traverse the array until the last (real) dimension is reached
@@ -188,20 +172,6 @@ def _array_to_list(data, shape, array_type):
 
 
 else:
-    unpack_mapping = Settings(
-        Integer8=StructInt8LE,
-        UnsignedInteger8=StructUInt8LE,
-        Integer16=StructInt16LE,
-        UnsignedInteger16=StructUInt16LE,
-        Integer32=StructInt32LE,
-        UnsignedInteger32=StructUInt32LE,
-        Integer64=StructInt64LE,
-        UnsignedInteger64=StructUInt64LE,
-        Real32=StructFloat,
-        Real64=StructDouble,
-        ComplexReal32=StructFloat,
-        ComplexReal64=StructDouble,
-    )
 
     def _array_to_list(data, shape, array_type):
         value, _ = _build_array_from_bytes(data, 0, array_type, shape, 0)
@@ -216,7 +186,7 @@ def _build_array_from_bytes(data, offset, array_type, dimensions, current_dim):
                 )
                 new_array.append(new_elem)
         else:
-            struct = unpack_mapping[array_type]
+            struct = STRUCT_MAPPING[array_type]
             # complex values, need two reals for each.
             if array_type == "ComplexReal32" or array_type == "ComplexReal64":
                 for i in range(dimensions[-1]):

wolframclient/tests/serializers/wl_serialization.py

@@ -92,7 +92,9 @@ def test_datetime(self):
 
     def test_date(self):
 
-        self.compare(test_datetime().date(), b'DateObject[{2000, 1, 1}, "Day", "Gregorian", None]')
+        self.compare(
+            test_datetime().date(), b'DateObject[{2000, 1, 1}, "Day", "Gregorian", None]'
+        )
 
     def test_timedelta(self):
 

wolframclient/tests/serializers/wxf_pythonarray.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from wolframclient.language.array import NumericArray
+from wolframclient.serializers import export
+from wolframclient.utils.api import numpy
+from wolframclient.utils.tests import TestCase as BaseTestCase
+
+
+class TestCase(BaseTestCase):
+    def test_python_array(self):
+
+        for array, numpy_type, wl_type in (
+            ([True, False, True, False, True, False], numpy.int8, "Integer8"),
+            ([1, 2, 3, 4, 5, 6], numpy.int8, "Integer8"),
+            ([1, 2, 3, 4, 5, 6], numpy.int32, "Integer32"),
+            ([1, 2, 3, 4, 5, 6], numpy.int64, "Integer64"),
+            ([1.2, 2.3, 3, 4, 5, 6], numpy.float32, "Real32"),
+            ([1.2, 2.3, 3, 4, 5, 6], numpy.float64, "Real64"),
+        ):
+
+            for shape in ((3, 2), None):
+
+                arr = numpy.array(array, numpy_type)
+                if shape:
+                    arr = arr.reshape(shape)
+
+                self.assertEqual(
+                    export(arr, target_format="wxf"),
+                    export(NumericArray(array, wl_type, shape=shape), target_format="wxf"),
+                )

wolframclient/utils/api.py

@@ -3,12 +3,12 @@
 from wolframclient.utils.importutils import API
 
 ast = API(
-    Module = 'ast.Module',
-    PyCF_ONLY_AST = 'ast.PyCF_ONLY_AST',
-    Expr = 'ast.Expr',
-    Expression = 'ast.Expression',
-    FunctionDef = 'ast.FunctionDef',
-    ClassDef = 'ast.ClassDef',
+    Module="ast.Module",
+    PyCF_ONLY_AST="ast.PyCF_ONLY_AST",
+    Expr="ast.Expr",
+    Expression="ast.Expression",
+    FunctionDef="ast.FunctionDef",
+    ClassDef="ast.ClassDef",
 )
 
 pytz = API(

wolframclient/utils/externalevaluate.py

@@ -33,7 +33,7 @@
     # https://github.com/ipython/ipython/issues/11590
     # PY_38 requires type_ignores to be a list, other versions are not accepting a second argument
 
-    def Module(code, type_ignores = []):
+    def Module(code, type_ignores=[]):
         return ast.Module(code, type_ignores)