Merge pull request #238 in LCL/wolframclientforpython from feature/opt to master

* commit '9f9d5e8e3c4b8f5888ac168f0c97e33942f8ac38':
  adding binary_deserialize load
  PackedArray must do the same
  _valid_type_or_fail now returns type
  code refactor
  using force_bytes for py2 compatibility
  using pack ability to serialiaze multiple values at once
  code refactor
  NumericArray is supposed to be initialized by a list of python numbers
  adding PackedArray benchmarks
  avoiding attribute lookup for every single element in the array
  speed optimizations in wxf serializer and zlib encoder

Author: crazyeng

wolframclient/cli/commands/benchmark.py

@@ -7,6 +7,7 @@
 from wolframclient.cli.utils import SimpleCommand
 from wolframclient.deserializers import binary_deserialize
 from wolframclient.language import wl
+from wolframclient.language.array import NumericArray, PackedArray
 from wolframclient.serializers import export
 from wolframclient.utils.debug import timed
 from wolframclient.utils.encoding import force_text
@@ -21,23 +22,31 @@ def repeat(el, n=1):
 class Command(SimpleCommand):
 
     col_size = 8
+    title_size = 14
     repetitions = 10
     complexity = [1, 2, 5, 10, 100, 1000]
 
     def add_arguments(self, parser):
         parser.add_argument("--profile", dest="profile", default=False, action="store_true")
 
-    def complexity_handler(self, complexity):
+    def expression_handler(self, complexity):
         return {
-            "symbols": repeat(wl.Symbol, complexity),
-            "strings": repeat("string", complexity),
-            "bytes": repeat(b"bytes", complexity),
-            "integers": repeat(1, complexity),
-            "decimals": repeat(decimal.Decimal("1.23"), complexity),
-            "floats": repeat(1.23, complexity),
-            "dict": repeat({1: 2, 3: 4, 5: 6}, complexity),
-            "list": repeat([1, 2, 3], complexity),
-            "functions": repeat(wl.Function(1, 2, 3), complexity),
+            "expr": {
+                "symbols": repeat(wl.Symbol, complexity),
+                "strings": repeat("string", complexity),
+                "bytes": repeat(b"bytes", complexity),
+                "integers": repeat(1, complexity),
+                "decimals": repeat(decimal.Decimal("1.23"), complexity),
+                "floats": repeat(1.23, complexity),
+                "dict": repeat({1: 2, 3: 4, 5: 6}, complexity),
+                "list": repeat([1, 2, 3], complexity),
+                "functions": repeat(wl.Function(1, 2, 3), complexity),
+            },
+            "array": {
+                "%s_%s" % (func.__name__, t): func(tuple(range(complexity * 100)), t)
+                for func in (PackedArray, NumericArray)
+                for t in ("Integer64", "Real64")
+            },
         }
 
     def formatted_time(self, function, *args, **opts):
@@ -47,10 +56,15 @@ def formatted_time(self, function, *args, **opts):
         return "%.5f" % (time / self.repetitions)
 
     def table_line(self, *iterable):
-        self.print(*(force_text(c).ljust(self.col_size) for c in iterable))
+        self.print(
+            *(
+                force_text(c).ljust(i and self.col_size or self.title_size)
+                for i, c in enumerate(iterable)
+            )
+        )
 
     def table_divider(self, length):
-        self.print(*("-" * self.col_size for i in range(length)))
+        self.print(*("-" * (i and self.col_size or self.title_size) for i in range(length)))
 
     def stream_generators(self, path):
         yield "Memory", lambda complexity, export_format, path=path: None
@@ -62,13 +76,13 @@ def report(self):
 
         path = tempfile.gettempdir()
 
-        benchmarks = [(c, self.complexity_handler(c)) for c in self.complexity]
+        benchmarks = [(c, self.expression_handler(c)) for c in self.complexity]
 
         self.table_line("dumping results in %s" % path)
         self.table_line()
 
         # running export to do all lazy loadings
-        export(1)
+        binary_deserialize(export(1, target_format = 'wxf'))
 
         self.table_line("* Binary deserialize")
         self.table_line()
@@ -102,24 +116,26 @@ def report(self):
             )
             self.table_divider(len(self.complexity) + 1)
 
-            for label, export_format, opts in (
-                ("wl", "wl", dict()),
-                ("wxf", "wxf", dict()),
-                ("wxf zip", "wxf", dict(compress=True)),
-            ):
-                self.table_line(
-                    label,
-                    *(
-                        self.formatted_time(
-                            export,
-                            expr,
-                            stream=stream_generator(complexity, export_format),
-                            target_format=export_format,
-                            **opts
+            for key in ("expr", "array"):
+                for label, export_format, opts in (
+                    ("wl", "wl", dict()),
+                    ("wxf", "wxf", dict()),
+                    ("wxf zip", "wxf", dict(compress=True)),
+                ):
+                    if key == "expr" or (key == "array" and not label == "wl"):
+                        self.table_line(
+                            key == "expr" and label or "%s %s" % (label, key),
+                            *(
+                                self.formatted_time(
+                                    export,
+                                    expr[key],
+                                    stream=stream_generator(complexity, export_format),
+                                    target_format=export_format,
+                                    **opts
+                                )
+                                for complexity, expr in benchmarks
+                            )
                         )
-                        for complexity, expr in benchmarks
-                    )
-                )
 
             self.table_line()
 

wolframclient/language/array.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
+import struct
+
 from wolframclient.exception import WolframLanguageException
 from wolframclient.serializers.wxfencoder import constants
-from wolframclient.utils.encoding import concatenate_bytes
+from wolframclient.utils.encoding import force_bytes
 
 try:
     from collections.abc import Sequence
@@ -15,19 +17,21 @@ def __init__(self, array, type, shape=None):
 
         self.array = array
         self.shape = shape or (len(array),)
-        self.type = type
-        self._valid_type_or_fail(type)
-        self.struct = constants.STRUCT_MAPPING[type]
+        self.type = self._valid_type_or_fail(type)
+        self.struct = constants.STRUCT_MAPPING[self.type]
 
     def _valid_type_or_fail(self, type):
         if type not in constants.STRUCT_MAPPING:
             raise WolframLanguageException(
                 "Type %s is not one of the supported array types: %s."
                 % (type, ", ".join(constants.STRUCT_MAPPING.keys()))
             )
+        return type
 
     def tobytes(self):
-        return concatenate_bytes(self.struct.pack(el) for el in self.array)
+        return struct.pack(
+            b"<%i%s" % (len(self), force_bytes(self.struct.format[1])), *self.array
+        )
 
     def __getitem__(self, k):
         return self.array[k]
@@ -43,3 +47,4 @@ def _valid_type_or_fail(self, type):
                 "Type %s is not one of the supported packed array types: %s."
                 % (type, ", ".join(sorted(constants.VALID_PACKED_ARRAY_LABEL_TYPES)))
             )
+        return type

wolframclient/serializers/wl.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
-from itertools import chain
+from itertools import chain, starmap
 
 from wolframclient.serializers.base import FormatSerializer
 from wolframclient.serializers.utils import py_encode_decimal, py_encode_text
@@ -9,16 +9,14 @@
 from wolframclient.utils.encoding import force_bytes, force_text
 
 
-def yield_with_separators(iterable, separator=b", ", first=None, last=None):
-    if first:
-        yield first
+def yield_with_separators(iterable, first, last, separator=b", "):
+    yield first
     for i, arg in enumerate(iterable):
         if i:
             yield separator
         for sub in arg:
             yield sub
-    if last:
-        yield last
+    yield last
 
 
 class WLSerializer(FormatSerializer):
@@ -60,16 +58,14 @@ def serialize_int(self, number):
         yield b"%i" % number
 
     def serialize_rule(self, lhs, rhs):
-        return yield_with_separators((lhs, rhs), separator=b" -> ")
+        return chain(lhs, (b" -> ",), rhs)
 
     def serialize_rule_delayed(self, lhs, rhs):
-        return yield_with_separators((lhs, rhs), separator=b" :> ")
+        return chain(lhs, (b" :> ",), rhs)
 
     def serialize_mapping(self, mapping, **opts):
         return yield_with_separators(
-            (self.serialize_rule(key, value) for key, value in mapping),
-            first=b"<|",
-            last=b"|>",
+            starmap(self.serialize_rule, mapping), first=b"<|", last=b"|>"
         )
 
     def serialize_association(self, mapping, **opts):

wolframclient/serializers/wxf.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
-from itertools import chain
+from itertools import chain, starmap
 
 from wolframclient.serializers.base import FormatSerializer
 from wolframclient.serializers.utils import py_encode_decimal, safe_len
@@ -20,7 +20,12 @@
 )
 from wolframclient.utils import six
 from wolframclient.utils.api import zlib
-from wolframclient.utils.encoding import force_bytes, force_text
+from wolframclient.utils.encoding import concatenate_bytes, force_bytes, force_text
+from wolframclient.utils.functional import partition
+
+
+def serialize_rule(key, value, sep=(WXF_CONSTANTS.Rule,)):
+    return chain(sep, key, value)
 
 
 def get_length(iterable, length=None):
@@ -37,6 +42,16 @@ def get_length(iterable, length=None):
     return iterable, len(iterable)
 
 
+def compress(data):
+
+    compressor = zlib.compressobj()
+
+    for token in map(compressor.compress, map(concatenate_bytes, partition(data, 100))):
+        yield token
+
+    yield compressor.flush()
+
+
 class WXFSerializer(FormatSerializer):
     """ Serialize python objects to WXF. """
 
@@ -46,25 +61,14 @@ def __init__(self, normalizer=None, compress=False, **opts):
 
     def generate_bytes(self, data):
 
-        yield WXF_VERSION
-
         if self.compress:
-            yield WXF_HEADER_COMPRESS
 
-        yield WXF_HEADER_SEPARATOR
+            return chain(
+                (WXF_VERSION, WXF_HEADER_COMPRESS, WXF_HEADER_SEPARATOR),
+                compress(self.encode(data)),
+            )
 
-        if self.compress:
-            compressor = zlib.compressobj()
-            if six.PY2:
-                for payload in self.encode(data):
-                    yield compressor.compress(six.binary_type(payload))
-            else:
-                for payload in self.encode(data):
-                    yield compressor.compress(payload)
-            yield compressor.flush()
-        else:
-            for payload in self.encode(data):
-                yield payload
+        return chain((WXF_VERSION, WXF_HEADER_SEPARATOR), self.encode(data))
 
     def serialize_symbol(self, name):
         yield WXF_CONSTANTS.Symbol
@@ -116,12 +120,9 @@ def serialize_string(self, string):
 
     def serialize_bytes(self, bytes, as_byte_array=not six.PY2):
         if as_byte_array:
-            yield WXF_CONSTANTS.BinaryString
-            yield varint_bytes(len(bytes))
-            yield bytes
+            return (WXF_CONSTANTS.BinaryString, varint_bytes(len(bytes)), bytes)
         else:
-            for token in self.serialize_string(force_text(bytes, encoding="iso8859-1")):
-                yield token
+            return self.serialize_string(force_text(bytes, encoding="iso8859-1"))
 
     def serialize_mapping(self, keyvalue, **opts):
         # the normalizer is always sending an generator key, value
@@ -130,9 +131,7 @@ def serialize_mapping(self, keyvalue, **opts):
 
         return chain(
             (WXF_CONSTANTS.Association, varint_bytes(length)),
-            chain.from_iterable(
-                chain((WXF_CONSTANTS.Rule,), key, value) for key, value in iterable
-            ),
+            chain.from_iterable(starmap(serialize_rule, iterable)),
         )
 
     def serialize_numeric_array(self, data, dimensions, wl_type):