py3 compat

Author: mr-c

Makefile

@@ -69,7 +69,7 @@ clean_sdist:
 
 .PHONY: test
 test: _check_venv
-	$(python) setup.py test --pytest-args "-vv --assert=plain $(tests)"
+	$(python) setup.py test --addopts "-vv --assert=plain --continue-on-collection-errors $(tests)"
 
 
 .PHONY: pypi

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+# Look for any python file, the default of test_*.py wouldn't work for us
+python_files=*.py
+# Run doctests and start test collection in the src dir
+addopts = --doctest-modules

setup.cfg

@@ -1,5 +1,2 @@
-[pytest]
-# Look for any python file, the default of test_*.py wouldn't work for us
-python_files=*.py
-# Run doctests and start test collection in the src dir
-addopts = --doctest-modules
+[aliases]
+test=pytest

setup.py

@@ -16,36 +16,12 @@
     package_dir={ '': 'src' },
     packages=find_packages( 'src' ),
     install_requires=[ 'future' ],
+    setup_requires=['pytest-runner'],
     tests_require=[
-        'pytest==2.7.2',
+        'pytest==3.5.0',
         'mock==1.0.1',
         'lockfile==0.11.0',
         'boto==2.38.0'],
     namespace_packages=[ 'bd2k' ] )
 
-from setuptools.command.test import test as TestCommand
-
-
-class PyTest( TestCommand ):
-    user_options = [ ('pytest-args=', 'a', "Arguments to pass to py.test") ]
-
-    def initialize_options( self ):
-        TestCommand.initialize_options( self )
-        self.pytest_args = [ ]
-
-    def finalize_options( self ):
-        TestCommand.finalize_options( self )
-        self.test_args = [ ]
-        self.test_suite = True
-
-    def run_tests( self ):
-        import pytest
-        # Sanitize command line arguments to avoid confusing Toil code attempting to parse them
-        sys.argv[ 1: ] = [ ]
-        errno = pytest.main( self.pytest_args )
-        sys.exit( errno )
-
-
-kwargs[ 'cmdclass' ] = { 'test': PyTest }
-
 setup( **kwargs )

src/bd2k/util/__init__.py

@@ -109,7 +109,8 @@ def ilen( it ):
     """
     Return the number of elements in an iterable
 
-    >>> ilen(xrange(0,100))
+    >>> from builtins import range
+    >>> ilen(range(0,100))
     100
     """
     return sum( 1 for _ in it )

src/bd2k/util/collections.py

@@ -152,7 +152,8 @@ def rindex( l, v ):
     2
     >>> rindex( (0,1,0,1), 0 )
     2
-    >>> rindex( xrange(3), 2 )
+    >>> from builtins import range
+    >>> rindex( range(3), 2 )
     2
     """
     try:

src/bd2k/util/d32.py

@@ -17,7 +17,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 # Inspired by Dominic Tarr's JavaScript at https://github.com/dominictarr/d64
-
+import codecs
+import sys
 from builtins import range
 from builtins import object
 from past.utils import old_div
@@ -28,23 +29,23 @@ class D32( object ):
 
     def __init__( self, alphabet ):
         super( D32, self ).__init__( )
-        self.alphabet = bytearray( alphabet )
+        self.alphabet = bytearray( alphabet.encode('utf-8') )
         self.lookup = bytearray( 255 )
         for i in range( 32 ):
             self.lookup[ self.alphabet[ i ] ] = i
 
     def encode( self, d ):
         """
         >>> encode = standard.encode
-        >>> encode('')
+        >>> encode(b'')  # doctest: +ALLOW_UNICODE
         ''
-        >>> encode('\\0')
+        >>> encode(b'\\0')  # doctest: +ALLOW_UNICODE
         '22'
-        >>> encode('\\xff')
+        >>> encode(b'\\xff')  # doctest: +ALLOW_UNICODE
         'zw'
-        >>> encode('\\0\\1\\2\\3\\4')
+        >>> encode(b'\\0\\1\\2\\3\\4')  # doctest: +ALLOW_UNICODE
         '222k62s6'
-        >>> encode('\\0\\1\\2\\3\\4\\5')
+        >>> encode(b'\\0\\1\\2\\3\\4\\5')  # doctest: +ALLOW_UNICODE
         '222k62s62o'
         """
         m = len( d )
@@ -56,7 +57,7 @@ def encode( self, d ):
 
         while i < m:
             if m - i < 5:
-                g = bytearray( d[ i: ] + '\0' * (5 - (m - i)) )
+                g = bytearray( d[ i: ] + b'\0' * (5 - (m - i)))
             else:
                 g = bytearray( d[ i:i + 5 ] )
             # bit              1          2          3
@@ -73,17 +74,18 @@ def encode( self, d ):
             e[ j + 7 ] = a[ g[ 4 ] & 31 ]
             j += 8
             i += 5
-        return str( e[ :-padding ] )
+        return codecs.decode( e[ :-padding ], 'ASCII' )
 
     def decode( self, e ):
         """
+        >>> import codecs
         >>> decode = standard.decode
 
         # >>> decode('222k62s62o')
         # '\\x00\\x01\\x02\\x03\\x04\\x05'
         # >>> decode('222k62s6')
         # '\\x00\\x01\\x02\\x03\\x04'
-        >>> decode('zw')
+        >>> codecs.decode(decode('zw'), 'unicode-escape')  # # doctest: +ALLOW_UNICODE
         '\\xff'
         """
         n = len( e )
@@ -108,7 +110,8 @@ def decode( self, e ):
             d[ i + 4 ] = g[ 6 ] << 5 & 255 | g[ 7 ]
             j += 8
             i += 5
-        return str( d[ :-padding ] )
+
+        return bytes(d[ :-padding ])
 
 
 # A variant of Base64 that maintains the lexicographical ordering such that for any given list of

src/bd2k/util/d64.py

@@ -19,16 +19,17 @@
 
 # Ported from JS found at https://github.com/dominictarr/d64
 
-
-
+import codecs
+from builtins import bytes
 from builtins import range
 from builtins import object
 from past.utils import old_div
+
 class D64( object ):
     def __init__( self, special_chars ):
         super( D64, self ).__init__( )
         alphabet = 'PYFGCRLAOEUIDHTNSQJKXBMWVZpyfgcrlaoeuidhtnsqjkxbmwvz1234567890'
-        self.alphabet = bytearray( sorted( alphabet + special_chars ) )
+        self.alphabet = bytearray( str(''.join(sorted( alphabet + special_chars))).encode( 'utf-8' ))
         self.lookup = bytearray( 255 )
         for i in range( 64 ):
             code = self.alphabet[ i ]
@@ -37,24 +38,25 @@ def __init__( self, special_chars ):
     def encode( self, data ):
         """
         >>> encode = standard.encode
-        >>> encode('')
+        >>> encode(b'')  # doctest: +ALLOW_UNICODE
         ''
-        >>> encode('\\x00')
+        >>> encode(b'\\x00')  # doctest: +ALLOW_UNICODE
         '..'
-        >>> encode('\\x00\\x01')
+        >>> encode(b'\\x00\\x01')  # doctest: +ALLOW_UNICODE
         '..3'
-        >>> encode('\\x00\\x01\\x02')
+        >>> encode(b'\\x00\\x01\\x02')  # doctest: +ALLOW_UNICODE
         '..31'
-        >>> encode('\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07')
+        >>> encode(b'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07')  # doctest: +ALLOW_UNICODE
         '..31.kF40VR'
         """
+        data = bytes( data )
         l = len( data )
         s = bytearray( old_div((l * 4 + 2), 3) )
         hang = 0
         j = 0
         a = self.alphabet
         for i in range( l ):
-            v = ord( data[ i ] )
+            v = data[ i ]
             r = i % 3
             if r == 0:
                 s[ j ] = a[ v >> 2 ]
@@ -75,20 +77,21 @@ def encode( self, data ):
         if l % 3:
             s[ j ] = a[ hang ]
 
-        return str( s )
+        return codecs.decode( s )
 
     def decode( self, e ):
         """
+        >>> import codecs
         >>> decode = standard.decode
-        >>> decode('')
+        >>> codecs.decode(decode(''), 'unicode-escape') # doctest: +ALLOW_UNICODE
         ''
-        >>> decode('..')
+        >>> codecs.decode(decode('..'), 'unicode-escape') # doctest: +ALLOW_UNICODE
         '\\x00'
-        >>> decode('..3')
+        >>> codecs.decode(decode('..3'), 'unicode-escape') # doctest: +ALLOW_UNICODE
         '\\x00\\x01'
-        >>> decode('..31')
+        >>> codecs.decode(decode('..31'), 'unicode-escape') # doctest: +ALLOW_UNICODE
         '\\x00\\x01\\x02'
-        >>> decode('..31.kF40VR')
+        >>> codecs.decode(decode('..31.kF40VR'), 'unicode-escape') # doctest: +ALLOW_UNICODE
         '\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07'
         """
         n = len( e )
@@ -115,7 +118,7 @@ def decode( self, e ):
                 j += 1
             else:
                 assert False
-        return str( b )
+        return bytes(b)
 
 
 standard = D64( '._' )

src/bd2k/util/exceptions.py

@@ -1,3 +1,4 @@
+from future.utils import raise_
 from builtins import object
 from contextlib import contextmanager
 import sys
@@ -35,7 +36,7 @@ def __exit__( self, *exc_info ):
         if self.log is not None and exc_info and exc_info[ 0 ]:
             self.log.warn( "Exception during panic", exc_info=exc_info )
         exc_type, exc_value, traceback = self.exc_info
-        raise exc_type, exc_value, traceback
+        raise_(exc_type, exc_value, traceback)
 
 
 class RequirementError( Exception ):
@@ -61,17 +62,17 @@ def require( value, message, *message_args ):
 
     >>> require(1 + 1 == 2, 'You made a terrible mistake')
 
-    >>> require(1 + 1 == 3, 'You made a terrible mistake')
+    >>> require(1 + 1 == 3, 'You made a terrible mistake')  # doctest: +IGNORE_EXCEPTION_DETAIL 
     Traceback (most recent call last):
     ...
     RequirementError: You made a terrible mistake
 
-    >>> require(1 + 1 == 3, 'You made a terrible mistake, %s', 'you fool')
+    >>> require(1 + 1 == 3, 'You made a terrible mistake, %s', 'you fool') # doctest: +IGNORE_EXCEPTION_DETAIL 
     Traceback (most recent call last):
     ...
     RequirementError: You made a terrible mistake, you fool
 
-    >>> require(1 + 1 == 3, 'You made a terrible mistake, %s %s', 'your', 'majesty')
+    >>> require(1 + 1 == 3, 'You made a terrible mistake, %s %s', 'your', 'majesty') # doctest: +IGNORE_EXCEPTION_DETAIL 
     Traceback (most recent call last):
     ...
     RequirementError: You made a terrible mistake, your majesty

src/bd2k/util/expando.py

@@ -24,13 +24,11 @@ class Expando(dict):
     >>> import json
     >>> s='{"foo":42}'
     >>> o = json.loads(s,object_hook=Expando)
-    >>> o
-    {u'foo': 42}
     >>> o.foo
     42
     >>> o.bar = 'hi'
-    >>> o
-    {u'foo': 42, 'bar': 'hi'}
+    >>> o.bar
+    'hi'
 
     And since Expando is a dict, it serializes back to JSON just fine: