Don't crash when checking the BOM on Unicode lines, with Python 2

Author: florentx

pep8.py

@@ -1018,8 +1018,6 @@ def readlines(filename):
             return f.readlines()
         finally:
             f.close()
-
-    BOM_UTF8 = '\xef\xbb\xbf'
     isidentifier = re.compile(r'[a-zA-Z_]\w*').match
     stdin_get_value = sys.stdin.read
 else:
@@ -1037,8 +1035,6 @@ def readlines(filename):
             return f.readlines()
         finally:
             f.close()
-
-    BOM_UTF8 = '\ufeff'
     isidentifier = str.isidentifier
 
     def stdin_get_value():
@@ -1204,8 +1200,13 @@ def __init__(self, filename=None, lines=None,
                 self.lines = []
         else:
             self.lines = lines
-        if self.lines and self.lines[0].startswith(BOM_UTF8):
-            self.lines[0] = self.lines[0][len(BOM_UTF8):]
+        if self.lines:
+            ord0 = ord(self.lines[0][0])
+            if ord0 in (0xef, 0xfeff):  # Strip the UTF-8 BOM
+                if ord0 == 0xfeff:
+                    self.lines[0] = self.lines[0][1:]
+                elif self.lines[0][:3] == '\xef\xbb\xbf':
+                    self.lines[0] = self.lines[0][3:]
         self.report = report or options.report
         self.report_error = self.report.error
 

testsuite/test_api.py

@@ -302,8 +302,20 @@ def test_styleguide_check_files(self):
         self.assertRaises(TypeError, pep8style.check_files, 42)
         # < 3.3 raises TypeError; >= 3.3 raises AttributeError
         self.assertRaises(Exception, pep8style.check_files, [42])
-        # TODO: runner
-        # TODO: input_file
+
+    def test_check_unicode(self):
+        # Do not crash if lines are Unicode (Python 2.x)
+        pep8.register_check(DummyChecker, ['Z701'])
+        source = '#\n'
+        if hasattr(source, 'decode'):
+            source = source.decode('ascii')
+
+        pep8style = pep8.StyleGuide()
+        count_errors = pep8style.input_file('stdin', lines=[source])
+
+        self.assertFalse(sys.stdout)
+        self.assertFalse(sys.stderr)
+        self.assertEqual(count_errors, 0)
 
     def test_check_nullbytes(self):
         pep8.register_check(DummyChecker, ['Z701'])
@@ -314,3 +326,6 @@ def test_check_nullbytes(self):
         self.assertTrue(sys.stdout[0].startswith("stdin:1:1: E901 TypeError"))
         self.assertFalse(sys.stderr)
         self.assertEqual(count_errors, 1)
+
+        # TODO: runner
+        # TODO: input_file