Improve the test framework to count the errors, motivated by issue #237

Author: florentx

testsuite/E12.py

@@ -52,13 +52,13 @@
    'key1': 'value',
    'key2': 'value',
 }
-#: E121
+#: E121 E121
 rv.update(dict.fromkeys((
     'qualif_nr', 'reasonComment_en', 'reasonComment_fr',
     'reasonComment_de', 'reasonComment_it'),
           '?'),
           "foo")
-#: E121
+#: E121 E121
 abricot = 3 + \
           4 + \
           5 + 6
@@ -121,7 +121,7 @@
     "bad", "hanging", "close"
     )
 #
-#: E123
+#: E123 E123 E123
 result = {
     'foo': [
         'bar', {
@@ -166,7 +166,7 @@
          help="exclude files or directories which match these "
               "comma separated patterns (default: %s)" % DEFAULT_EXCLUDE
               )
-#: E124 E128
+#: E124 E128 E128
 if line_removed:
     self.event(cr, uid,
         name="Removing the option for contract",
@@ -208,14 +208,14 @@ def qualify_by_address(
     1, 2, 3,
     4, 5, 6,
         ]
-#: E126
+#: E126 E126
 abris = 3 + \
         4 + \
         5 + 6
 #: E126
 fixed = re.sub(r'\t+', ' ', target[c::-1], 1)[::-1] + \
         target[c + 1:]
-#: E126
+#: E126 E126
 rv.update(dict.fromkeys((
             'qualif_nr', 'reasonComment_en', 'reasonComment_fr',
             'reasonComment_de', 'reasonComment_it'),
@@ -313,13 +313,13 @@ def qualify_by_address(self, cr, uid, ids, context=None,
 #: E127
 foo(1, 2, 3,
              4, 5, 6)
-#: E128
+#: E128 E128
 if line_removed:
     self.event(cr, uid,
               name="Removing the option for contract",
               description="contract line has been removed",
                )
-#: E124 E127
+#: E124 E127 E127
 if line_removed:
     self.event(cr, uid,
                 name="Removing the option for contract",

testsuite/E21.py

@@ -1,6 +1,6 @@
 #: E211
 spam (1)
-#: E211
+#: E211 E211
 dict ['key'] = list [index]
 #: E211
 dict['key'] ['subkey'] = list[index]

testsuite/E22.py

@@ -1,15 +1,15 @@
 #: E221
 a = 12 + 3
 b = 4  + 5
-#: E221
+#: E221 E221
 x             = 1
 y             = 2
 long_variable = 3
-#: E221
+#: E221 E221
 x[0]          = 1
 x[1]          = 2
 long_variable = 3
-#: E221
+#: E221 E221
 x = f(x)          + 1
 y = long_variable + 2
 z = x[0]          + 3
@@ -23,11 +23,11 @@
 #: E222
 a = a +  1
 b = b + 10
-#: E222
+#: E222 E222
 x =            -1
 y =            -2
 long_variable = 3
-#: E222
+#: E222 E222
 x[0] =          1
 x[1] =          2
 long_variable = 3
@@ -68,9 +68,9 @@
 _1kB = _1MB >>10
 #: E225
 _1kB = _1MB>> 10
-#: E225
+#: E225 E225
 i=i+ 1
-#: E225
+#: E225 E225
 i=i +1
 #: E225 E226
 i=i+1
@@ -86,15 +86,15 @@
 
 #: E226
 z = 2**30
-#: E226
+#: E226 E226
 c = (a+b) * (a-b)
 #: E226
 norman = True+False
 #: E226
 x = x*2 - 1
 #: E226
 x = x/2 - 1
-#: E226
+#: E226 E226
 hypot2 = x*x + y*y
 #: E226
 c = (a + b)*(a - b)

testsuite/E24.py

@@ -6,7 +6,7 @@
 a = (1,	2)  # tab before 2
 #: Okay
 b = (1, 20)  # space before 20
-#: E241
+#: E241 E241 E241
 # issue 135
 more_spaces = [a,    b,
                ef,  +h,

testsuite/E25.py

@@ -1,12 +1,12 @@
-#: E251
+#: E251 E251
 def foo(bar = False):
     '''Test function with an error in declaration'''
     pass
 #: E251
 foo(bar= True)
 #: E251
 foo(bar =True)
-#: E251
+#: E251 E251
 foo(bar = True)
 #: E251
 y = bar(root= "sdasd")

testsuite/E72.py

@@ -32,7 +32,7 @@
 assert type(res) is not type((1, ))
 #: E211 E721
 assert type(res) == type ([2, ])
-#: E201 E202 E721
+#: E201 E201 E202 E721
 assert type(res) == type( ( ) )
 #: E201 E202 E721
 assert type(res) == type( (0, ) )

testsuite/E90.py

@@ -2,13 +2,13 @@
 }
 #: E901
 = [x
-#: E901 E101 W191
+#: E901 E101 E101 W191 W191
 while True:
     try:
 	    pass
 	except:
 		print 'Whoops'
-#: E122 E225 E251 E701
+#: E122 E122 E225 E251 E251 E701
 
 # Do not crash if code is invalid
 if msg:

testsuite/W19.py

@@ -38,7 +38,7 @@
 	pass
 #:
 
-#: E101 W191
+#: E101 E101 W191 W191
 if start[1] > end_col and not (
         over_indent == 4 and indent_next):
 	return(0, "E121 continuation line over-"
@@ -56,7 +56,7 @@ def long_function_name(
 if ((row < 0 or self.moduleCount <= row or
      col < 0 or self.moduleCount <= col)):
 	raise Exception("%s,%s - %s" % (row, col, self.moduleCount))
-#: E101 W191
+#: E101 E101 E101 E101 W191 W191 W191 W191 W191 W191
 if bar:
 	return(
 	    start, 'E121 lines starting with a '
@@ -86,14 +86,14 @@ def long_function_name(
     b == """abc def ghi
 jkl mno"""):
 	return True
-#: E101 W191
+#: E101 W191 W191
 if length > options.max_line_length:
 	return options.max_line_length, \
 	    "E501 line too long (%d characters)" % length
 
 
 #
-#: E101 W191
+#: E101 W191 W191
 if os.path.exists(os.path.join(path, PEP8_BIN)):
 	cmd = ([os.path.join(path, PEP8_BIN)] +
 	       self._pep8_options(targetfile))

testsuite/support.py

@@ -28,21 +28,28 @@ def __init__(self, options):
     def get_file_results(self):
         # Check if the expected errors were found
         label = '%s:%s:1' % (self.filename, self.line_offset)
-        codes = sorted(self.expected)
-        for code in codes:
+        for code in self.expected:
             if not self.counters.get(code):
                 self.file_errors += 1
                 self.total_errors += 1
                 print('%s: error %s not found' % (label, code))
+            else:
+                self.counters[code] -= 1
+        for code, extra in sorted(self.counters.items()):
+            if code not in self._benchmark_keys:
+                if extra:
+                    self.file_errors += 1
+                    self.total_errors += 1
+                    print('%s: error %s found too many times (+%d)' %
+                          (label, code, extra))
+                # Reset counters
+                del self.counters[code]
         if self._verbose and not self.file_errors:
             print('%s: passed (%s)' %
-                  (label, ' '.join(codes) or 'Okay'))
+                  (label, ' '.join(self.expected) or 'Okay'))
         self.counters['test cases'] += 1
         if self.file_errors:
             self.counters['failed tests'] += 1
-        # Reset counters
-        for key in set(self.counters) - set(self._benchmark_keys):
-            del self.counters[key]
         self.messages = {}
         return super(TestReport, self).get_file_results()