77import os
88import subprocess
99import sys
10+ import time
1011import argparse
1112from glob import glob
13+ from rich .live import Live
14+ from rich .console import Console
15+ from rich .table import Table
1216
1317sys .path .append ("../tools" )
1418import pyboard
@@ -37,58 +41,80 @@ def compute_stats(lst):
3741 return avg , var ** 0.5
3842
3943
40- def run_script_on_target (target , script ):
44+ def run_script_on_target (target , script , run_command = None ):
4145 output = b""
4246 err = None
4347
4448 if isinstance (target , pyboard .Pyboard ):
4549 # Run via pyboard interface
4650 try :
4751 target .enter_raw_repl ()
52+ start_ts = time .monotonic_ns ()
4853 output = target .exec_ (script )
54+ if run_command :
55+ start_ts = time .monotonic_ns ()
56+ output = target .exec_ (run_command )
57+ end_ts = time .monotonic_ns ()
4958 except pyboard .PyboardError as er :
59+ end_ts = time .monotonic_ns ()
5060 err = er
61+ finally :
62+ target .exit_raw_repl ()
5163 else :
5264 # Run local executable
5365 try :
66+ if run_command :
67+ script += run_command
68+ start_ts = time .monotonic_ns ()
5469 p = subprocess .run (
5570 target , stdout = subprocess .PIPE , stderr = subprocess .STDOUT , input = script
5671 )
72+ end_ts = time .monotonic_ns ()
5773 output = p .stdout
5874 except subprocess .CalledProcessError as er :
75+ end_ts = time .monotonic_ns ()
5976 err = er
6077
61- return str (output .strip (), "ascii" ), err
78+ return str (output .strip (), "ascii" ), err , ( end_ts - start_ts ) // 1000
6279
6380
6481def run_feature_test (target , test ):
6582 with open ("feature_check/" + test + ".py" , "rb" ) as f :
6683 script = f .read ()
67- output , err = run_script_on_target (target , script )
84+ output , err , _ = run_script_on_target (target , script )
6885 if err is None :
6986 return output
7087 else :
7188 return "CRASH: %r" % err
7289
7390
74- def run_benchmark_on_target (target , script ):
75- output , err = run_script_on_target (target , script )
91+ def run_benchmark_on_target (target , script , run_command = None ):
92+ output , err , runtime_us = run_script_on_target (target , script , run_command )
7693 if err is None :
7794 time , norm , result = output .split (None , 2 )
7895 try :
79- return int (time ), int (norm ), result
96+ return int (time ), int (norm ), result , runtime_us
8097 except ValueError :
81- return - 1 , - 1 , "CRASH: %r" % output
98+ return - 1 , - 1 , "CRASH: %r" % output , runtime_us
8299 else :
83- return - 1 , - 1 , "CRASH: %r" % err
100+ return - 1 , - 1 , "CRASH: %r" % err , runtime_us
84101
85102
86- def run_benchmarks (target , param_n , param_m , n_average , test_list ):
103+ def run_benchmarks (console , target , param_n , param_m , n_average , test_list ):
87104 skip_complex = run_feature_test (target , "complex" ) != "complex"
88105 skip_native = run_feature_test (target , "native_check" ) != "native"
89106
107+ table = Table (show_header = True )
108+ table .add_column ("Test" )
109+ table .add_column ("Time" , justify = "right" )
110+ table .add_column ("Score" , justify = "right" )
111+ table .add_column ("Ref Time" , justify = "right" )
112+
113+ live = Live (table , console = console )
114+ live .start ()
115+
90116 for test_file in sorted (test_list ):
91- print (test_file + ": " , end = "" )
117+ # print(test_file + ": ", end="")
92118
93119 # Check if test should be skipped
94120 skip = (
@@ -99,14 +125,15 @@ def run_benchmarks(target, param_n, param_m, n_average, test_list):
99125 )
100126 if skip :
101127 print ("skip" )
128+ table .add_row (test_file , * (["skip" ] * 6 ))
102129 continue
103130
104131 # Create test script
105132 with open (test_file , "rb" ) as f :
106133 test_script = f .read ()
107134 with open (BENCH_SCRIPT_DIR + "benchrun.py" , "rb" ) as f :
108135 test_script += f .read ()
109- test_script + =b"bm_run(%u, %u)\n " % (param_n , param_m )
136+ bm_run = b"bm_run(%u, %u)\n " % (param_n , param_m )
110137
111138 # Write full test script if needed
112139 if 0 :
@@ -115,43 +142,59 @@ def run_benchmarks(target, param_n, param_m, n_average, test_list):
115142
116143 # Run MicroPython a given number of times
117144 times = []
145+ runtimes = []
118146 scores = []
119147 error = None
120148 result_out = None
121149 for _ in range (n_average ):
122- time , norm , result = run_benchmark_on_target (target , test_script )
123- if time < 0 or norm < 0 :
150+ self_time , norm , result , runtime_us = run_benchmark_on_target (
151+ target , test_script , bm_run
152+ )
153+ if self_time < 0 or norm < 0 :
124154 error = result
125155 break
126156 if result_out is None :
127157 result_out = result
128158 elif result != result_out :
129159 error = "FAIL self"
130160 break
131- times .append (time )
132- scores .append (1e6 * norm / time )
161+ times .append (self_time )
162+ runtimes .append (runtime_us )
163+ scores .append (1e6 * norm / self_time )
133164
134165 # Check result against truth if needed
135166 if error is None and result_out != "None" :
136- _ , _ , result_exp = run_benchmark_on_target (PYTHON_TRUTH , test_script )
167+ _ , _ , result_exp , _ = run_benchmark_on_target (PYTHON_TRUTH , test_script , bm_run )
137168 if result_out != result_exp :
138169 error = "FAIL truth"
139170
140171 if error is not None :
141- print (error )
172+ print (test_file , error )
173+ if error == "no matching params" :
174+ table .add_row (test_file , * ([None ] * 3 ))
175+ else :
176+ table .add_row (test_file , * (["error" ] * 3 ))
142177 else :
143178 t_avg , t_sd = compute_stats (times )
179+ r_avg , r_sd = compute_stats (runtimes )
144180 s_avg , s_sd = compute_stats (scores )
145- print (
146- "{:.2f} {:.4f} {:.2f} {:.4f}" .format (
147- t_avg , 100 * t_sd / t_avg , s_avg , 100 * s_sd / s_avg
148- )
181+ # print(
182+ # "{:.2f} {:.4f} {:.2f} {:.4f} {:.2f} {:.4f}".format(
183+ # t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg, r_avg, 100 * r_sd / r_avg
184+ # )
185+ # )
186+ table .add_row (
187+ test_file ,
188+ f"{ t_avg :.2f} { 100 * t_sd / t_avg :.1f} ,
189+ f"{ s_avg :.2f} { 100 * s_sd / s_avg :.1f} ,
190+ f"{ r_avg :.2f} { 100 * r_sd / r_avg :.1f} ,
149191 )
150192 if 0 :
151193 print (" times: " , times )
152194 print (" scores:" , scores )
153195
154- sys .stdout .flush ()
196+ live .update (table , refresh = True )
197+ live .stop ()
155198
156199
157200def parse_output (filename ):
@@ -268,9 +311,10 @@ def main():
268311 else :
269312 tests = sorted (args .files )
270313
314+ console = Console ()
271315 print ("N={} M={} n_average={}" .format (N , M , n_average ))
272316
273- run_benchmarks (target , N , M , n_average , tests )
317+ run_benchmarks (console , target , N , M , n_average , tests )
274318
275319 if isinstance (target , pyboard .Pyboard ):
276320 target .exit_raw_repl ()
0 commit comments