1#!/usr/bin/python 2# 3# Copyright (C) 2014 Free Software Foundation, Inc. 4# 5# This script is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 3, or (at your option) 8# any later version. 9 10import sys 11import getopt 12import re 13import io 14from datetime import datetime 15from operator import attrgetter 16 17# True if unrecognised lines should cause a fatal error. Might want to turn 18# this on by default later. 19strict = False 20 21# True if the order of .log segments should match the .sum file, false if 22# they should keep the original order. 23sort_logs = True 24 25# A version of open() that is safe against whatever binary output 26# might be added to the log. 27def safe_open (filename): 28 if sys.version_info >= (3, 0): 29 return open (filename, 'r', errors = 'surrogateescape') 30 return open (filename, 'r') 31 32# Force stdout to handle escape sequences from a safe_open file. 33if sys.version_info >= (3, 0): 34 sys.stdout = io.TextIOWrapper (sys.stdout.buffer, 35 errors = 'surrogateescape') 36 37class Named: 38 def __init__ (self, name): 39 self.name = name 40 41class ToolRun (Named): 42 def __init__ (self, name): 43 Named.__init__ (self, name) 44 # The variations run for this tool, mapped by --target_board name. 45 self.variations = dict() 46 47 # Return the VariationRun for variation NAME. 48 def get_variation (self, name): 49 if name not in self.variations: 50 self.variations[name] = VariationRun (name) 51 return self.variations[name] 52 53class VariationRun (Named): 54 def __init__ (self, name): 55 Named.__init__ (self, name) 56 # A segment of text before the harness runs start, describing which 57 # baseboard files were loaded for the target. 58 self.header = None 59 # The harnesses run for this variation, mapped by filename. 60 self.harnesses = dict() 61 # A list giving the number of times each type of result has 62 # been seen. 63 self.counts = [] 64 65 # Return the HarnessRun for harness NAME. 66 def get_harness (self, name): 67 if name not in self.harnesses: 68 self.harnesses[name] = HarnessRun (name) 69 return self.harnesses[name] 70 71class HarnessRun (Named): 72 def __init__ (self, name): 73 Named.__init__ (self, name) 74 # Segments of text that make up the harness run, mapped by a test-based 75 # key that can be used to order them. 76 self.segments = dict() 77 # Segments of text that make up the harness run but which have 78 # no recognized test results. These are typically harnesses that 79 # are completely skipped for the target. 80 self.empty = [] 81 # A list of results. Each entry is a pair in which the first element 82 # is a unique sorting key and in which the second is the full 83 # PASS/FAIL line. 84 self.results = [] 85 86 # Add a segment of text to the harness run. If the segment includes 87 # test results, KEY is an example of one of them, and can be used to 88 # combine the individual segments in order. If the segment has no 89 # test results (e.g. because the harness doesn't do anything for the 90 # current configuration) then KEY is None instead. In that case 91 # just collect the segments in the order that we see them. 92 def add_segment (self, key, segment): 93 if key: 94 assert key not in self.segments 95 self.segments[key] = segment 96 else: 97 self.empty.append (segment) 98 99class Segment: 100 def __init__ (self, filename, start): 101 self.filename = filename 102 self.start = start 103 self.lines = 0 104 105class Prog: 106 def __init__ (self): 107 # The variations specified on the command line. 108 self.variations = [] 109 # The variations seen in the input files. 110 self.known_variations = set() 111 # The tools specified on the command line. 112 self.tools = [] 113 # Whether to create .sum rather than .log output. 114 self.do_sum = True 115 # Regexps used while parsing. 116 self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$') 117 self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') 118 self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' 119 r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' 120 r'|KFAIL):\s*(.+)') 121 self.completed_re = re.compile (r'.* completed at (.*)') 122 # Pieces of text to write at the head of the output. 123 # start_line is a pair in which the first element is a datetime 124 # and in which the second is the associated 'Test Run By' line. 125 self.start_line = None 126 self.native_line = '' 127 self.target_line = '' 128 self.host_line = '' 129 self.acats_premable = '' 130 # Pieces of text to write at the end of the output. 131 # end_line is like start_line but for the 'runtest completed' line. 132 self.acats_failures = [] 133 self.version_output = '' 134 self.end_line = None 135 # Known summary types. 136 self.count_names = [ 137 '# of expected passes\t\t', 138 '# of unexpected failures\t', 139 '# of unexpected successes\t', 140 '# of expected failures\t\t', 141 '# of unknown successes\t\t', 142 '# of known failures\t\t', 143 '# of untested testcases\t\t', 144 '# of unresolved testcases\t', 145 '# of unsupported tests\t\t' 146 ] 147 self.runs = dict() 148 149 def usage (self): 150 name = sys.argv[0] 151 sys.stderr.write ('Usage: ' + name 152 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... 153 154 tool The tool (e.g. g++, libffi) for which to create a 155 new test summary file. If not specified then output 156 is created for all tools. 157 variant-list One or more test variant names. If the list is 158 not specified then one is constructed from all 159 variants in the files for <tool>. 160 sum-file A test summary file with the format of those 161 created by runtest from DejaGnu. 162 If -L is used, merge *.log files instead of *.sum. In this 163 mode the exact order of lines may not be preserved, just different 164 Running *.exp chunks should be in correct order. 165''') 166 sys.exit (1) 167 168 def fatal (self, what, string): 169 if not what: 170 what = sys.argv[0] 171 sys.stderr.write (what + ': ' + string + '\n') 172 sys.exit (1) 173 174 # Parse the command-line arguments. 175 def parse_cmdline (self): 176 try: 177 (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') 178 if len (self.files) == 0: 179 self.usage() 180 for (option, value) in options: 181 if option == '-l': 182 self.variations.append (value) 183 elif option == '-t': 184 self.tools.append (value) 185 else: 186 self.do_sum = False 187 except getopt.GetoptError as e: 188 self.fatal (None, e.msg) 189 190 # Try to parse time string TIME, returning an arbitrary time on failure. 191 # Getting this right is just a nice-to-have so failures should be silent. 192 def parse_time (self, time): 193 try: 194 return datetime.strptime (time, '%c') 195 except ValueError: 196 return datetime.now() 197 198 # Parse an integer and abort on failure. 199 def parse_int (self, filename, value): 200 try: 201 return int (value) 202 except ValueError: 203 self.fatal (filename, 'expected an integer, got: ' + value) 204 205 # Return a list that represents no test results. 206 def zero_counts (self): 207 return [0 for x in self.count_names] 208 209 # Return the ToolRun for tool NAME. 210 def get_tool (self, name): 211 if name not in self.runs: 212 self.runs[name] = ToolRun (name) 213 return self.runs[name] 214 215 # Add the result counts in list FROMC to TOC. 216 def accumulate_counts (self, toc, fromc): 217 for i in range (len (self.count_names)): 218 toc[i] += fromc[i] 219 220 # Parse the list of variations after 'Schedule of variations:'. 221 # Return the number seen. 222 def parse_variations (self, filename, file): 223 num_variations = 0 224 while True: 225 line = file.readline() 226 if line == '': 227 self.fatal (filename, 'could not parse variation list') 228 if line == '\n': 229 break 230 self.known_variations.add (line.strip()) 231 num_variations += 1 232 return num_variations 233 234 # Parse from the first line after 'Running target ...' to the end 235 # of the run's summary. 236 def parse_run (self, filename, file, tool, variation, num_variations): 237 header = None 238 harness = None 239 segment = None 240 final_using = 0 241 242 # If this is the first run for this variation, add any text before 243 # the first harness to the header. 244 if not variation.header: 245 segment = Segment (filename, file.tell()) 246 variation.header = segment 247 248 # Parse up until the first line of the summary. 249 if num_variations == 1: 250 end = '\t\t=== ' + tool.name + ' Summary ===\n' 251 else: 252 end = ('\t\t=== ' + tool.name + ' Summary for ' 253 + variation.name + ' ===\n') 254 while True: 255 line = file.readline() 256 if line == '': 257 self.fatal (filename, 'no recognised summary line') 258 if line == end: 259 break 260 261 # Look for the start of a new harness. 262 if line.startswith ('Running ') and line.endswith (' ...\n'): 263 # Close off the current harness segment, if any. 264 if harness: 265 segment.lines -= final_using 266 harness.add_segment (first_key, segment) 267 name = line[len ('Running '):-len(' ...\n')] 268 harness = variation.get_harness (name) 269 segment = Segment (filename, file.tell()) 270 first_key = None 271 final_using = 0 272 continue 273 274 # Record test results. Associate the first test result with 275 # the harness segment, so that if a run for a particular harness 276 # has been split up, we can reassemble the individual segments 277 # in a sensible order. 278 # 279 # dejagnu sometimes issues warnings about the testing environment 280 # before running any tests. Treat them as part of the header 281 # rather than as a test result. 282 match = self.result_re.match (line) 283 if match and (harness or not line.startswith ('WARNING:')): 284 if not harness: 285 self.fatal (filename, 'saw test result before harness name') 286 name = match.group (2) 287 # Ugly hack to get the right order for gfortran. 288 if name.startswith ('gfortran.dg/g77/'): 289 name = 'h' + name 290 key = (name, len (harness.results)) 291 harness.results.append ((key, line)) 292 if not first_key and sort_logs: 293 first_key = key 294 295 # 'Using ...' lines are only interesting in a header. Splitting 296 # the test up into parallel runs leads to more 'Using ...' lines 297 # than there would be in a single log. 298 if line.startswith ('Using '): 299 final_using += 1 300 else: 301 final_using = 0 302 303 # Add other text to the current segment, if any. 304 if segment: 305 segment.lines += 1 306 307 # Close off the final harness segment, if any. 308 if harness: 309 segment.lines -= final_using 310 harness.add_segment (first_key, segment) 311 312 # Parse the rest of the summary (the '# of ' lines). 313 if len (variation.counts) == 0: 314 variation.counts = self.zero_counts() 315 while True: 316 before = file.tell() 317 line = file.readline() 318 if line == '': 319 break 320 if line == '\n': 321 continue 322 if not line.startswith ('# '): 323 file.seek (before) 324 break 325 found = False 326 for i in range (len (self.count_names)): 327 if line.startswith (self.count_names[i]): 328 count = line[len (self.count_names[i]):-1].strip() 329 variation.counts[i] += self.parse_int (filename, count) 330 found = True 331 break 332 if not found: 333 self.fatal (filename, 'unknown test result: ' + line[:-1]) 334 335 # Parse an acats run, which uses a different format from dejagnu. 336 # We have just skipped over '=== acats configuration ==='. 337 def parse_acats_run (self, filename, file): 338 # Parse the preamble, which describes the configuration and logs 339 # the creation of support files. 340 record = (self.acats_premable == '') 341 if record: 342 self.acats_premable = '\t\t=== acats configuration ===\n' 343 while True: 344 line = file.readline() 345 if line == '': 346 self.fatal (filename, 'could not parse acats preamble') 347 if line == '\t\t=== acats tests ===\n': 348 break 349 if record: 350 self.acats_premable += line 351 352 # Parse the test results themselves, using a dummy variation name. 353 tool = self.get_tool ('acats') 354 variation = tool.get_variation ('none') 355 self.parse_run (filename, file, tool, variation, 1) 356 357 # Parse the failure list. 358 while True: 359 before = file.tell() 360 line = file.readline() 361 if line.startswith ('*** FAILURES: '): 362 self.acats_failures.append (line[len ('*** FAILURES: '):-1]) 363 continue 364 file.seek (before) 365 break 366 367 # Parse the final summary at the end of a log in order to capture 368 # the version output that follows it. 369 def parse_final_summary (self, filename, file): 370 record = (self.version_output == '') 371 while True: 372 line = file.readline() 373 if line == '': 374 break 375 if line.startswith ('# of '): 376 continue 377 if record: 378 self.version_output += line 379 if line == '\n': 380 break 381 382 # Parse a .log or .sum file. 383 def parse_file (self, filename, file): 384 tool = None 385 target = None 386 num_variations = 1 387 while True: 388 line = file.readline() 389 if line == '': 390 return 391 392 # Parse the list of variations, which comes before the test 393 # runs themselves. 394 if line.startswith ('Schedule of variations:'): 395 num_variations = self.parse_variations (filename, file) 396 continue 397 398 # Parse a testsuite run for one tool/variation combination. 399 if line.startswith ('Running target '): 400 name = line[len ('Running target '):-1] 401 if not tool: 402 self.fatal (filename, 'could not parse tool name') 403 if name not in self.known_variations: 404 self.fatal (filename, 'unknown target: ' + name) 405 self.parse_run (filename, file, tool, 406 tool.get_variation (name), 407 num_variations) 408 # If there is only one variation then there is no separate 409 # summary for it. Record any following version output. 410 if num_variations == 1: 411 self.parse_final_summary (filename, file) 412 continue 413 414 # Parse the start line. In the case where several files are being 415 # parsed, pick the one with the earliest time. 416 match = self.test_run_re.match (line) 417 if match: 418 time = self.parse_time (match.group (2)) 419 if not self.start_line or self.start_line[0] > time: 420 self.start_line = (time, line) 421 continue 422 423 # Parse the form used for native testing. 424 if line.startswith ('Native configuration is '): 425 self.native_line = line 426 continue 427 428 # Parse the target triplet. 429 if line.startswith ('Target is '): 430 self.target_line = line 431 continue 432 433 # Parse the host triplet. 434 if line.startswith ('Host is '): 435 self.host_line = line 436 continue 437 438 # Parse the acats premable. 439 if line == '\t\t=== acats configuration ===\n': 440 self.parse_acats_run (filename, file) 441 continue 442 443 # Parse the tool name. 444 match = self.tool_re.match (line) 445 if match: 446 tool = self.get_tool (match.group (1)) 447 continue 448 449 # Skip over the final summary (which we instead create from 450 # individual runs) and parse the version output. 451 if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': 452 if file.readline() != '\n': 453 self.fatal (filename, 'expected blank line after summary') 454 self.parse_final_summary (filename, file) 455 continue 456 457 # Parse the completion line. In the case where several files 458 # are being parsed, pick the one with the latest time. 459 match = self.completed_re.match (line) 460 if match: 461 time = self.parse_time (match.group (1)) 462 if not self.end_line or self.end_line[0] < time: 463 self.end_line = (time, line) 464 continue 465 466 # Sanity check to make sure that important text doesn't get 467 # dropped accidentally. 468 if strict and line.strip() != '': 469 self.fatal (filename, 'unrecognised line: ' + line[:-1]) 470 471 # Output a segment of text. 472 def output_segment (self, segment): 473 with safe_open (segment.filename) as file: 474 file.seek (segment.start) 475 for i in range (segment.lines): 476 sys.stdout.write (file.readline()) 477 478 # Output a summary giving the number of times each type of result has 479 # been seen. 480 def output_summary (self, tool, counts): 481 for i in range (len (self.count_names)): 482 name = self.count_names[i] 483 # dejagnu only prints result types that were seen at least once, 484 # but acats always prints a number of unexpected failures. 485 if (counts[i] > 0 486 or (tool.name == 'acats' 487 and name.startswith ('# of unexpected failures'))): 488 sys.stdout.write ('%s%d\n' % (name, counts[i])) 489 490 # Output unified .log or .sum information for a particular variation, 491 # with a summary at the end. 492 def output_variation (self, tool, variation): 493 self.output_segment (variation.header) 494 for harness in sorted (variation.harnesses.values(), 495 key = attrgetter ('name')): 496 sys.stdout.write ('Running ' + harness.name + ' ...\n') 497 if self.do_sum: 498 harness.results.sort() 499 for (key, line) in harness.results: 500 sys.stdout.write (line) 501 else: 502 # Rearrange the log segments into test order (but without 503 # rearranging text within those segments). 504 for key in sorted (harness.segments.keys()): 505 self.output_segment (harness.segments[key]) 506 for segment in harness.empty: 507 self.output_segment (segment) 508 if len (self.variations) > 1: 509 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' 510 + variation.name + ' ===\n\n') 511 self.output_summary (tool, variation.counts) 512 513 # Output unified .log or .sum information for a particular tool, 514 # with a summary at the end. 515 def output_tool (self, tool): 516 counts = self.zero_counts() 517 if tool.name == 'acats': 518 # acats doesn't use variations, so just output everything. 519 # It also has a different approach to whitespace. 520 sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') 521 for variation in tool.variations.values(): 522 self.output_variation (tool, variation) 523 self.accumulate_counts (counts, variation.counts) 524 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') 525 else: 526 # Output the results in the usual dejagnu runtest format. 527 sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' 528 'Schedule of variations:\n') 529 for name in self.variations: 530 if name in tool.variations: 531 sys.stdout.write (' ' + name + '\n') 532 sys.stdout.write ('\n') 533 for name in self.variations: 534 if name in tool.variations: 535 variation = tool.variations[name] 536 sys.stdout.write ('Running target ' 537 + variation.name + '\n') 538 self.output_variation (tool, variation) 539 self.accumulate_counts (counts, variation.counts) 540 sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') 541 self.output_summary (tool, counts) 542 543 def main (self): 544 self.parse_cmdline() 545 try: 546 # Parse the input files. 547 for filename in self.files: 548 with safe_open (filename) as file: 549 self.parse_file (filename, file) 550 551 # Decide what to output. 552 if len (self.variations) == 0: 553 self.variations = sorted (self.known_variations) 554 else: 555 for name in self.variations: 556 if name not in self.known_variations: 557 self.fatal (None, 'no results for ' + name) 558 if len (self.tools) == 0: 559 self.tools = sorted (self.runs.keys()) 560 561 # Output the header. 562 if self.start_line: 563 sys.stdout.write (self.start_line[1]) 564 sys.stdout.write (self.native_line) 565 sys.stdout.write (self.target_line) 566 sys.stdout.write (self.host_line) 567 sys.stdout.write (self.acats_premable) 568 569 # Output the main body. 570 for name in self.tools: 571 if name not in self.runs: 572 self.fatal (None, 'no results for ' + name) 573 self.output_tool (self.runs[name]) 574 575 # Output the footer. 576 if len (self.acats_failures) > 0: 577 sys.stdout.write ('*** FAILURES: ' 578 + ' '.join (self.acats_failures) + '\n') 579 sys.stdout.write (self.version_output) 580 if self.end_line: 581 sys.stdout.write (self.end_line[1]) 582 except IOError as e: 583 self.fatal (e.filename, e.strerror) 584 585Prog().main() 586