1#!/bin/bash
2# Copyright (C) 2006, 2008 Free Software Foundation
3#
4# Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
5# Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
6#
7# See usage() below.
8
9usage () {
10    cat <<EOF >&2
11Usage:
12    dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
13    <variant-name> names the desired variant, "/" must be written as "\/".
14    Use the empty string ("") for the first variant in each file.
15    Output is to stdout.
16    Non-verbose output is degradation info like PASS->FAIL.
17    -v adds improvement info like FAIL->PASS.
18    -v -v adds info like tests that are no longer run.
19    -v -v -v adds info for tests that have not changed status.
20    -v -v -v -v is used for debugging.
21EOF
22}
23
24verbose=0
25while test "$1" = "-v"; do
26    verbose=`expr $verbose + 1`
27    shift
28done
29
30if test $# -ne 3 ; then
31    usage
32    exit 1
33fi
34
35if test ! -f "$2"; then
36    echo "unable to open $2" >&2
37    exit 1
38fi
39
40if test ! -f "$3"; then
41    echo "unable to open $3" >&2
42    exit 1
43fi
44
45# Command differences for various platforms.
46case `uname -s` in
47Darwin|NetBSD)
48    E=-E	# sed
49    ;;
50*)
51    E=-r	# sed
52    ;;
53esac
54
55# sections are identified by separator lines beginning with '\t\t==='.
56# section 0 identifies run date, target, and host.
57# section 1 and subsequent contain test data for a target variant.
58# -skip to /^Running target/ and use that line to identify the variant.
59# -subsequent lines contain the result data.  They begin with:
60# '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
61VARIANT="$1"
62OFILE="$2"
63OBASE=`basename "$2"`
64NFILE="$3"
65NBASE=`basename "$3"`
66
67echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
68echo
69
70header="^Running target $VARIANT"
71
72temp=`grep "$header" $OFILE`
73if test -z "$temp"; then
74    echo "Error: variant \"$VARIANT\" not found in $OFILE."
75    exit 1
76fi
77temp=`grep "$header" $NFILE`
78if test -z "$temp"; then
79    echo "Error: variant \"$VARIANT\" not found in $NFILE."
80    exit 1
81fi
82unset temp
83
84# Copy out the old file's section 0.
85echo "Older log file: $OFILE"
86sed $E -e '/^[[:space:]]+===/,$d' $OFILE
87
88# Copy out the new file's section 0.
89echo "Newer log file: $NFILE"
90sed $E -e '/^[[:space:]]+===/,$d' $NFILE
91
92# Create a temporary file from the old file's interesting section.
93sed $E -e "1,/$header/d" \
94  -e '/^[[:space:]]+===/,$d' \
95  -e '/^[A-Z]+:/!d' \
96  -e '/^(WARNING|ERROR):/d' \
97  -e 's/\r$//' \
98  -e 's/^/O:/' \
99  $OFILE |
100  sort -s -t : -k 3b - \
101  >/tmp/o$$-$OBASE
102
103# Create a temporary file from the new file's interesting section.
104sed $E -e "1,/$header/d" \
105  -e '/^[[:space:]]+===/,$d' \
106  -e '/^[A-Z]+:/!d' \
107  -e '/^(WARNING|ERROR):/d' \
108  -e 's/\r$//' \
109  -e 's/^/N:/' \
110  $NFILE |
111  sort -s -t : -k 3b - \
112  >/tmp/n$$-$NBASE
113
114# Merge the two files, then compare adjacent lines.
115# Comparison is complicated by tests that may be run multiple times.
116# If that case, we assume that the order is the same in both files.
117cat <<EOF >compare-$$.awk
118BEGIN {
119    FS = ":"
120    queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
121    verbose = verbose + 0	# Make sure it's defined.
122}
123
124# FIFO circular queue
125function push(st, nm) {
126    queueN += 1; status[queueN] = st; name[queueN] = nm
127}
128function peek() {
129    result = 0
130    if (queueN >= queue1) result = queue1
131    return result
132}
133function drop() {
134    queue1 += 1
135    if (queue1 > queueN) { queue1 = 1; queueN = 0; }
136}
137
138function compare(st, nm) {
139    old = peek()
140    if (old == 0) {
141        # This new test wasn't run last time.
142        if (verbose >= 2) printf("NA->%s:%s\n", st, nm)
143    }
144    else {
145	# Compare this new test to the first queued old one.
146	if (verbose >= 4) {
147	    printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
148	     status[old], name[old], st, nm)
149	}
150        if (name[old] != nm) {
151	    # The old test wasn't run this time and
152	    # the new test wasn't run last time.
153	    if (verbose >= 2) {
154		printf("%s->NA:%s\n", status[old], name[old])
155		if (nm != "") printf("NA->%s:%s\n", st, nm)
156	    }
157	    drop()
158        }
159	else {
160	    notable = 0
161	    if (status[old] == st) {
162	        # Status of this test has not changed.
163		if (verbose >= 3) printf("%s:%s\n", st, nm)
164	    }
165	    else if(status[old] == "PASS" && st == "XFAIL") {
166	        if (verbose >= 1) notable = 1
167	    }
168	    else if(status[old] == "PASS" || st == "FAIL") {
169	        # Test did pass but doesn't now
170		# or didn't fail but does now.
171		notable = 1
172	    }
173	    else if(st == "PASS") {
174	        # Test didn't pass but does now.
175		if (verbose >= 1) notable = 1
176	    }
177	    else if(verbose >= 2) {
178	        # Miscellaneous status change.
179		notable = 1
180	    }
181	    if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
182	    drop()
183	}
184    }
185}
186
187/^O:/ {
188    while (old = peek()) {
189	if (name[old] == \$3) break;
190	# The queued test is no longer run.
191	compare("", "");
192    }
193    # Save this test for later comparison.
194    push(\$2, \$3)
195}
196
197/^N:/ {
198    compare(\$2, \$3)
199}
200
201END {
202    while (old = peek()) compare("", "")
203}
204EOF
205sort -m -s -t : -k 3b /tmp/o$$-$OBASE /tmp/n$$-$NBASE |
206 awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
207
208# Delete the temporary files.
209rm -f compare-$$.awk /tmp/o$$-$OBASE /tmp/n$$-$NBASE
210
211exit 0
212