1#!/bin/sh
2# This script automatically test the given tool with the tool's test cases,
3# reporting anything of interest.
4
5# Written by Mike Stump <mrs@cygnus.com>
6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
7
8usage()
9{
10	if [ -n "$1" ] ; then
11		echo "$0: Error: $1" >&2
12		echo >&2
13	fi
14	cat >&2 <<EOUSAGE
15Usage: $0 [-strict] PREVIOUS CURRENT
16
17Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
18
19	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
20
21	Unless -strict is given, these discrepancies are not counted as errors:
22		missing/extra .sum files when comparing directories
23		tests that failed in PREVIOUS but pass in CURRENT
24		tests that were not in PREVIOUS but appear in CURRENT
25		tests in PREVIOUS that are missing in CURRENT
26
27	Exit with the following values:
28		0 if there is nothing of interest
29		1 if there are errors when comparing single test case files
30		N for the number of errors found when comparing directories
31EOUSAGE
32	exit 2
33}
34
35export LC_ALL=C
36
37tool=gxx
38
39tmp1=/tmp/$tool-testing.$$a
40tmp2=/tmp/$tool-testing.$$b
41now_s=/tmp/$tool-testing.$$d
42before_s=/tmp/$tool-testing.$$e
43lst1=/tmp/$tool-lst1.$$
44lst2=/tmp/$tool-lst2.$$
45lst3=/tmp/$tool-lst3.$$
46lst4=/tmp/$tool-lst4.$$
47lst5=/tmp/$tool-lst5.$$
48sum1=/tmp/$tool-sum1.$$
49sum2=/tmp/$tool-sum2.$$
50tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
51
52[ "$1" = "-strict" ] && strict=$1 && shift
53[ "$1" = "-?" ] && usage
54[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
55
56trap "rm -f $tmps" 0 1 2 3 5 9 13 15
57exit_status=0
58
59if [ -d "$1" -a -d "$2" ] ; then
60	find "$1/" -name '*.sum' >$lst1
61	find "$2/" -name '*.sum' >$lst2
62	echo "# Comparing directories"
63	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
64	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
65	echo
66	# remove leading directory components to compare
67	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
68	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
69	comm -23 $lst3 $lst4 >$lst5
70	if [ -s $lst5 ] ; then
71		echo "# Extra sum files in Dir1=$1"
72		sed -e "s|^|< $1/|" $lst5
73		echo
74		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
75	fi
76	comm -13 $lst3 $lst4 >$lst5
77	if [ -s $lst5 ] ; then
78		echo "# Extra sum files in Dir2=$2"
79		sed -e "s|^|> $2/|" $lst5
80		echo
81		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
82	fi
83	comm -12 $lst3 $lst4 | sort -u >$lst5
84	if [ ! -s $lst5 ] ; then
85		echo "# No common sum files"
86		exit_status=`expr $exit_status + 1`
87		exit $exit_status
88	fi
89	cmnsums=`cat $lst5 | wc -l`
90	echo "# Comparing $cmnsums common sum files"
91	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
92	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
93	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
94	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
95	ret=$?
96	if [ $ret -ne 0 ]; then
97		exit_status=`expr $exit_status + 1`
98		echo "## Differences found: $fname"
99	fi
100	if [ $exit_status -ne 0 ]; then
101		echo "# $exit_status differences in $cmnsums common sum files found"
102	else
103		echo "# No differences found in $cmnsums common sum files"
104	fi
105	exit $exit_status
106elif [ -d "$1" -o -d "$2" ] ; then
107	usage "Must specify either two directories or two files"
108fi
109
110sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
111sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2
112
113before=$tmp1
114now=$tmp2
115
116
117if sort -k 2 </dev/null >/dev/null 2>&1; then
118  skip1='-k 2'
119else
120  skip1='+1'
121fi
122
123sort -t ':' $skip1 "$now" > "$now_s"
124sort -t ':' $skip1 "$before" > "$before_s"
125
126grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
127grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
128
129grep -s . $tmp2 >/dev/null
130if [ $? = 0 ]; then
131	echo "Tests that now fail, but worked before:"
132	echo
133	cat $tmp2
134	echo
135	exit_status=1
136fi
137
138grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
139grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2
140
141grep -s . $tmp2 >/dev/null
142if [ $? = 0 ]; then
143	echo "Tests that now work, but didn't before:"
144	echo
145	cat $tmp2
146	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
147	echo
148fi
149
150grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
151grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
152
153grep -s . $tmp2 >/dev/null
154if [ $? = 0 ]; then
155	echo "New tests that FAIL:"
156	echo
157	cat $tmp2
158	echo
159	exit_status=1
160fi
161
162grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
163grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2
164
165grep -s . $tmp2 >/dev/null
166if [ $? = 0 ]; then
167	echo "New tests that PASS:"
168	echo
169	cat $tmp2
170	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
171	echo
172fi
173
174grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
175grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
176
177grep -s . $tmp2 >/dev/null
178if [ $? = 0 ]; then
179	echo "Old tests that passed, that have disappeared: (Eeek!)"
180	echo
181	cat $tmp2
182	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
183	echo
184fi
185
186grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
187grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2
188
189grep -s . $tmp2 >/dev/null
190if [ $? = 0 ]; then
191	echo "Old tests that failed, that have disappeared: (Eeek!)"
192	echo
193	cat $tmp2
194	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
195	echo
196fi
197
198exit $exit_status
199