1#!/bin/sh 2# This script automatically test the given tool with the tool's test cases, 3# reporting anything of interest. 4 5# Written by Mike Stump <mrs@cygnus.com> 6# Subdir comparison added by Quentin Neill <quentin.neill@amd.com> 7 8usage() 9{ 10 if [ -n "$1" ] ; then 11 echo "$0: Error: $1" >&2 12 echo >&2 13 fi 14 cat >&2 <<EOUSAGE 15Usage: $0 [-strict] PREVIOUS CURRENT 16 17Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. 18 19 If PREVIOUS and CURRENT are directories, find and compare any *.sum files. 20 21 Unless -strict is given, these discrepancies are not counted as errors: 22 missing/extra .sum files when comparing directories 23 tests that failed in PREVIOUS but pass in CURRENT 24 tests that were not in PREVIOUS but appear in CURRENT 25 tests in PREVIOUS that are missing in CURRENT 26 27 Exit with the following values: 28 0 if there is nothing of interest 29 1 if there are errors when comparing single test case files 30 N for the number of errors found when comparing directories 31EOUSAGE 32 exit 2 33} 34 35export LC_ALL=C 36 37tool=gxx 38 39tmp1=/tmp/$tool-testing.$$a 40tmp2=/tmp/$tool-testing.$$b 41now_s=/tmp/$tool-testing.$$d 42before_s=/tmp/$tool-testing.$$e 43lst1=/tmp/$tool-lst1.$$ 44lst2=/tmp/$tool-lst2.$$ 45lst3=/tmp/$tool-lst3.$$ 46lst4=/tmp/$tool-lst4.$$ 47lst5=/tmp/$tool-lst5.$$ 48sum1=/tmp/$tool-sum1.$$ 49sum2=/tmp/$tool-sum2.$$ 50tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" 51 52[ "$1" = "-strict" ] && strict=$1 && shift 53[ "$1" = "-?" ] && usage 54[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" 55 56trap "rm -f $tmps" 0 1 2 3 5 9 13 15 57exit_status=0 58 59if [ -d "$1" -a -d "$2" ] ; then 60 find "$1/" -name '*.sum' >$lst1 61 find "$2/" -name '*.sum' >$lst2 62 echo "# Comparing directories" 63 echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" 64 echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" 65 echo 66 # remove leading directory components to compare 67 sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 68 sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 69 comm -23 $lst3 $lst4 >$lst5 70 if [ -s $lst5 ] ; then 71 echo "# Extra sum files in Dir1=$1" 72 sed -e "s|^|< $1/|" $lst5 73 echo 74 [ -n "$strict" ] && exit_status=`expr $exit_status + 1` 75 fi 76 comm -13 $lst3 $lst4 >$lst5 77 if [ -s $lst5 ] ; then 78 echo "# Extra sum files in Dir2=$2" 79 sed -e "s|^|> $2/|" $lst5 80 echo 81 [ -n "$strict" ] && exit_status=`expr $exit_status + 1` 82 fi 83 comm -12 $lst3 $lst4 | sort -u >$lst5 84 if [ ! -s $lst5 ] ; then 85 echo "# No common sum files" 86 exit_status=`expr $exit_status + 1` 87 exit $exit_status 88 fi 89 cmnsums=`cat $lst5 | wc -l` 90 echo "# Comparing $cmnsums common sum files" 91 ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 92 ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 93 echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" 94 ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 95 ret=$? 96 if [ $ret -ne 0 ]; then 97 exit_status=`expr $exit_status + 1` 98 echo "## Differences found: $fname" 99 fi 100 if [ $exit_status -ne 0 ]; then 101 echo "# $exit_status differences in $cmnsums common sum files found" 102 else 103 echo "# No differences found in $cmnsums common sum files" 104 fi 105 exit $exit_status 106elif [ -d "$1" -o -d "$2" ] ; then 107 usage "Must specify either two directories or two files" 108fi 109 110sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1 111sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2 112 113before=$tmp1 114now=$tmp2 115 116 117if sort -k 2 </dev/null >/dev/null 2>&1; then 118 skip1='-k 2' 119else 120 skip1='+1' 121fi 122 123sort -t ':' $skip1 "$now" > "$now_s" 124sort -t ':' $skip1 "$before" > "$before_s" 125 126grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 127grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 128 129grep -s . $tmp2 >/dev/null 130if [ $? = 0 ]; then 131 echo "Tests that now fail, but worked before:" 132 echo 133 cat $tmp2 134 echo 135 exit_status=1 136fi 137 138grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 139grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 140 141grep -s . $tmp2 >/dev/null 142if [ $? = 0 ]; then 143 echo "Tests that now work, but didn't before:" 144 echo 145 cat $tmp2 146 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 147 echo 148fi 149 150grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 151grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 152 153grep -s . $tmp2 >/dev/null 154if [ $? = 0 ]; then 155 echo "New tests that FAIL:" 156 echo 157 cat $tmp2 158 echo 159 exit_status=1 160fi 161 162grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 163grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 164 165grep -s . $tmp2 >/dev/null 166if [ $? = 0 ]; then 167 echo "New tests that PASS:" 168 echo 169 cat $tmp2 170 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 171 echo 172fi 173 174grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 175grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 176 177grep -s . $tmp2 >/dev/null 178if [ $? = 0 ]; then 179 echo "Old tests that passed, that have disappeared: (Eeek!)" 180 echo 181 cat $tmp2 182 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 183 echo 184fi 185 186grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 187grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 188 189grep -s . $tmp2 >/dev/null 190if [ $? = 0 ]; then 191 echo "Old tests that failed, that have disappeared: (Eeek!)" 192 echo 193 cat $tmp2 194 [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 195 echo 196fi 197 198exit $exit_status 199