1#!/bin/sh
2
3# Tests a set of patches from a directory.
4# Copyright (C) 2007, 2008, 2011  Free Software Foundation, Inc.
5# Contributed by Sebastian Pop <sebastian.pop@amd.com>
6
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 3 of the License, or
10# (at your option) any later version.
11
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16
17# You should have received a copy of the GNU General Public License
18# along with this program; if not, write to the Free Software
19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
20
21cat <<EOF
22
23WARNING: This script should only be fed with patches from known
24         authorized and trusted sources.  Don't even think about
25         hooking it up to a raw feed from the gcc-patches list or
26         you'll regret it.
27
28EOF
29
30args=$@
31
32svnpath=svn://gcc.gnu.org/svn/gcc
33dashj=
34default_standby=1
35standby=$default_standby
36default_watermark=0.60
37watermark=$default_watermark
38savecompilers=false
39nopristinecache=false
40nogpg=false
41stop=false
42
43usage() {
44    cat <<EOF
45patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
46                [-svnpath URL] [-stop] [-nopristinecache]
47                <source_dir> [patches_dir [state_dir [build_dir]]]
48
49    J is the flag passed to make.  Default is empty string.
50
51    STANDBY is the number of minutes between checks for new patches in
52    PATCHES_DIR.  Default is ${default_standby} minutes.
53
54    WATERMARK is the 5 minute average system charge under which a new
55    compile can start.  Default is ${default_watermark}.
56
57    SAVECOMPILERS copies the compilers in the same directory as the
58    test results for the non patched version.  Default is not copy.
59
60    NOPRISTINECACHE prevents use of cached test results from any earlier
61    test runs on the pristine version of the branch and revision under
62    test (the default behaviour).  This should be used when testing the
63    same revision and patch with multiple sets of configure options, as
64    these may affect the set of baseline failures.
65
66    NOGPG can be used to avoid checking the GPG signature of patches.
67
68    URL is the location of the GCC SVN repository.  The default is
69    ${svnpath}.
70
71    STOP exits when PATCHES_DIR is empty.
72
73    SOURCE_DIR is the directory containing GCC's toplevel configure.
74
75    PATCHES_DIR is the directory containing the patches to be tested.
76    Default is SOURCE_DIR/patches.
77
78    STATE_DIR is where the tester maintains its internal state.
79    Default is SOURCE_DIR/state.
80
81    BUILD_DIR is the build tree, a temporary directory that this
82    script will delete and recreate.  Default is SOURCE_DIR/obj.
83
84EOF
85    exit 1
86}
87
88makedir () {
89    DIRNAME=$1
90    mkdir -p $DIRNAME
91    if [ $? -ne 0 ]; then
92	echo "ERROR: could not make directory $DIRNAME"
93	exit 1
94    fi
95}
96
97while [ $# -ne 0 ]; do
98    case $1 in
99	-j*)
100	    dashj=$1; shift
101	    ;;
102	-standby)
103	    [[ $# > 2 ]] || usage
104	    standby=$2; shift; shift
105	    ;;
106	-watermark)
107	    [[ $# > 2 ]] || usage
108	    watermark=$2; shift; shift
109	    ;;
110	-savecompilers)
111	    savecompilers=true; shift
112	    ;;
113	-nopristinecache)
114	    nopristinecache=true; shift
115	    ;;
116	-nogpg)
117	    nogpg=true; shift
118	    ;;
119	-stop)
120	    stop=true; shift
121	    ;;
122	-svnpath)
123	    svnpath=$2; shift; shift
124	    ;;
125	-*) 
126	    echo "Invalid option: $1"
127	    usage
128	    ;;
129	*)
130	    break
131	    ;;
132    esac
133done
134
135test $# -eq 0 && usage
136
137SOURCE=$1
138PATCHES=
139STATE=
140BUILD=
141
142if [[ $# < 2 ]]; then
143    PATCHES=$SOURCE/patches
144else
145    PATCHES=$2
146fi
147if [[ $# < 3 ]]; then
148    STATE=$SOURCE/state
149else
150    STATE=$3
151fi
152if [[ $# < 4 ]]; then
153    BUILD=$SOURCE/obj
154else
155    BUILD=$4
156fi
157
158[ -d $PATCHES ] || makedir $PATCHES
159[ -d $STATE ] || makedir $STATE
160[ -d $STATE/patched ] || makedir $STATE/patched
161[ -d $SOURCE ] || makedir $SOURCE
162[ -f $SOURCE/config.guess ] || {
163    cd $SOURCE
164    svn -q co $svnpath/trunk .
165    if [ $? -ne 0 ]; then
166	echo "ERROR: initial svn checkout failed"
167	exit 1
168    fi
169}
170
171# This can contain required local settings:
172#  default_config  configure options, always passed
173#  default_make    make bootstrap options, always passed
174#  default_check   make check options, always passed
175[ -f $STATE/defaults ] && . $STATE/defaults
176
177VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
178
179exec >> $STATE/tester.log 2>&1 || exit 1
180set -x
181
182TESTING=$STATE/testing
183REPORT=$TESTING/report
184PRISTINE=$TESTING/pristine
185PATCHED=$TESTING/patched
186PATCH=
187TARGET=`$SOURCE/config.guess || exit 1` 
188TESTLOGS="gcc/testsuite/gcc/gcc.sum
189gcc/testsuite/gfortran/gfortran.sum
190gcc/testsuite/g++/g++.sum
191gcc/testsuite/objc/objc.sum
192$TARGET/libstdc++-v3/testsuite/libstdc++.sum
193$TARGET/libffi/testsuite/libffi.sum
194$TARGET/libjava/testsuite/libjava.sum
195$TARGET/libgomp/testsuite/libgomp.sum
196$TARGET/libmudflap/testsuite/libmudflap.sum"
197COMPILERS="gcc/cc1
198gcc/cc1obj
199gcc/cc1plus
200gcc/f951
201gcc/jc1
202gcc/gnat1
203gcc/tree1"
204
205now () {
206    echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
207}
208
209report () {
210    echo "$@" >> $REPORT
211}
212
213freport () {
214    if [ -s $1 ]; then
215	report "(cat $1"
216	cat $1 >> $REPORT
217	report "tac)"
218    fi
219}
220
221cleanup () {
222    cd $SOURCE
223    svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
224}
225
226selfexec () {
227    exec ${CONFIG_SHELL-/bin/sh} $0 $args
228}
229
230update () {
231    svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
232    if [ x$svn_branch = x ]; then
233	svn_branch=trunk
234    fi
235
236    svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
237    if [ x$svn_revision = x ]; then
238	svn_revision=HEAD
239    fi
240
241    cleanup
242    cd $SOURCE
243    case $svn_branch in
244	trunk)
245	    if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
246		report "failed to update svn sources with"
247		report "svn switch -r $svn_revision $svnpath/trunk"
248		freport $TESTING/svn
249		return 1
250	    fi
251	    ;;
252
253	${svnpath}*)
254	    if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
255		report "failed to update svn sources with"
256		report "svn switch -r $svn_revision $svn_branch"
257		freport $TESTING/svn
258		return 1
259	    fi
260	    ;;
261
262	*)
263	    if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
264		report "failed to update svn sources with"
265		report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
266		freport $TESTING/svn
267		return 1
268	    fi
269	    ;;
270    esac
271    contrib/gcc_update --touch
272
273    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
274    if [[ $VERSION < $current_version ]]; then
275	if [ -f $SOURCE/contrib/patch_tester.sh ]; then
276	    selfexec
277	fi
278    fi
279
280    return 0
281}
282
283apply_patch () {
284    if [ $nogpg = false ]; then
285	if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
286	    report "your patch failed to verify:"
287	    freport $TESTING/gpgverify
288	    return 1
289	fi
290    fi
291
292    cd $SOURCE
293    if ! patch -p0 < $PATCH &> $TESTING/patching ; then
294	report "your patch failed to apply:"
295	report "(check that the patch was created at the top level)"
296	freport $TESTING/patching
297	return 1
298    fi
299
300    # Just assume indexes for now -- not really great, but svn always
301    # makes them.
302    grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
303	# If the patch resulted in an empty file, delete it.
304	# This is how svn reports deletions.
305	if [ ! -s $file ]; then
306	    rm -f $file
307	    report "Deleting empty file $file"
308	fi
309    done
310}
311
312save_compilers () {
313    for COMPILER in $COMPILERS ; do
314	if [ -f $BUILD/$COMPILER ]; then
315	    cp $BUILD/$COMPILER $PRISTINE
316	fi
317    done
318}
319
320bootntest () {
321    rm -rf $BUILD
322    mkdir $BUILD
323    cd $BUILD
324
325    CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
326    CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
327    if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
328	report "configure with `basename $1` version failed with:"
329	freport $1/configure
330	return 1
331    fi
332
333    MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
334    MAKE_ARGS="$default_make $MAKE_ARGS"
335    if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
336	report "bootstrap with `basename $1` version failed with last lines:"
337	tail -30 $1/bootstrap > $1/last_bootstrap
338	freport $1/last_bootstrap
339	report "grep --context=20 Error bootstrap:"
340	grep --context=20 Error $1/bootstrap > $1/bootstrap_error
341	freport $1/bootstrap_error
342	return 1
343    fi
344
345    CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
346    CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
347    eval make $dashj $CHECK_OPTIONS -k check &> $1/check
348
349    SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
350    if [ x$SUITESRUN = x ]; then
351	report "check with `basename $1` version failed, no testsuites were run"
352	return 1
353    fi
354
355    for LOG in $TESTLOGS ; do
356	if [ -f $BUILD/$LOG ]; then
357	    mv $BUILD/$LOG $1
358	    mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
359	fi
360    done
361
362    return 0
363}
364
365bootntest_patched () {
366    cleanup
367    mkdir -p $PATCHED
368    apply_patch && bootntest $PATCHED
369    return $?
370}
371
372# Build the pristine tree with exactly the same options as the patch under test.
373bootntest_pristine () {
374    cleanup
375    current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
376    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
377    PRISTINE=$STATE/$current_branch/$current_version
378
379    if [ $nopristinecache = true ]; then
380      rm -rf $PRISTINE
381    fi
382    if [ -d $PRISTINE ]; then
383	ln -s $PRISTINE $TESTING/pristine
384	return 0
385    else
386	mkdir -p $PRISTINE
387	ln -s $PRISTINE $TESTING/pristine
388	bootntest $PRISTINE
389	RETVAL=$?
390	if [ $RETVAL = 0 -a $savecompilers = true ]; then
391	    save_compilers
392	fi
393	return $RETVAL
394    fi
395}
396
397regtest () {
398    touch $1/report
399    touch $1/passes
400    touch $1/failed
401    touch $1/regress
402
403    for LOG in $TESTLOGS ; do
404	NLOG=`basename $LOG`
405	if [ -f $1/$NLOG ]; then
406	    awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
407	fi
408    done | sort | uniq > $1/failed
409
410    comm -12 $1/failed $1/passes >> $1/regress
411    NUMREGRESS=`wc -l < $1/regress | tr -d ' '`
412
413    if [ $NUMREGRESS -eq 0 ] ; then
414	for LOG in $TESTLOGS ; do
415	    NLOG=`basename $LOG`
416	    if [ -f $1/$NLOG ] ; then
417		awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
418	    fi
419	done | sort | uniq | comm -23 - $1/failed > $1/passes
420	echo "there are no regressions with your patch." >> $1/report
421    else
422	echo "with your patch there are $NUMREGRESS regressions." >> $1/report
423	echo "list of regressions with your patch:" >> $1/report
424	cat $1/regress >> $1/report
425    fi
426}
427
428contrib_compare_tests () {
429    report "comparing logs with contrib/compare_tests:"
430    for LOG in $TESTLOGS ; do
431 	NLOG=`basename $LOG`
432 	if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
433 	    $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
434 	    freport $TESTING/compare_$NLOG
435 	fi
436    done
437}
438
439compare_passes () {
440    regtest $PRISTINE
441    cp $PRISTINE/passes $PATCHED
442    regtest $PATCHED
443    freport $PATCHED/report
444    report "FAILs with patched version:"
445    freport $PATCHED/failed
446    report "FAILs with pristine version:"
447    freport $PRISTINE/failed
448
449    # contrib_compare_tests
450}
451
452write_report () {
453    backup_patched=$STATE/patched/`now`
454    report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."
455
456    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
457    if [ x$EMAIL != x ]; then
458	mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
459    fi
460
461    mv $TESTING $backup_patched
462}
463
464announce () {
465    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
466    if [ x$EMAIL != x ]; then
467
468	START_REPORT=$TESTING/start_report
469	echo "Hi, " >> $START_REPORT
470	echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
471	echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
472	echo "Bye, your automatic tester." >> $START_REPORT
473	mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
474    fi
475}
476
477# After selfexec, $TESTING is already set up.  
478if [ -d $TESTING ]; then
479    # The only file in $TESTING is the patch.
480    PATCH=`ls -rt -1 $TESTING | head -1`
481    PATCH=$TESTING/$PATCH
482    if [ -f $PATCH ]; then
483	bootntest_patched && bootntest_pristine && compare_passes
484	write_report
485    fi
486fi
487
488firstpatch=true
489while true; do
490    PATCH=`ls -rt -1 $PATCHES | head -1`
491    if [ x$PATCH = x ]; then
492	if [ $stop = true ]; then
493	    if [ $firstpatch = true ]; then
494		echo "No patches ready to test, quitting."
495		exit 1
496	    else
497		echo "No more patches to test."
498		exit 0
499	    fi
500	fi
501	sleep ${standby}m
502    else
503	firstpatch=false
504	sysload=`uptime | cut -d, -f 5`
505	if [[ $sysload > $watermark ]]; then
506	    # Wait a bit when system load is too high.
507	    sleep ${standby}m
508	else
509	    mkdir -p $TESTING
510	    mv $PATCHES/$PATCH $TESTING/
511	    PATCH=$TESTING/$PATCH
512
513	    announce
514	    update && bootntest_patched && bootntest_pristine && compare_passes
515	    write_report
516	fi
517    fi
518done
519