1#!/bin/sh 2 3# Tests a set of patches from a directory. 4# Copyright (C) 2007, 2008, 2011 Free Software Foundation, Inc. 5# Contributed by Sebastian Pop <sebastian.pop@amd.com> 6 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 3 of the License, or 10# (at your option) any later version. 11 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16 17# You should have received a copy of the GNU General Public License 18# along with this program; if not, write to the Free Software 19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 21cat <<EOF 22 23WARNING: This script should only be fed with patches from known 24 authorized and trusted sources. Don't even think about 25 hooking it up to a raw feed from the gcc-patches list or 26 you'll regret it. 27 28EOF 29 30args=$@ 31 32svnpath=svn://gcc.gnu.org/svn/gcc 33dashj= 34default_standby=1 35standby=$default_standby 36default_watermark=0.60 37watermark=$default_watermark 38savecompilers=false 39nopristinecache=false 40nogpg=false 41stop=false 42 43usage() { 44 cat <<EOF 45patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg] 46 [-svnpath URL] [-stop] [-nopristinecache] 47 <source_dir> [patches_dir [state_dir [build_dir]]] 48 49 J is the flag passed to make. Default is empty string. 50 51 STANDBY is the number of minutes between checks for new patches in 52 PATCHES_DIR. Default is ${default_standby} minutes. 53 54 WATERMARK is the 5 minute average system charge under which a new 55 compile can start. Default is ${default_watermark}. 56 57 SAVECOMPILERS copies the compilers in the same directory as the 58 test results for the non patched version. Default is not copy. 59 60 NOPRISTINECACHE prevents use of cached test results from any earlier 61 test runs on the pristine version of the branch and revision under 62 test (the default behaviour). This should be used when testing the 63 same revision and patch with multiple sets of configure options, as 64 these may affect the set of baseline failures. 65 66 NOGPG can be used to avoid checking the GPG signature of patches. 67 68 URL is the location of the GCC SVN repository. The default is 69 ${svnpath}. 70 71 STOP exits when PATCHES_DIR is empty. 72 73 SOURCE_DIR is the directory containing GCC's toplevel configure. 74 75 PATCHES_DIR is the directory containing the patches to be tested. 76 Default is SOURCE_DIR/patches. 77 78 STATE_DIR is where the tester maintains its internal state. 79 Default is SOURCE_DIR/state. 80 81 BUILD_DIR is the build tree, a temporary directory that this 82 script will delete and recreate. Default is SOURCE_DIR/obj. 83 84EOF 85 exit 1 86} 87 88makedir () { 89 DIRNAME=$1 90 mkdir -p $DIRNAME 91 if [ $? -ne 0 ]; then 92 echo "ERROR: could not make directory $DIRNAME" 93 exit 1 94 fi 95} 96 97while [ $# -ne 0 ]; do 98 case $1 in 99 -j*) 100 dashj=$1; shift 101 ;; 102 -standby) 103 [[ $# > 2 ]] || usage 104 standby=$2; shift; shift 105 ;; 106 -watermark) 107 [[ $# > 2 ]] || usage 108 watermark=$2; shift; shift 109 ;; 110 -savecompilers) 111 savecompilers=true; shift 112 ;; 113 -nopristinecache) 114 nopristinecache=true; shift 115 ;; 116 -nogpg) 117 nogpg=true; shift 118 ;; 119 -stop) 120 stop=true; shift 121 ;; 122 -svnpath) 123 svnpath=$2; shift; shift 124 ;; 125 -*) 126 echo "Invalid option: $1" 127 usage 128 ;; 129 *) 130 break 131 ;; 132 esac 133done 134 135test $# -eq 0 && usage 136 137SOURCE=$1 138PATCHES= 139STATE= 140BUILD= 141 142if [[ $# < 2 ]]; then 143 PATCHES=$SOURCE/patches 144else 145 PATCHES=$2 146fi 147if [[ $# < 3 ]]; then 148 STATE=$SOURCE/state 149else 150 STATE=$3 151fi 152if [[ $# < 4 ]]; then 153 BUILD=$SOURCE/obj 154else 155 BUILD=$4 156fi 157 158[ -d $PATCHES ] || makedir $PATCHES 159[ -d $STATE ] || makedir $STATE 160[ -d $STATE/patched ] || makedir $STATE/patched 161[ -d $SOURCE ] || makedir $SOURCE 162[ -f $SOURCE/config.guess ] || { 163 cd $SOURCE 164 svn -q co $svnpath/trunk . 165 if [ $? -ne 0 ]; then 166 echo "ERROR: initial svn checkout failed" 167 exit 1 168 fi 169} 170 171# This can contain required local settings: 172# default_config configure options, always passed 173# default_make make bootstrap options, always passed 174# default_check make check options, always passed 175[ -f $STATE/defaults ] && . $STATE/defaults 176 177VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 178 179exec >> $STATE/tester.log 2>&1 || exit 1 180set -x 181 182TESTING=$STATE/testing 183REPORT=$TESTING/report 184PRISTINE=$TESTING/pristine 185PATCHED=$TESTING/patched 186PATCH= 187TARGET=`$SOURCE/config.guess || exit 1` 188TESTLOGS="gcc/testsuite/gcc/gcc.sum 189gcc/testsuite/gfortran/gfortran.sum 190gcc/testsuite/g++/g++.sum 191gcc/testsuite/objc/objc.sum 192$TARGET/libstdc++-v3/testsuite/libstdc++.sum 193$TARGET/libffi/testsuite/libffi.sum 194$TARGET/libjava/testsuite/libjava.sum 195$TARGET/libgomp/testsuite/libgomp.sum 196$TARGET/libmudflap/testsuite/libmudflap.sum" 197COMPILERS="gcc/cc1 198gcc/cc1obj 199gcc/cc1plus 200gcc/f951 201gcc/jc1 202gcc/gnat1 203gcc/tree1" 204 205now () { 206 echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"` 207} 208 209report () { 210 echo "$@" >> $REPORT 211} 212 213freport () { 214 if [ -s $1 ]; then 215 report "(cat $1" 216 cat $1 >> $REPORT 217 report "tac)" 218 fi 219} 220 221cleanup () { 222 cd $SOURCE 223 svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v 224} 225 226selfexec () { 227 exec ${CONFIG_SHELL-/bin/sh} $0 $args 228} 229 230update () { 231 svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"` 232 if [ x$svn_branch = x ]; then 233 svn_branch=trunk 234 fi 235 236 svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"` 237 if [ x$svn_revision = x ]; then 238 svn_revision=HEAD 239 fi 240 241 cleanup 242 cd $SOURCE 243 case $svn_branch in 244 trunk) 245 if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then 246 report "failed to update svn sources with" 247 report "svn switch -r $svn_revision $svnpath/trunk" 248 freport $TESTING/svn 249 return 1 250 fi 251 ;; 252 253 ${svnpath}*) 254 if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then 255 report "failed to update svn sources with" 256 report "svn switch -r $svn_revision $svn_branch" 257 freport $TESTING/svn 258 return 1 259 fi 260 ;; 261 262 *) 263 if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then 264 report "failed to update svn sources with" 265 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch" 266 freport $TESTING/svn 267 return 1 268 fi 269 ;; 270 esac 271 contrib/gcc_update --touch 272 273 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 274 if [[ $VERSION < $current_version ]]; then 275 if [ -f $SOURCE/contrib/patch_tester.sh ]; then 276 selfexec 277 fi 278 fi 279 280 return 0 281} 282 283apply_patch () { 284 if [ $nogpg = false ]; then 285 if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then 286 report "your patch failed to verify:" 287 freport $TESTING/gpgverify 288 return 1 289 fi 290 fi 291 292 cd $SOURCE 293 if ! patch -p0 < $PATCH &> $TESTING/patching ; then 294 report "your patch failed to apply:" 295 report "(check that the patch was created at the top level)" 296 freport $TESTING/patching 297 return 1 298 fi 299 300 # Just assume indexes for now -- not really great, but svn always 301 # makes them. 302 grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do 303 # If the patch resulted in an empty file, delete it. 304 # This is how svn reports deletions. 305 if [ ! -s $file ]; then 306 rm -f $file 307 report "Deleting empty file $file" 308 fi 309 done 310} 311 312save_compilers () { 313 for COMPILER in $COMPILERS ; do 314 if [ -f $BUILD/$COMPILER ]; then 315 cp $BUILD/$COMPILER $PRISTINE 316 fi 317 done 318} 319 320bootntest () { 321 rm -rf $BUILD 322 mkdir $BUILD 323 cd $BUILD 324 325 CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"` 326 CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS" 327 if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then 328 report "configure with `basename $1` version failed with:" 329 freport $1/configure 330 return 1 331 fi 332 333 MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"` 334 MAKE_ARGS="$default_make $MAKE_ARGS" 335 if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then 336 report "bootstrap with `basename $1` version failed with last lines:" 337 tail -30 $1/bootstrap > $1/last_bootstrap 338 freport $1/last_bootstrap 339 report "grep --context=20 Error bootstrap:" 340 grep --context=20 Error $1/bootstrap > $1/bootstrap_error 341 freport $1/bootstrap_error 342 return 1 343 fi 344 345 CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"` 346 CHECK_OPTIONS="$default_check $CHECK_OPTIONS" 347 eval make $dashj $CHECK_OPTIONS -k check &> $1/check 348 349 SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`" 350 if [ x$SUITESRUN = x ]; then 351 report "check with `basename $1` version failed, no testsuites were run" 352 return 1 353 fi 354 355 for LOG in $TESTLOGS ; do 356 if [ -f $BUILD/$LOG ]; then 357 mv $BUILD/$LOG $1 358 mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1 359 fi 360 done 361 362 return 0 363} 364 365bootntest_patched () { 366 cleanup 367 mkdir -p $PATCHED 368 apply_patch && bootntest $PATCHED 369 return $? 370} 371 372# Build the pristine tree with exactly the same options as the patch under test. 373bootntest_pristine () { 374 cleanup 375 current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"` 376 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 377 PRISTINE=$STATE/$current_branch/$current_version 378 379 if [ $nopristinecache = true ]; then 380 rm -rf $PRISTINE 381 fi 382 if [ -d $PRISTINE ]; then 383 ln -s $PRISTINE $TESTING/pristine 384 return 0 385 else 386 mkdir -p $PRISTINE 387 ln -s $PRISTINE $TESTING/pristine 388 bootntest $PRISTINE 389 RETVAL=$? 390 if [ $RETVAL = 0 -a $savecompilers = true ]; then 391 save_compilers 392 fi 393 return $RETVAL 394 fi 395} 396 397regtest () { 398 touch $1/report 399 touch $1/passes 400 touch $1/failed 401 touch $1/regress 402 403 for LOG in $TESTLOGS ; do 404 NLOG=`basename $LOG` 405 if [ -f $1/$NLOG ]; then 406 awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG 407 fi 408 done | sort | uniq > $1/failed 409 410 comm -12 $1/failed $1/passes >> $1/regress 411 NUMREGRESS=`wc -l < $1/regress | tr -d ' '` 412 413 if [ $NUMREGRESS -eq 0 ] ; then 414 for LOG in $TESTLOGS ; do 415 NLOG=`basename $LOG` 416 if [ -f $1/$NLOG ] ; then 417 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG 418 fi 419 done | sort | uniq | comm -23 - $1/failed > $1/passes 420 echo "there are no regressions with your patch." >> $1/report 421 else 422 echo "with your patch there are $NUMREGRESS regressions." >> $1/report 423 echo "list of regressions with your patch:" >> $1/report 424 cat $1/regress >> $1/report 425 fi 426} 427 428contrib_compare_tests () { 429 report "comparing logs with contrib/compare_tests:" 430 for LOG in $TESTLOGS ; do 431 NLOG=`basename $LOG` 432 if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then 433 $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG 434 freport $TESTING/compare_$NLOG 435 fi 436 done 437} 438 439compare_passes () { 440 regtest $PRISTINE 441 cp $PRISTINE/passes $PATCHED 442 regtest $PATCHED 443 freport $PATCHED/report 444 report "FAILs with patched version:" 445 freport $PATCHED/failed 446 report "FAILs with pristine version:" 447 freport $PRISTINE/failed 448 449 # contrib_compare_tests 450} 451 452write_report () { 453 backup_patched=$STATE/patched/`now` 454 report "The files used for the validation of your patch are stored in $backup_patched on the tester machine." 455 456 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 457 if [ x$EMAIL != x ]; then 458 mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL 459 fi 460 461 mv $TESTING $backup_patched 462} 463 464announce () { 465 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 466 if [ x$EMAIL != x ]; then 467 468 START_REPORT=$TESTING/start_report 469 echo "Hi, " >> $START_REPORT 470 echo "I'm the automatic tester running on $TARGET." >> $START_REPORT 471 echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT 472 echo "Bye, your automatic tester." >> $START_REPORT 473 mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL 474 fi 475} 476 477# After selfexec, $TESTING is already set up. 478if [ -d $TESTING ]; then 479 # The only file in $TESTING is the patch. 480 PATCH=`ls -rt -1 $TESTING | head -1` 481 PATCH=$TESTING/$PATCH 482 if [ -f $PATCH ]; then 483 bootntest_patched && bootntest_pristine && compare_passes 484 write_report 485 fi 486fi 487 488firstpatch=true 489while true; do 490 PATCH=`ls -rt -1 $PATCHES | head -1` 491 if [ x$PATCH = x ]; then 492 if [ $stop = true ]; then 493 if [ $firstpatch = true ]; then 494 echo "No patches ready to test, quitting." 495 exit 1 496 else 497 echo "No more patches to test." 498 exit 0 499 fi 500 fi 501 sleep ${standby}m 502 else 503 firstpatch=false 504 sysload=`uptime | cut -d, -f 5` 505 if [[ $sysload > $watermark ]]; then 506 # Wait a bit when system load is too high. 507 sleep ${standby}m 508 else 509 mkdir -p $TESTING 510 mv $PATCHES/$PATCH $TESTING/ 511 PATCH=$TESTING/$PATCH 512 513 announce 514 update && bootntest_patched && bootntest_pristine && compare_passes 515 write_report 516 fi 517 fi 518done 519