hgforest.sh revision 680:575f2ca947ab
1177633Sdfr#!/bin/sh 2177633Sdfr 3261057Smav# 4261057Smav# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. 5261057Smav# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6177633Sdfr# 7261057Smav# This code is free software; you can redistribute it and/or modify it 8261057Smav# under the terms of the GNU General Public License version 2 only, as 9261057Smav# published by the Free Software Foundation. 10261057Smav# 11261057Smav# This code is distributed in the hope that it will be useful, but WITHOUT 12261057Smav# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13261057Smav# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14261057Smav# version 2 for more details (a copy is included in the LICENSE file that 15261057Smav# accompanied this code). 16261057Smav# 17261057Smav# You should have received a copy of the GNU General Public License version 18261057Smav# 2 along with this work; if not, write to the Free Software Foundation, 19261057Smav# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20261057Smav# 21261057Smav# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22261057Smav# or visit www.oracle.com if you need additional information or have any 23261057Smav# questions. 24261057Smav# 25261057Smav 26261057Smav# Shell script for a fast parallel forest command 27261057Smavcommand="$1" 28261057Smavpull_extra_base="$2" 29177633Sdfr 30177633Sdfr# Python always buffers stdout significantly, thus we will not see any output from hg clone jdk, 31177633Sdfr# until a lot of time has passed! By passing -u to python, we get incremental updates 32177633Sdfr# on stdout. Much nicer. 33177633Sdfrwhichhg="`which hg 2> /dev/null | grep -v '^no hg in'`" 34177633Sdfr 35177633Sdfrif [ "${whichhg}" = "" ] ; then 36177633Sdfr echo Cannot find hg! 37177633Sdfr exit 1 38177633Sdfrfi 39177633Sdfr 40177633Sdfrif [ "" = "$command" ] ; then 41177633Sdfr echo No command to hg supplied! 42177633Sdfr exit 1 43177633Sdfrfi 44177633Sdfr 45177633Sdfrhas_hash_bang="`head -n 1 "${whichhg}" | cut -b 1-2`" 46177633Sdfrpython="" 47177633Sdfrbpython="" 48177633Sdfr 49177633Sdfrif [ "#!" = "$has_hash_bang" ] ; then 50177633Sdfr python="`head -n 1 ${whichhg} | cut -b 3-`" 51177633Sdfr bpython="`basename "$python"`" 52177633Sdfrfi 53177633Sdfr 54177633Sdfrif [ -x "$python" -a ! -d "$python" -a "`${python} -V 2>&1 | cut -f 1 -d ' '`" = "Python" ] ; then 55177633Sdfr hg="${python} -u ${whichhg}" 56177633Sdfrelse 57177633Sdfr echo Cannot find python from hg launcher. Running plain hg, which probably has buffered stdout. 58177633Sdfr hg="hg" 59177633Sdfrfi 60177633Sdfr 61177633Sdfr# Clean out the temporary directory that stores the pid files. 62177633Sdfrtmp=/tmp/forest.$$ 63177633Sdfrrm -f -r ${tmp} 64177633Sdfrmkdir -p ${tmp} 65177633Sdfr 66177633Sdfrsafe_interrupt () { 67177633Sdfr if [ -d ${tmp} ]; then 68177633Sdfr if [ "`ls ${tmp}/*.pid`" != "" ]; then 69177633Sdfr echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" 70177633Sdfr sleep 1 71177633Sdfr # Pipe stderr to dev/null to silence kill, that complains when trying to kill 72177633Sdfr # a subprocess that has already exited. 73177633Sdfr kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null 74177633Sdfr wait 75177633Sdfr echo Interrupt complete! 76177633Sdfr fi 77177633Sdfr fi 78177633Sdfr rm -f -r ${tmp} 79177633Sdfr exit 1 80177633Sdfr} 81177633Sdfr 82177633Sdfrnice_exit () { 83177633Sdfr if [ -d ${tmp} ]; then 84177633Sdfr if [ "`ls ${tmp}`" != "" ]; then 85177633Sdfr wait 86177633Sdfr fi 87177633Sdfr fi 88177633Sdfr rm -f -r ${tmp} 89177633Sdfr} 90177633Sdfr 91177633Sdfrtrap 'safe_interrupt' INT QUIT 92177633Sdfrtrap 'nice_exit' EXIT 93177633Sdfr 94177633Sdfr# Only look in specific locations for possible forests (avoids long searches) 95177633Sdfrpull_default="" 96177633Sdfrrepos="" 97177633Sdfrrepos_extra="" 98177633Sdfrif [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 99177633Sdfr subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" 100177633Sdfr if [ -f .hg/hgrc ] ; then 101177633Sdfr pull_default=`hg paths default` 102177633Sdfr if [ "${pull_default}" = "" ] ; then 103177633Sdfr echo "ERROR: Need initial clone with 'hg paths default' defined" 104177633Sdfr exit 1 105177633Sdfr fi 106177633Sdfr fi 107177633Sdfr if [ "${pull_default}" = "" ] ; then 108177633Sdfr echo "ERROR: Need initial repository to use this script" 109177633Sdfr exit 1 110177633Sdfr fi 111177633Sdfr for i in ${subrepos} ; do 112177633Sdfr if [ ! -f ${i}/.hg/hgrc ] ; then 113177633Sdfr repos="${repos} ${i}" 114177633Sdfr fi 115177633Sdfr done 116177633Sdfr if [ "${pull_extra_base}" != "" ] ; then 117177633Sdfr subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" 118177633Sdfr pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` 119177633Sdfr pull_extra="${pull_extra_base}/${pull_default_tail}" 120177633Sdfr for i in ${subrepos_extra} ; do 121177633Sdfr if [ ! -f ${i}/.hg/hgrc ] ; then 122177633Sdfr repos_extra="${repos_extra} ${i}" 123177633Sdfr fi 124177633Sdfr done 125177633Sdfr fi 126177633Sdfr at_a_time=2 127177633Sdfr # Any repos to deal with? 128177633Sdfr if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then 129177633Sdfr exit 130177633Sdfr fi 131177633Sdfrelse 132177633Sdfr hgdirs=`ls -d ./.hg ./*/.hg ./*/*/.hg ./*/*/*/.hg ./*/*/*/*/.hg 2>/dev/null` 133177633Sdfr # Derive repository names from the .hg directory locations 134177633Sdfr for i in ${hgdirs} ; do 135177633Sdfr repos="${repos} `echo ${i} | sed -e 's@/.hg$@@'`" 136177633Sdfr done 137177633Sdfr for i in ${repos} ; do 138177633Sdfr if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then 139177633Sdfr locked="${i} ${locked}" 140177633Sdfr fi 141177633Sdfr done 142177633Sdfr at_a_time=8 143177633Sdfr # Any repos to deal with? 144177633Sdfr if [ "${repos}" = "" ] ; then 145177633Sdfr echo "No repositories to process." 146177633Sdfr exit 147177633Sdfr fi 148177633Sdfr if [ "${locked}" != "" ] ; then 149177633Sdfr echo "These repositories are locked: ${locked}" 150177633Sdfr exit 151177633Sdfr fi 152177633Sdfrfi 153177633Sdfr 154177633Sdfr# Echo out what repositories we do a command on. 155177633Sdfrecho "# Repositories: ${repos} ${repos_extra}" 156177633Sdfrecho 157177633Sdfr 158177633Sdfr# Run the supplied command on all repos in parallel. 159177633Sdfrn=0 160177633Sdfrfor i in ${repos} ${repos_extra} ; do 161177633Sdfr n=`expr ${n} '+' 1` 162177633Sdfr repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` 163177633Sdfr reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` 164177633Sdfr pull_base="${pull_default}" 165177633Sdfr for j in $repos_extra ; do 166177633Sdfr if [ "$i" = "$j" ] ; then 167177633Sdfr pull_base="${pull_extra}" 168177633Sdfr fi 169177633Sdfr done 170177633Sdfr ( 171177633Sdfr ( 172177633Sdfr if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 173177633Sdfr pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" 174177633Sdfr echo ${hg} clone ${pull_newrepo} ${i} 175177633Sdfr path="`dirname ${i}`" 176177633Sdfr if [ "${path}" != "." ] ; then 177177633Sdfr times=0 178177633Sdfr while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists 179177633Sdfr do 180177633Sdfr times=`expr ${times} '+' 1` 181177633Sdfr if [ `expr ${times} '%' 10` -eq 0 ] ; then 182177633Sdfr echo ${path} still not created, waiting... 183177633Sdfr fi 184177633Sdfr sleep 5 185177633Sdfr done 186177633Sdfr fi 187177633Sdfr (${hg} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 188177633Sdfr else 189177633Sdfr echo "cd ${i} && ${hg} $*" 190177633Sdfr cd ${i} && (${hg} "$@"; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 191177633Sdfr fi 192177633Sdfr echo $! > ${tmp}/${repopidfile}.pid 193177633Sdfr ) 2>&1 | sed -e "s@^@${reponame}: @") & 194177633Sdfr 195177633Sdfr if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then 196177633Sdfr sleep 2 197177633Sdfr echo Waiting 5 secs before spawning next background command. 198177633Sdfr sleep 3 199177633Sdfr fi 200177633Sdfrdone 201177633Sdfr# Wait for all hg commands to complete 202177633Sdfrwait 203177633Sdfr 204177633Sdfr# Terminate with exit 0 only if all subprocesses were successful 205177633Sdfrec=0 206177633Sdfrif [ -d ${tmp} ]; then 207177633Sdfr for rc in ${tmp}/*.pid.rc ; do 208177633Sdfr exit_code=`cat ${rc} | tr -d ' \n\r'` 209177633Sdfr if [ "${exit_code}" != "0" ] ; then 210184588Sdfr echo "WARNING: ${rc} exited abnormally." 211177633Sdfr ec=1 212177633Sdfr fi 213177633Sdfr done 214fi 215exit ${ec} 216