1148871Scperciva#!/bin/sh 2148871Scperciva 3148871Scperciva#- 4148871Scperciva# Copyright 2004-2005 Colin Percival 5148871Scperciva# All rights reserved 6148871Scperciva# 7148871Scperciva# Redistribution and use in source and binary forms, with or without 8148871Scperciva# modification, are permitted providing that the following conditions 9148871Scperciva# are met: 10148871Scperciva# 1. Redistributions of source code must retain the above copyright 11148871Scperciva# notice, this list of conditions and the following disclaimer. 12148871Scperciva# 2. Redistributions in binary form must reproduce the above copyright 13148871Scperciva# notice, this list of conditions and the following disclaimer in the 14148871Scperciva# documentation and/or other materials provided with the distribution. 15148871Scperciva# 16148871Scperciva# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17148871Scperciva# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18148871Scperciva# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19148871Scperciva# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20148871Scperciva# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21148871Scperciva# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22148871Scperciva# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23148871Scperciva# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24148871Scperciva# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25148871Scperciva# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26148871Scperciva# POSSIBILITY OF SUCH DAMAGE. 27148871Scperciva 28148871Scperciva# $FreeBSD: releng/10.1/usr.sbin/portsnap/portsnap/portsnap.sh 306941 2016-10-10 07:18:54Z delphij $ 29148871Scperciva 30148871Scperciva#### Usage function -- called from command-line handling code. 31148871Scperciva 32148871Scperciva# Usage instructions. Options not listed: 33148871Scperciva# --debug -- don't filter output from utilities 34148871Scperciva# --no-stats -- don't show progress statistics while fetching files 35148871Scpercivausage() { 36148871Scperciva cat <<EOF 37149027Scpercivausage: `basename $0` [options] command ... [path] 38148871Scperciva 39148871ScpercivaOptions: 40148871Scperciva -d workdir -- Store working files in workdir 41148871Scperciva (default: /var/db/portsnap/) 42148871Scperciva -f conffile -- Read configuration options from conffile 43148871Scperciva (default: /etc/portsnap.conf) 44148871Scperciva -I -- Update INDEX only. (update command only) 45148871Scperciva -k KEY -- Trust an RSA key with SHA256 hash of KEY 46158523Scperciva -l descfile -- Merge the specified local describes file into the INDEX. 47148871Scperciva -p portsdir -- Location of uncompressed ports tree 48148871Scperciva (default: /usr/ports/) 49148871Scperciva -s server -- Server from which to fetch updates. 50148871Scperciva (default: portsnap.FreeBSD.org) 51239078Seadler --interactive -- interactive: override auto-detection of calling process 52239078Seadler (use this when calling portsnap from an interactive, non- 53239078Seadler terminal application AND NEVER ELSE). 54148871Scperciva path -- Extract only parts of the tree starting with the given 55148871Scperciva string. (extract command only) 56148871ScpercivaCommands: 57148871Scperciva fetch -- Fetch a compressed snapshot of the ports tree, 58148871Scperciva or update an existing snapshot. 59148871Scperciva cron -- Sleep rand(3600) seconds, and then fetch updates. 60148871Scperciva extract -- Extract snapshot of ports tree, replacing existing 61148871Scperciva files and directories. 62148871Scperciva update -- Update ports tree to match current snapshot, replacing 63148871Scperciva files and directories which have changed. 64148871ScpercivaEOF 65148871Scperciva exit 0 66148871Scperciva} 67148871Scperciva 68148871Scperciva#### Parameter handling functions. 69148871Scperciva 70148871Scperciva# Initialize parameters to null, just in case they're 71148871Scperciva# set in the environment. 72148871Scpercivainit_params() { 73148871Scperciva KEYPRINT="" 74148871Scperciva EXTRACTPATH="" 75148871Scperciva WORKDIR="" 76148871Scperciva PORTSDIR="" 77148871Scperciva CONFFILE="" 78148871Scperciva COMMAND="" 79149027Scperciva COMMANDS="" 80148871Scperciva QUIETREDIR="" 81148871Scperciva QUIETFLAG="" 82148871Scperciva STATSREDIR="" 83148879Scperciva XARGST="" 84148871Scperciva NDEBUG="" 85148871Scperciva DDSTATS="" 86148871Scperciva INDEXONLY="" 87148871Scperciva SERVERNAME="" 88149824Scperciva REFUSE="" 89158523Scperciva LOCALDESC="" 90239078Seadler INTERACTIVE="" 91148871Scperciva} 92148871Scperciva 93148871Scperciva# Parse the command line 94148871Scpercivaparse_cmdline() { 95148871Scperciva while [ $# -gt 0 ]; do 96148871Scperciva case "$1" in 97148871Scperciva -d) 98148871Scperciva if [ $# -eq 1 ]; then usage; fi 99148871Scperciva if [ ! -z "${WORKDIR}" ]; then usage; fi 100148871Scperciva shift; WORKDIR="$1" 101148871Scperciva ;; 102148871Scperciva --debug) 103148871Scperciva QUIETREDIR="/dev/stderr" 104148871Scperciva STATSREDIR="/dev/stderr" 105148871Scperciva QUIETFLAG=" " 106148871Scperciva NDEBUG=" " 107148879Scperciva XARGST="-t" 108148871Scperciva DDSTATS=".." 109148871Scperciva ;; 110239078Seadler --interactive) 111239078Seadler INTERACTIVE="YES" 112239078Seadler ;; 113148871Scperciva -f) 114148871Scperciva if [ $# -eq 1 ]; then usage; fi 115148871Scperciva if [ ! -z "${CONFFILE}" ]; then usage; fi 116148871Scperciva shift; CONFFILE="$1" 117148871Scperciva ;; 118148871Scperciva -h | --help | help) 119148871Scperciva usage 120148871Scperciva ;; 121148871Scperciva -I) 122148871Scperciva INDEXONLY="YES" 123148871Scperciva ;; 124148871Scperciva -k) 125148871Scperciva if [ $# -eq 1 ]; then usage; fi 126148871Scperciva if [ ! -z "${KEYPRINT}" ]; then usage; fi 127148871Scperciva shift; KEYPRINT="$1" 128148871Scperciva ;; 129158523Scperciva -l) 130158523Scperciva if [ $# -eq 1 ]; then usage; fi 131158523Scperciva if [ ! -z "${LOCALDESC}" ]; then usage; fi 132158523Scperciva shift; LOCALDESC="$1" 133158523Scperciva ;; 134148871Scperciva --no-stats) 135148871Scperciva if [ -z "${STATSREDIR}" ]; then 136148871Scperciva STATSREDIR="/dev/null" 137148871Scperciva DDSTATS=".. " 138148871Scperciva fi 139148871Scperciva ;; 140148871Scperciva -p) 141148871Scperciva if [ $# -eq 1 ]; then usage; fi 142148871Scperciva if [ ! -z "${PORTSDIR}" ]; then usage; fi 143148871Scperciva shift; PORTSDIR="$1" 144148871Scperciva ;; 145148871Scperciva -s) 146148871Scperciva if [ $# -eq 1 ]; then usage; fi 147148871Scperciva if [ ! -z "${SERVERNAME}" ]; then usage; fi 148148871Scperciva shift; SERVERNAME="$1" 149148871Scperciva ;; 150201251Scperciva cron | extract | fetch | update | alfred) 151149027Scperciva COMMANDS="${COMMANDS} $1" 152148871Scperciva ;; 153235208Seadler up) 154235208Seadler COMMANDS="${COMMANDS} update" 155235208Seadler ;; 156148871Scperciva *) 157148871Scperciva if [ $# -gt 1 ]; then usage; fi 158149027Scperciva if echo ${COMMANDS} | grep -vq extract; then 159149027Scperciva usage 160149027Scperciva fi 161148871Scperciva EXTRACTPATH="$1" 162148871Scperciva ;; 163148871Scperciva esac 164148871Scperciva shift 165148871Scperciva done 166148871Scperciva 167149027Scperciva if [ -z "${COMMANDS}" ]; then 168148871Scperciva usage 169148871Scperciva fi 170148871Scperciva} 171148871Scperciva 172148871Scperciva# If CONFFILE was specified at the command-line, make 173148871Scperciva# sure that it exists and is readable. 174148871Scpercivasanity_conffile() { 175148871Scperciva if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 176148871Scperciva echo -n "File does not exist " 177148871Scperciva echo -n "or is not readable: " 178148871Scperciva echo ${CONFFILE} 179148871Scperciva exit 1 180148871Scperciva fi 181148871Scperciva} 182148871Scperciva 183148871Scperciva# If a configuration file hasn't been specified, use 184148871Scperciva# the default value (/etc/portsnap.conf) 185148871Scpercivadefault_conffile() { 186148871Scperciva if [ -z "${CONFFILE}" ]; then 187148871Scperciva CONFFILE="/etc/portsnap.conf" 188148871Scperciva fi 189148871Scperciva} 190148871Scperciva 191148871Scperciva# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 192148871Scperciva# file if they haven't already been set. If the configuration 193148871Scperciva# file doesn't exist, do nothing. 194149824Scperciva# Also read REFUSE (which cannot be set via the command line) if it is 195149824Scperciva# present in the configuration file. 196148871Scpercivaparse_conffile() { 197148871Scperciva if [ -r "${CONFFILE}" ]; then 198148871Scperciva for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 199148871Scperciva eval _=\$${X} 200148871Scperciva if [ -z "${_}" ]; then 201148871Scperciva eval ${X}=`grep "^${X}=" "${CONFFILE}" | 202148871Scperciva cut -f 2- -d '=' | tail -1` 203148871Scperciva fi 204148871Scperciva done 205149824Scperciva 206149824Scperciva if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 207149824Scperciva REFUSE="^(` 208149824Scperciva grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 209149824Scperciva cut -c 7- | xargs echo | tr ' ' '|' 210149824Scperciva `)" 211149824Scperciva fi 212179073Scperciva 213179073Scperciva if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 214179073Scperciva INDEXPAIRS="` 215179073Scperciva grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 216179073Scperciva cut -c 7- | tr ' ' '|' | xargs echo`" 217179073Scperciva fi 218148871Scperciva fi 219148871Scperciva} 220148871Scperciva 221148871Scperciva# If parameters have not been set, use default values 222148871Scpercivadefault_params() { 223148871Scperciva _QUIETREDIR="/dev/null" 224148871Scperciva _QUIETFLAG="-q" 225148871Scperciva _STATSREDIR="/dev/stdout" 226148871Scperciva _WORKDIR="/var/db/portsnap" 227148871Scperciva _PORTSDIR="/usr/ports" 228148871Scperciva _NDEBUG="-n" 229158523Scperciva _LOCALDESC="/dev/null" 230158523Scperciva for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 231158523Scperciva NDEBUG LOCALDESC; do 232148871Scperciva eval _=\$${X} 233148871Scperciva eval __=\$_${X} 234148871Scperciva if [ -z "${_}" ]; then 235148871Scperciva eval ${X}=${__} 236148871Scperciva fi 237148871Scperciva done 238239078Seadler if [ -z "${INTERACTIVE}" ]; then 239239078Seadler if [ -t 0 ]; then 240239078Seadler INTERACTIVE="YES" 241239078Seadler else 242239078Seadler INTERACTIVE="NO" 243239078Seadler fi 244239078Seadler fi 245148871Scperciva} 246148871Scperciva 247148871Scperciva# Perform sanity checks and set some final parameters 248148871Scperciva# in preparation for fetching files. Also chdir into 249148871Scperciva# the working directory. 250148871Scpercivafetch_check_params() { 251148871Scperciva export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 252148871Scperciva 253148871Scperciva _SERVERNAME_z=\ 254148871Scperciva"SERVERNAME must be given via command line or configuration file." 255148871Scperciva _KEYPRINT_z="Key must be given via -k option or configuration file." 256148871Scperciva _KEYPRINT_bad="Invalid key fingerprint: " 257148871Scperciva _WORKDIR_bad="Directory does not exist or is not writable: " 258148871Scperciva 259148871Scperciva if [ -z "${SERVERNAME}" ]; then 260148871Scperciva echo -n "`basename $0`: " 261148871Scperciva echo "${_SERVERNAME_z}" 262148871Scperciva exit 1 263148871Scperciva fi 264148871Scperciva if [ -z "${KEYPRINT}" ]; then 265148871Scperciva echo -n "`basename $0`: " 266148871Scperciva echo "${_KEYPRINT_z}" 267148871Scperciva exit 1 268148871Scperciva fi 269148871Scperciva if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 270148871Scperciva echo -n "`basename $0`: " 271148871Scperciva echo -n "${_KEYPRINT_bad}" 272148871Scperciva echo ${KEYPRINT} 273148871Scperciva exit 1 274148871Scperciva fi 275148871Scperciva if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 276148871Scperciva echo -n "`basename $0`: " 277148871Scperciva echo -n "${_WORKDIR_bad}" 278148871Scperciva echo ${WORKDIR} 279148871Scperciva exit 1 280148871Scperciva fi 281148871Scperciva cd ${WORKDIR} || exit 1 282148871Scperciva 283148871Scperciva BSPATCH=/usr/bin/bspatch 284148871Scperciva SHA256=/sbin/sha256 285148871Scperciva PHTTPGET=/usr/libexec/phttpget 286148871Scperciva} 287148871Scperciva 288148871Scperciva# Perform sanity checks and set some final parameters 289148871Scperciva# in preparation for extracting or updating ${PORTSDIR} 290154088Scperciva# Complain if ${PORTSDIR} exists but is not writable, 291154088Scperciva# but don't complain if ${PORTSDIR} doesn't exist. 292148871Scpercivaextract_check_params() { 293148871Scperciva _WORKDIR_bad="Directory does not exist: " 294154088Scperciva _PORTSDIR_bad="Directory is not writable: " 295148871Scperciva 296148871Scperciva if ! [ -d "${WORKDIR}" ]; then 297148871Scperciva echo -n "`basename $0`: " 298148871Scperciva echo -n "${_WORKDIR_bad}" 299148871Scperciva echo ${WORKDIR} 300148871Scperciva exit 1 301148871Scperciva fi 302154088Scperciva if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 303148871Scperciva echo -n "`basename $0`: " 304148871Scperciva echo -n "${_PORTSDIR_bad}" 305148871Scperciva echo ${PORTSDIR} 306148871Scperciva exit 1 307148871Scperciva fi 308148871Scperciva 309148871Scperciva if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 310148871Scperciva -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 311148871Scperciva echo "No snapshot available. Try running" 312148871Scperciva echo "# `basename $0` fetch" 313148871Scperciva exit 1 314148871Scperciva fi 315148871Scperciva 316148871Scperciva MKINDEX=/usr/libexec/make_index 317148871Scperciva} 318148871Scperciva 319148871Scperciva# Perform sanity checks and set some final parameters 320148871Scperciva# in preparation for updating ${PORTSDIR} 321148871Scpercivaupdate_check_params() { 322148871Scperciva extract_check_params 323148871Scperciva 324148871Scperciva if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 325148871Scperciva echo "${PORTSDIR} was not created by portsnap." 326148871Scperciva echo -n "You must run '`basename $0` extract' before " 327148871Scperciva echo "running '`basename $0` update'." 328148871Scperciva exit 1 329148871Scperciva fi 330148871Scperciva 331148871Scperciva} 332148871Scperciva 333148871Scperciva#### Core functionality -- the actual work gets done here 334148871Scperciva 335148871Scperciva# Use an SRV query to pick a server. If the SRV query doesn't provide 336148871Scperciva# a useful answer, use the server name specified by the user. 337148871Scperciva# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 338148871Scperciva# from that; or if no servers are returned, use ${SERVERNAME}. 339148871Scperciva# This allows a user to specify "portsnap.freebsd.org" (in which case 340148871Scperciva# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 341148871Scperciva# (in which case portsnap will use that particular server, since there 342148871Scperciva# won't be an SRV entry for that name). 343148871Scperciva# 344158273Scperciva# We ignore the Port field, since we are always going to use port 80. 345158273Scperciva 346158273Scperciva# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 347158273Scperciva# no mirrors are available for any reason. 348158273Scpercivafetch_pick_server_init() { 349158273Scperciva : > serverlist_tried 350158273Scperciva 351150159Scperciva# Check that host(1) exists (i.e., that the system wasn't built with the 352156813Sru# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 353150159Scperciva if ! which -s host; then 354158273Scperciva : > serverlist_full 355158273Scperciva return 1 356150159Scperciva fi 357150159Scperciva 358158273Scperciva echo -n "Looking up ${SERVERNAME} mirrors... " 359148871Scperciva 360148871Scperciva# Issue the SRV query and pull out the Priority, Weight, and Target fields. 361158245Scperciva# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 362158245Scperciva# "$name server selection ..."; we allow either format. 363158245Scperciva MLIST="_http._tcp.${SERVERNAME}" 364158245Scperciva host -t srv "${MLIST}" | 365265742Sdelphij sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | 366158245Scperciva cut -f 1,2,4 -d ' ' | 367158273Scperciva sed -e 's/\.$//' | 368158273Scperciva sort > serverlist_full 369148871Scperciva 370148871Scperciva# If no records, give up -- we'll just use the server name we were given. 371158273Scperciva if [ `wc -l < serverlist_full` -eq 0 ]; then 372158273Scperciva echo "none found." 373158273Scperciva return 1 374158273Scperciva fi 375158273Scperciva 376158273Scperciva# Report how many mirrors we found. 377158273Scperciva echo `wc -l < serverlist_full` "mirrors found." 378158274Scperciva 379158274Scperciva# Generate a random seed for use in picking mirrors. If HTTP_PROXY 380158274Scperciva# is set, this will be used to generate the seed; otherwise, the seed 381158274Scperciva# will be random. 382158301Scperciva if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 383158301Scperciva RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 384158274Scperciva tr -d 'a-f' | 385158274Scperciva cut -c 1-9` 386158274Scperciva else 387158274Scperciva RANDVALUE=`jot -r 1 0 999999999` 388158274Scperciva fi 389158273Scperciva} 390158273Scperciva 391158273Scperciva# Pick a mirror. Returns 1 if we have run out of mirrors to try. 392158273Scpercivafetch_pick_server() { 393158273Scperciva# Generate a list of not-yet-tried mirrors 394158273Scperciva sort serverlist_tried | 395158273Scperciva comm -23 serverlist_full - > serverlist 396158273Scperciva 397158273Scperciva# Have we run out of mirrors? 398148871Scperciva if [ `wc -l < serverlist` -eq 0 ]; then 399158273Scperciva echo "No mirrors remaining, giving up." 400158273Scperciva return 1 401148871Scperciva fi 402148871Scperciva 403148871Scperciva# Find the highest priority level (lowest numeric value). 404148871Scperciva SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 405148871Scperciva 406148871Scperciva# Add up the weights of the response lines at that priority level. 407148871Scperciva SRV_WSUM=0; 408148871Scperciva while read X; do 409148871Scperciva case "$X" in 410148871Scperciva ${SRV_PRIORITY}\ *) 411148871Scperciva SRV_W=`echo $X | cut -f 2 -d ' '` 412148871Scperciva SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 413148871Scperciva ;; 414148871Scperciva esac 415148871Scperciva done < serverlist 416148871Scperciva 417148871Scperciva# If all the weights are 0, pretend that they are all 1 instead. 418148871Scperciva if [ ${SRV_WSUM} -eq 0 ]; then 419148871Scperciva SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 420148871Scperciva SRV_W_ADD=1 421148871Scperciva else 422148871Scperciva SRV_W_ADD=0 423148871Scperciva fi 424148871Scperciva 425158274Scperciva# Pick a value between 0 and the sum of the weights - 1 426158274Scperciva SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 427148871Scperciva 428158273Scperciva# Read through the list of mirrors and set SERVERNAME. Write the line 429158273Scperciva# corresponding to the mirror we selected into serverlist_tried so that 430158273Scperciva# we won't try it again. 431148871Scperciva while read X; do 432148871Scperciva case "$X" in 433148871Scperciva ${SRV_PRIORITY}\ *) 434148871Scperciva SRV_W=`echo $X | cut -f 2 -d ' '` 435148871Scperciva SRV_W=$(($SRV_W + $SRV_W_ADD)) 436158274Scperciva if [ $SRV_RND -lt $SRV_W ]; then 437148871Scperciva SERVERNAME=`echo $X | cut -f 3 -d ' '` 438158273Scperciva echo "$X" >> serverlist_tried 439148871Scperciva break 440148871Scperciva else 441148871Scperciva SRV_RND=$(($SRV_RND - $SRV_W)) 442148871Scperciva fi 443148871Scperciva ;; 444148871Scperciva esac 445148871Scperciva done < serverlist 446148871Scperciva} 447148871Scperciva 448148871Scperciva# Check that we have a public key with an appropriate hash, or 449158273Scperciva# fetch the key if it doesn't exist. Returns 1 if the key has 450158273Scperciva# not yet been fetched. 451148871Scpercivafetch_key() { 452148871Scperciva if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 453158273Scperciva return 0 454148871Scperciva fi 455148871Scperciva 456158273Scperciva echo -n "Fetching public key from ${SERVERNAME}... " 457148871Scperciva rm -f pub.ssl 458148871Scperciva fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 459148871Scperciva 2>${QUIETREDIR} || true 460148871Scperciva if ! [ -r pub.ssl ]; then 461148871Scperciva echo "failed." 462148871Scperciva return 1 463148871Scperciva fi 464148871Scperciva if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 465148871Scperciva echo "key has incorrect hash." 466148871Scperciva rm -f pub.ssl 467148871Scperciva return 1 468148871Scperciva fi 469148871Scperciva echo "done." 470148871Scperciva} 471148871Scperciva 472148871Scperciva# Fetch a snapshot tag 473148871Scpercivafetch_tag() { 474148871Scperciva rm -f snapshot.ssl tag.new 475148871Scperciva 476158273Scperciva echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 477158273Scperciva fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 478148871Scperciva 2>${QUIETREDIR} || true 479148871Scperciva if ! [ -r $1.ssl ]; then 480148871Scperciva echo "failed." 481148871Scperciva return 1 482148871Scperciva fi 483148871Scperciva 484148871Scperciva openssl rsautl -pubin -inkey pub.ssl -verify \ 485148871Scperciva < $1.ssl > tag.new 2>${QUIETREDIR} || true 486148871Scperciva rm $1.ssl 487148871Scperciva 488148871Scperciva if ! [ `wc -l < tag.new` = 1 ] || 489148871Scperciva ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 490148871Scperciva echo "invalid snapshot tag." 491148871Scperciva return 1 492148871Scperciva fi 493148871Scperciva 494148871Scperciva echo "done." 495148871Scperciva 496148871Scperciva SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 497148871Scperciva SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 498148871Scperciva} 499148871Scperciva 500148871Scperciva# Sanity-check the date on a snapshot tag 501148871Scpercivafetch_snapshot_tagsanity() { 502148871Scperciva if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 503148871Scperciva echo "Snapshot appears to be more than a year old!" 504148871Scperciva echo "(Is the system clock correct?)" 505149868Scperciva echo "Cowardly refusing to proceed any further." 506148871Scperciva return 1 507148871Scperciva fi 508148871Scperciva if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 509148871Scperciva echo -n "Snapshot appears to have been created more than " 510148871Scperciva echo "one day into the future!" 511148871Scperciva echo "(Is the system clock correct?)" 512148871Scperciva echo "Cowardly refusing to proceed any further." 513148871Scperciva return 1 514148871Scperciva fi 515148871Scperciva} 516148871Scperciva 517148871Scperciva# Sanity-check the date on a snapshot update tag 518148871Scpercivafetch_update_tagsanity() { 519148871Scperciva fetch_snapshot_tagsanity || return 1 520148871Scperciva 521148871Scperciva if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 522148871Scperciva echo -n "Latest snapshot on server is " 523148871Scperciva echo "older than what we already have!" 524148871Scperciva echo -n "Cowardly refusing to downgrade from " 525148871Scperciva date -r ${OLDSNAPSHOTDATE} 526148879Scperciva echo "to `date -r ${SNAPSHOTDATE}`." 527148871Scperciva return 1 528148871Scperciva fi 529148871Scperciva} 530148871Scperciva 531148871Scperciva# Compare old and new tags; return 1 if update is unnecessary 532148871Scpercivafetch_update_neededp() { 533148871Scperciva if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 534148871Scperciva echo -n "Latest snapshot on server matches " 535148871Scperciva echo "what we already have." 536148871Scperciva echo "No updates needed." 537148871Scperciva rm tag.new 538148871Scperciva return 1 539148871Scperciva fi 540148871Scperciva if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 541148871Scperciva echo -n "Ports tree hasn't changed since " 542148871Scperciva echo "last snapshot." 543148871Scperciva echo "No updates needed." 544148871Scperciva rm tag.new 545148871Scperciva return 1 546148871Scperciva fi 547148871Scperciva 548148871Scperciva return 0 549148871Scperciva} 550148871Scperciva 551148871Scperciva# Fetch snapshot metadata file 552148871Scpercivafetch_metadata() { 553148871Scperciva rm -f ${SNAPSHOTHASH} tINDEX.new 554148871Scperciva 555148871Scperciva echo ${NDEBUG} "Fetching snapshot metadata... " 556226028Sjilles fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ 557148871Scperciva 2>${QUIETREDIR} || return 558226028Sjilles if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then 559148871Scperciva echo "snapshot metadata corrupt." 560148871Scperciva return 1 561148871Scperciva fi 562148871Scperciva mv ${SNAPSHOTHASH} tINDEX.new 563148871Scperciva echo "done." 564148871Scperciva} 565148871Scperciva 566148871Scperciva# Warn user about bogus metadata 567148871Scpercivafetch_metadata_freakout() { 568148871Scperciva echo 569148871Scperciva echo "Portsnap metadata is correctly signed, but contains" 570148871Scperciva echo "at least one line which appears bogus." 571148871Scperciva echo "Cowardly refusing to proceed any further." 572148871Scperciva} 573148871Scperciva 574148871Scperciva# Sanity-check a snapshot metadata file 575148871Scpercivafetch_metadata_sanity() { 576148871Scperciva if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 577148871Scperciva fetch_metadata_freakout 578148871Scperciva return 1 579148871Scperciva fi 580148871Scperciva if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 581148871Scperciva echo 582148871Scperciva echo "Portsnap metadata appears bogus." 583148871Scperciva echo "Cowardly refusing to proceed any further." 584148871Scperciva return 1 585148871Scperciva fi 586148871Scperciva} 587148871Scperciva 588148871Scperciva# Take a list of ${oldhash}|${newhash} and output a list of needed patches 589148871Scpercivafetch_make_patchlist() { 590241999Seadler local IFS='|' 591227483Salfred echo "" 1>${QUIETREDIR} 592227483Salfred grep -vE "^([0-9a-f]{64})\|\1$" | 593227483Salfred while read X Y; do 594227483Salfred printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} 595227483Salfred if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi 596227483Salfred echo "${X}|${Y}" 597148871Scperciva done 598227483Salfred echo "" 1>${QUIETREDIR} 599148871Scperciva} 600148871Scperciva 601148871Scperciva# Print user-friendly progress statistics 602148871Scpercivafetch_progress() { 603148871Scperciva LNC=0 604148871Scperciva while read x; do 605148871Scperciva LNC=$(($LNC + 1)) 606148871Scperciva if [ $(($LNC % 10)) = 0 ]; then 607149023Scperciva echo -n $LNC 608148871Scperciva elif [ $(($LNC % 2)) = 0 ]; then 609148871Scperciva echo -n . 610148871Scperciva fi 611148871Scperciva done 612148871Scperciva echo -n " " 613148871Scperciva} 614148871Scperciva 615244165Salfredpct_fmt() 616244165Salfred{ 617244165Salfred printf " \r" 618244165Salfred printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc` 619244165Salfred} 620244165Salfred 621244165Salfredfetch_progress_percent() { 622244165Salfred TOTAL=$1 623244165Salfred LNC=0 624244165Salfred pct_fmt $LNC $TOTAL 625244165Salfred while read x; do 626244165Salfred LNC=$(($LNC + 1)) 627244165Salfred if [ $(($LNC % 100)) = 0 ]; then 628244165Salfred pct_fmt $LNC $TOTAL 629244165Salfred elif [ $(($LNC % 10)) = 0 ]; then 630244165Salfred echo -n . 631244165Salfred fi 632244165Salfred done 633244165Salfred pct_fmt $LNC $TOTAL 634244165Salfred echo " done. " 635244165Salfred} 636244165Salfred 637148871Scperciva# Sanity-check an index file 638148871Scpercivafetch_index_sanity() { 639148871Scperciva if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 640148871Scperciva fgrep -q "./" INDEX.new; then 641148871Scperciva fetch_metadata_freakout 642148871Scperciva return 1 643148871Scperciva fi 644148871Scperciva} 645148871Scperciva 646148871Scperciva# Verify a list of files 647148871Scpercivafetch_snapshot_verify() { 648148871Scperciva while read F; do 649306941Sdelphij if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then 650148871Scperciva echo "snapshot corrupt." 651148871Scperciva return 1 652148871Scperciva fi 653148871Scperciva done 654148871Scperciva return 0 655148871Scperciva} 656148871Scperciva 657148871Scperciva# Fetch a snapshot tarball, extract, and verify. 658148871Scpercivafetch_snapshot() { 659158273Scperciva while ! fetch_tag snapshot; do 660158273Scperciva fetch_pick_server || return 1 661158273Scperciva done 662148871Scperciva fetch_snapshot_tagsanity || return 1 663148871Scperciva fetch_metadata || return 1 664148871Scperciva fetch_metadata_sanity || return 1 665148871Scperciva 666148871Scperciva rm -rf snap/ 667148871Scperciva 668148871Scperciva# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 669148871Scperciva# probably take a while, so the progrees reports that fetch(1) generates 670148871Scperciva# will be useful for keeping the users' attention from drifting. 671148871Scperciva echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 672154693Scperciva fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 673148871Scperciva 674148871Scperciva echo -n "Extracting snapshot... " 675216575Ssimon tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 676148871Scperciva rm ${SNAPSHOTHASH}.tgz 677148871Scperciva echo "done." 678148871Scperciva 679148871Scperciva echo -n "Verifying snapshot integrity... " 680148871Scperciva# Verify the metadata files 681148871Scperciva cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 682148871Scperciva# Extract the index 683148871Scperciva rm -f INDEX.new 684306941Sdelphij gunzip -c < snap/`look INDEX tINDEX.new | 685148871Scperciva cut -f 2 -d '|'`.gz > INDEX.new 686148871Scperciva fetch_index_sanity || return 1 687148871Scperciva# Verify the snapshot contents 688148871Scperciva cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 689306941Sdelphij cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u > files.expected 690306941Sdelphij find snap -mindepth 1 | sed -E 's^snap/(.*)\.gz^\1^' | sort > files.snap 691306941Sdelphij if ! cmp -s files.expected files.snap; then 692306941Sdelphij echo "unexpected files in snapshot." 693306941Sdelphij return 1 694306941Sdelphij fi 695306941Sdelphij rm files.expected files.snap 696148871Scperciva echo "done." 697148871Scperciva 698148871Scperciva# Move files into their proper locations 699148871Scperciva rm -f tag INDEX tINDEX 700148871Scperciva rm -rf files 701148871Scperciva mv tag.new tag 702148871Scperciva mv tINDEX.new tINDEX 703148871Scperciva mv INDEX.new INDEX 704148871Scperciva mv snap/ files/ 705148871Scperciva 706148871Scperciva return 0 707148871Scperciva} 708148871Scperciva 709148871Scperciva# Update a compressed snapshot 710148871Scpercivafetch_update() { 711148871Scperciva rm -f patchlist diff OLD NEW filelist INDEX.new 712148871Scperciva 713148871Scperciva OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 714148871Scperciva OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 715148871Scperciva 716158273Scperciva while ! fetch_tag latest; do 717158273Scperciva fetch_pick_server || return 1 718158273Scperciva done 719148871Scperciva fetch_update_tagsanity || return 1 720148871Scperciva fetch_update_neededp || return 0 721148871Scperciva fetch_metadata || return 1 722148871Scperciva fetch_metadata_sanity || return 1 723148871Scperciva 724148871Scperciva echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 725148871Scperciva echo "to `date -r ${SNAPSHOTDATE}`." 726148871Scperciva 727148871Scperciva# Generate a list of wanted metadata patches 728148871Scperciva join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 729148871Scperciva fetch_make_patchlist > patchlist 730148871Scperciva 731148871Scperciva# Attempt to fetch metadata patches 732148871Scperciva echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 733148871Scperciva echo ${NDEBUG} "metadata patches.${DDSTATS}" 734148871Scperciva tr '|' '-' < patchlist | 735148871Scperciva lam -s "tp/" - -s ".gz" | 736148879Scperciva xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 737148871Scperciva 2>${STATSREDIR} | fetch_progress 738148871Scperciva echo "done." 739148871Scperciva 740148871Scperciva# Attempt to apply metadata patches 741148871Scperciva echo -n "Applying metadata patches... " 742241999Seadler local oldifs="$IFS" IFS='|' 743227483Salfred while read X Y; do 744148871Scperciva if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 745148871Scperciva gunzip -c < ${X}-${Y}.gz > diff 746148871Scperciva gunzip -c < files/${X}.gz > OLD 747148871Scperciva cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 748148871Scperciva grep '^\+' diff | cut -c 2- | 749148871Scperciva sort -k 1,1 -t '|' -m - ptmp > NEW 750148871Scperciva if [ `${SHA256} -q NEW` = ${Y} ]; then 751148871Scperciva mv NEW files/${Y} 752148871Scperciva gzip -n files/${Y} 753148871Scperciva fi 754148871Scperciva rm -f diff OLD NEW ${X}-${Y}.gz ptmp 755148871Scperciva done < patchlist 2>${QUIETREDIR} 756241999Seadler IFS="$oldifs" 757148871Scperciva echo "done." 758148871Scperciva 759148871Scperciva# Update metadata without patches 760148871Scperciva join -t '|' -v 2 tINDEX tINDEX.new | 761148871Scperciva cut -f 2 -d '|' /dev/stdin patchlist | 762148871Scperciva while read Y; do 763148871Scperciva if [ ! -f "files/${Y}.gz" ]; then 764148871Scperciva echo ${Y}; 765148871Scperciva fi 766148871Scperciva done > filelist 767148871Scperciva echo -n "Fetching `wc -l < filelist | tr -d ' '` " 768148871Scperciva echo ${NDEBUG} "metadata files... " 769148871Scperciva lam -s "f/" - -s ".gz" < filelist | 770148879Scperciva xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 771148871Scperciva 2>${QUIETREDIR} 772148871Scperciva 773148871Scperciva while read Y; do 774227483Salfred echo -n "Verifying ${Y}... " 1>${QUIETREDIR} 775148871Scperciva if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 776148871Scperciva mv ${Y}.gz files/${Y}.gz 777148871Scperciva else 778148871Scperciva echo "metadata is corrupt." 779148871Scperciva return 1 780148871Scperciva fi 781227483Salfred echo "ok." 1>${QUIETREDIR} 782148871Scperciva done < filelist 783148871Scperciva echo "done." 784148871Scperciva 785148871Scperciva# Extract the index 786227483Salfred echo -n "Extracting index... " 1>${QUIETREDIR} 787306941Sdelphij gunzip -c < files/`look INDEX tINDEX.new | 788148871Scperciva cut -f 2 -d '|'`.gz > INDEX.new 789148871Scperciva fetch_index_sanity || return 1 790148871Scperciva 791149824Scperciva# If we have decided to refuse certain updates, construct a hybrid index which 792149824Scperciva# is equal to the old index for parts of the tree which we don't want to 793149824Scperciva# update, and equal to the new index for parts of the tree which gets updates. 794149824Scperciva# This means that we should always have a "complete snapshot" of the ports 795149824Scperciva# tree -- with the caveat that it isn't actually a snapshot. 796149824Scperciva if [ ! -z "${REFUSE}" ]; then 797149824Scperciva echo "Refusing to download updates for ${REFUSE}" \ 798149824Scperciva >${QUIETREDIR} 799149824Scperciva 800149824Scperciva grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 801149824Scperciva grep -E "${REFUSE}" INDEX | 802149824Scperciva sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 803149824Scperciva rm -f INDEX.tmp 804149824Scperciva fi 805149824Scperciva 806148871Scperciva# Generate a list of wanted ports patches 807227483Salfred echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} 808148871Scperciva join -t '|' -o 1.2,2.2 INDEX INDEX.new | 809148871Scperciva fetch_make_patchlist > patchlist 810227483Salfred echo " done." 1>${QUIETREDIR} 811148871Scperciva 812148871Scperciva# Attempt to fetch ports patches 813244165Salfred patchcnt=`wc -l < patchlist | tr -d ' '` 814244165Salfred echo -n "Fetching $patchcnt " 815148871Scperciva echo ${NDEBUG} "patches.${DDSTATS}" 816244165Salfred echo " " 817148871Scperciva tr '|' '-' < patchlist | lam -s "bp/" - | 818148879Scperciva xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 819244165Salfred 2>${STATSREDIR} | fetch_progress_percent $patchcnt 820148871Scperciva echo "done." 821148871Scperciva 822148871Scperciva# Attempt to apply ports patches 823227483Salfred PATCHCNT=`wc -l patchlist` 824227483Salfred echo "Applying patches... " 825241999Seadler local oldifs="$IFS" IFS='|' 826227483Salfred I=0 827227483Salfred while read X Y; do 828227483Salfred I=$(($I + 1)) 829227483Salfred F="${X}-${Y}" 830227483Salfred if [ ! -f "${F}" ]; then 831227483Salfred printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" 832227483Salfred continue; 833227483Salfred fi 834227483Salfred echo " Processing ${F}..." 1>${QUIETREDIR} 835148871Scperciva gunzip -c < files/${X}.gz > OLD 836148871Scperciva ${BSPATCH} OLD NEW ${X}-${Y} 837148871Scperciva if [ `${SHA256} -q NEW` = ${Y} ]; then 838148871Scperciva mv NEW files/${Y} 839148871Scperciva gzip -n files/${Y} 840148871Scperciva fi 841148871Scperciva rm -f diff OLD NEW ${X}-${Y} 842148871Scperciva done < patchlist 2>${QUIETREDIR} 843241999Seadler IFS="$oldifs" 844148871Scperciva echo "done." 845148871Scperciva 846148871Scperciva# Update ports without patches 847148871Scperciva join -t '|' -v 2 INDEX INDEX.new | 848148871Scperciva cut -f 2 -d '|' /dev/stdin patchlist | 849148871Scperciva while read Y; do 850148871Scperciva if [ ! -f "files/${Y}.gz" ]; then 851148871Scperciva echo ${Y}; 852148871Scperciva fi 853148871Scperciva done > filelist 854148871Scperciva echo -n "Fetching `wc -l < filelist | tr -d ' '` " 855148871Scperciva echo ${NDEBUG} "new ports or files... " 856148871Scperciva lam -s "f/" - -s ".gz" < filelist | 857148879Scperciva xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 858148871Scperciva 2>${QUIETREDIR} 859148871Scperciva 860227483Salfred I=0 861148871Scperciva while read Y; do 862227483Salfred I=$(($I + 1)) 863227483Salfred printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} 864148871Scperciva if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 865148871Scperciva mv ${Y}.gz files/${Y}.gz 866148871Scperciva else 867148871Scperciva echo "snapshot is corrupt." 868148871Scperciva return 1 869148871Scperciva fi 870148871Scperciva done < filelist 871148871Scperciva echo "done." 872148871Scperciva 873148871Scperciva# Remove files which are no longer needed 874253224Scperciva cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles 875253224Scperciva cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | 876148871Scperciva lam -s "files/" - -s ".gz" | xargs rm -f 877148871Scperciva rm patchlist filelist oldfiles 878148871Scperciva 879148871Scperciva# We're done! 880148871Scperciva mv INDEX.new INDEX 881148871Scperciva mv tINDEX.new tINDEX 882148871Scperciva mv tag.new tag 883148871Scperciva 884148871Scperciva return 0 885148871Scperciva} 886148871Scperciva 887148871Scperciva# Do the actual work involved in "fetch" / "cron". 888148871Scpercivafetch_run() { 889158273Scperciva fetch_pick_server_init && fetch_pick_server 890148871Scperciva 891158273Scperciva while ! fetch_key; do 892158273Scperciva fetch_pick_server || return 1 893158273Scperciva done 894148871Scperciva 895148871Scperciva if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 896148871Scperciva fetch_snapshot || return 1 897148871Scperciva fi 898148871Scperciva fetch_update || return 1 899148871Scperciva} 900148871Scperciva 901148871Scperciva# Build a ports INDEX file 902148871Scpercivaextract_make_index() { 903179073Scperciva if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 904179073Scperciva echo -n "$1 not provided by portsnap server; " 905179073Scperciva echo "$2 not being generated." 906179073Scperciva else 907306941Sdelphij gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 908158523Scperciva cut -f 2 -d '|'`.gz" | 909158523Scperciva cat - ${LOCALDESC} | 910158523Scperciva ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 911179073Scperciva fi 912148871Scperciva} 913148871Scperciva 914148871Scperciva# Create INDEX, INDEX-5, INDEX-6 915148871Scpercivaextract_indices() { 916148871Scperciva echo -n "Building new INDEX files... " 917179073Scperciva for PAIR in ${INDEXPAIRS}; do 918179073Scperciva INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 919179073Scperciva DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 920179073Scperciva extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 921179073Scperciva done 922148871Scperciva echo "done." 923148871Scperciva} 924148871Scperciva 925149824Scperciva# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 926149824Scperciva# merge the values from any exiting .portsnap.INDEX file. 927148871Scpercivaextract_metadata() { 928149824Scperciva if [ -z "${REFUSE}" ]; then 929149984Scperciva sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 930149824Scperciva elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 931149824Scperciva grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 932149824Scperciva > ${PORTSDIR}/.portsnap.INDEX.tmp 933149824Scperciva grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 934149824Scperciva sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 935149824Scperciva > ${PORTSDIR}/.portsnap.INDEX 936149824Scperciva rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 937149824Scperciva else 938149824Scperciva grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 939149824Scperciva > ${PORTSDIR}/.portsnap.INDEX 940149824Scperciva fi 941148871Scperciva} 942148871Scperciva 943148871Scperciva# Do the actual work involved in "extract" 944148871Scpercivaextract_run() { 945241999Seadler local oldifs="$IFS" IFS='|' 946154088Scperciva mkdir -p ${PORTSDIR} || return 1 947154088Scperciva 948149824Scperciva if ! 949149824Scperciva if ! [ -z "${EXTRACTPATH}" ]; then 950149824Scperciva grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 951149824Scperciva elif ! [ -z "${REFUSE}" ]; then 952149824Scperciva grep -vE "${REFUSE}" ${WORKDIR}/INDEX 953149824Scperciva else 954149824Scperciva cat ${WORKDIR}/INDEX 955227483Salfred fi | while read FILE HASH; do 956148871Scperciva echo ${PORTSDIR}/${FILE} 957148871Scperciva if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 958148871Scperciva echo "files/${HASH}.gz not found -- snapshot corrupt." 959148871Scperciva return 1 960148871Scperciva fi 961148871Scperciva case ${FILE} in 962148871Scperciva */) 963159062Scperciva rm -rf ${PORTSDIR}/${FILE%/} 964148871Scperciva mkdir -p ${PORTSDIR}/${FILE} 965216575Ssimon tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 966148871Scperciva -C ${PORTSDIR}/${FILE} 967148871Scperciva ;; 968148871Scperciva *) 969148871Scperciva rm -f ${PORTSDIR}/${FILE} 970216575Ssimon tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 971148871Scperciva -C ${PORTSDIR} ${FILE} 972148871Scperciva ;; 973148871Scperciva esac 974149041Scperciva done; then 975149041Scperciva return 1 976149041Scperciva fi 977148871Scperciva if [ ! -z "${EXTRACTPATH}" ]; then 978148871Scperciva return 0; 979148871Scperciva fi 980148871Scperciva 981241999Seadler IFS="$oldifs" 982241999Seadler 983148871Scperciva extract_metadata 984148871Scperciva extract_indices 985148871Scperciva} 986148871Scperciva 987227484Salfredupdate_run_extract() { 988227483Salfred local IFS='|' 989227483Salfred 990227483Salfred# Install new files 991227483Salfred echo "Extracting new files:" 992227483Salfred if ! 993227483Salfred if ! [ -z "${REFUSE}" ]; then 994227483Salfred grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 995227483Salfred else 996227483Salfred sort ${WORKDIR}/INDEX 997227483Salfred fi | 998227483Salfred comm -13 ${PORTSDIR}/.portsnap.INDEX - | 999227483Salfred while read FILE HASH; do 1000227483Salfred echo ${PORTSDIR}/${FILE} 1001227483Salfred if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 1002227483Salfred echo "files/${HASH}.gz not found -- snapshot corrupt." 1003227483Salfred return 1 1004227483Salfred fi 1005227483Salfred case ${FILE} in 1006227483Salfred */) 1007227483Salfred mkdir -p ${PORTSDIR}/${FILE} 1008227483Salfred tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1009227483Salfred -C ${PORTSDIR}/${FILE} 1010227483Salfred ;; 1011227483Salfred *) 1012227483Salfred tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1013227483Salfred -C ${PORTSDIR} ${FILE} 1014227483Salfred ;; 1015227483Salfred esac 1016227483Salfred done; then 1017227483Salfred return 1 1018227483Salfred fi 1019227483Salfred} 1020227483Salfred 1021148871Scperciva# Do the actual work involved in "update" 1022148871Scpercivaupdate_run() { 1023148871Scperciva if ! [ -z "${INDEXONLY}" ]; then 1024148871Scperciva extract_indices >/dev/null || return 1 1025148871Scperciva return 0 1026148871Scperciva fi 1027148871Scperciva 1028148981Scperciva if sort ${WORKDIR}/INDEX | 1029149042Scperciva cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 1030148958Scperciva echo "Ports tree is already up to date." 1031148958Scperciva return 0 1032148958Scperciva fi 1033148958Scperciva 1034149824Scperciva# If we are REFUSEing to touch certain directories, don't remove files 1035149824Scperciva# from those directories (even if they are out of date) 1036148871Scperciva echo -n "Removing old files and directories... " 1037149824Scperciva if ! [ -z "${REFUSE}" ]; then 1038149824Scperciva sort ${WORKDIR}/INDEX | 1039149824Scperciva comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1040149824Scperciva grep -vE "${REFUSE}" | 1041158473Scperciva lam -s "${PORTSDIR}/" - | 1042158473Scperciva sed -e 's|/$||' | xargs rm -rf 1043149824Scperciva else 1044149824Scperciva sort ${WORKDIR}/INDEX | 1045149824Scperciva comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1046158473Scperciva lam -s "${PORTSDIR}/" - | 1047158473Scperciva sed -e 's|/$||' | xargs rm -rf 1048149824Scperciva fi 1049148871Scperciva echo "done." 1050148871Scperciva 1051227484Salfred update_run_extract || return 1 1052148871Scperciva extract_metadata 1053148871Scperciva extract_indices 1054148871Scperciva} 1055148871Scperciva 1056148871Scperciva#### Main functions -- call parameter-handling and core functions 1057148871Scperciva 1058148871Scperciva# Using the command line, configuration file, and defaults, 1059148871Scperciva# set all the parameters which are needed later. 1060148871Scpercivaget_params() { 1061148871Scperciva init_params 1062148871Scperciva parse_cmdline $@ 1063148871Scperciva sanity_conffile 1064148871Scperciva default_conffile 1065148871Scperciva parse_conffile 1066148871Scperciva default_params 1067148871Scperciva} 1068148871Scperciva 1069148871Scperciva# Fetch command. Make sure that we're being called 1070148871Scperciva# interactively, then run fetch_check_params and fetch_run 1071148871Scpercivacmd_fetch() { 1072239078Seadler if [ "${INTERACTIVE}" != "YES" ]; then 1073148871Scperciva echo -n "`basename $0` fetch should not " 1074148871Scperciva echo "be run non-interactively." 1075239078Seadler echo "Run `basename $0` cron instead" 1076148871Scperciva exit 1 1077148871Scperciva fi 1078148871Scperciva fetch_check_params 1079148871Scperciva fetch_run || exit 1 1080148871Scperciva} 1081148871Scperciva 1082148871Scperciva# Cron command. Make sure the parameters are sensible; wait 1083148871Scperciva# rand(3600) seconds; then fetch updates. While fetching updates, 1084148871Scperciva# send output to a temporary file; only print that file if the 1085148871Scperciva# fetching failed. 1086148871Scpercivacmd_cron() { 1087148871Scperciva fetch_check_params 1088148871Scperciva sleep `jot -r 1 0 3600` 1089148871Scperciva 1090148871Scperciva TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1091148871Scperciva if ! fetch_run >> ${TMPFILE}; then 1092148871Scperciva cat ${TMPFILE} 1093148871Scperciva rm ${TMPFILE} 1094148871Scperciva exit 1 1095148871Scperciva fi 1096148871Scperciva 1097148871Scperciva rm ${TMPFILE} 1098148871Scperciva} 1099148871Scperciva 1100148871Scperciva# Extract command. Make sure the parameters are sensible, 1101148871Scperciva# then extract the ports tree (or part thereof). 1102148871Scpercivacmd_extract() { 1103148871Scperciva extract_check_params 1104148871Scperciva extract_run || exit 1 1105148871Scperciva} 1106148871Scperciva 1107148871Scperciva# Update command. Make sure the parameters are sensible, 1108148871Scperciva# then update the ports tree. 1109148871Scpercivacmd_update() { 1110148871Scperciva update_check_params 1111148871Scperciva update_run || exit 1 1112148871Scperciva} 1113148871Scperciva 1114201251Scperciva# Alfred command. Run 'fetch' or 'cron' depending on 1115201251Scperciva# whether stdin is a terminal; then run 'update' or 1116201251Scperciva# 'extract' depending on whether ${PORTSDIR} exists. 1117201251Scpercivacmd_alfred() { 1118239078Seadler if [ "${INTERACTIVE}" = "YES" ]; then 1119201251Scperciva cmd_fetch 1120201251Scperciva else 1121201251Scperciva cmd_cron 1122201251Scperciva fi 1123251507Sdelphij if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 1124201251Scperciva cmd_update 1125201251Scperciva else 1126201251Scperciva cmd_extract 1127201251Scperciva fi 1128201251Scperciva} 1129201251Scperciva 1130148871Scperciva#### Entry point 1131148871Scperciva 1132148871Scperciva# Make sure we find utilities from the base system 1133148871Scpercivaexport PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1134148871Scperciva 1135163564Scperciva# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1136163564Scpercivaexport LC_ALL=C 1137163564Scperciva 1138148871Scpercivaget_params $@ 1139149027Scpercivafor COMMAND in ${COMMANDS}; do 1140149027Scperciva cmd_${COMMAND} 1141149027Scpercivadone 1142