1/*-
2 * Copyright (c) 2000-2014 Dag-Erling Smørgrav
3 * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer
11 *    in this position and unchanged.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 *    derived from this software without specific prior written permission
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/socket.h>
35#include <sys/stat.h>
36#include <sys/time.h>
37
38#include <ctype.h>
39#include <err.h>
40#include <errno.h>
41#include <getopt.h>
42#include <signal.h>
43#include <stdint.h>
44#include <stdio.h>
45#include <stdlib.h>
46#include <string.h>
47#include <termios.h>
48#include <unistd.h>
49
50#include <fetch.h>
51
52#define MINBUFSIZE	16384
53#define TIMEOUT		120
54
55/* Option flags */
56static int	 A_flag;	/*    -A: do not follow 302 redirects */
57static int	 a_flag;	/*    -a: auto retry */
58static off_t	 B_size;	/*    -B: buffer size */
59static int	 b_flag;	/*!   -b: workaround TCP bug */
60static char    *c_dirname;	/*    -c: remote directory */
61static int	 d_flag;	/*    -d: direct connection */
62static int	 F_flag;	/*    -F: restart without checking mtime  */
63static char	*f_filename;	/*    -f: file to fetch */
64static char	*h_hostname;	/*    -h: host to fetch from */
65static int	 i_flag;	/*    -i: specify file for mtime comparison */
66static char	*i_filename;	/*        name of input file */
67static int	 l_flag;	/*    -l: link rather than copy file: URLs */
68static int	 m_flag;	/* -[Mm]: mirror mode */
69static char	*N_filename;	/*    -N: netrc file name */
70static int	 n_flag;	/*    -n: do not preserve modification time */
71static int	 o_flag;	/*    -o: specify output file */
72static int	 o_directory;	/*        output file is a directory */
73static char	*o_filename;	/*        name of output file */
74static int	 o_stdout;	/*        output file is stdout */
75static int	 once_flag;	/*    -1: stop at first successful file */
76static int	 p_flag;	/* -[Pp]: use passive FTP */
77static int	 R_flag;	/*    -R: don't delete partial files */
78static int	 r_flag;	/*    -r: restart previous transfer */
79static off_t	 S_size;        /*    -S: require size to match */
80static int	 s_flag;        /*    -s: show size, don't fetch */
81static long	 T_secs;	/*    -T: transfer timeout in seconds */
82static int	 t_flag;	/*!   -t: workaround TCP bug */
83static int	 U_flag;	/*    -U: do not use high ports */
84static int	 v_level = 1;	/*    -v: verbosity level */
85static int	 v_tty;		/*        stdout is a tty */
86static pid_t	 pgrp;		/*        our process group */
87static long	 w_secs;	/*    -w: retry delay */
88static int	 family = PF_UNSPEC;	/* -[46]: address family to use */
89
90static int	 sigalrm;	/* SIGALRM received */
91static int	 siginfo;	/* SIGINFO received */
92static int	 sigint;	/* SIGINT received */
93
94static long	 ftp_timeout = TIMEOUT;	/* default timeout for FTP transfers */
95static long	 http_timeout = TIMEOUT;/* default timeout for HTTP transfers */
96static char	*buf;		/* transfer buffer */
97
98enum options
99{
100	OPTION_BIND_ADDRESS,
101	OPTION_NO_FTP_PASSIVE_MODE,
102	OPTION_HTTP_REFERER,
103	OPTION_HTTP_USER_AGENT,
104	OPTION_NO_PROXY,
105	OPTION_SSL_CA_CERT_FILE,
106	OPTION_SSL_CA_CERT_PATH,
107	OPTION_SSL_CLIENT_CERT_FILE,
108	OPTION_SSL_CLIENT_KEY_FILE,
109	OPTION_SSL_CRL_FILE,
110	OPTION_SSL_NO_SSL3,
111	OPTION_SSL_NO_TLS1,
112	OPTION_SSL_NO_VERIFY_HOSTNAME,
113	OPTION_SSL_NO_VERIFY_PEER
114};
115
116
117static struct option longopts[] =
118{
119	/* mapping to single character argument */
120	{ "one-file", no_argument, NULL, '1' },
121	{ "ipv4-only", no_argument, NULL, '4' },
122	{ "ipv6-only", no_argument, NULL, '6' },
123	{ "no-redirect", no_argument, NULL, 'A' },
124	{ "retry", no_argument, NULL, 'a' },
125	{ "buffer-size", required_argument, NULL, 'B' },
126	/* -c not mapped, since it's deprecated */
127	{ "direct", no_argument, NULL, 'd' },
128	{ "force-restart", no_argument, NULL, 'F' },
129	/* -f not mapped, since it's deprecated */
130	/* -h not mapped, since it's deprecated */
131	{ "if-modified-since", required_argument, NULL, 'i' },
132	{ "symlink", no_argument, NULL, 'l' },
133	/* -M not mapped since it's the same as -m */
134	{ "mirror", no_argument, NULL, 'm' },
135	{ "netrc", required_argument, NULL, 'N' },
136	{ "no-mtime", no_argument, NULL, 'n' },
137	{ "output", required_argument, NULL, 'o' },
138	/* -P not mapped since it's the same as -p */
139	{ "passive", no_argument, NULL, 'p' },
140	{ "quiet", no_argument, NULL, 'q' },
141	{ "keep-output", no_argument, NULL, 'R' },
142	{ "restart", no_argument, NULL, 'r' },
143	{ "require-size", required_argument, NULL, 'S' },
144	{ "print-size", no_argument, NULL, 's' },
145	{ "timeout", required_argument, NULL, 'T' },
146	{ "passive-portrange-default", no_argument, NULL, 'T' },
147	{ "verbose", no_argument, NULL, 'v' },
148	{ "retry-delay", required_argument, NULL, 'w' },
149
150	/* options without a single character equivalent */
151	{ "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS },
152	{ "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE },
153	{ "referer", required_argument, NULL, OPTION_HTTP_REFERER },
154	{ "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
155	{ "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
156	{ "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
157	{ "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
158	{ "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
159	{ "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE },
160	{ "crl", required_argument, NULL, OPTION_SSL_CRL_FILE },
161	{ "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 },
162	{ "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 },
163	{ "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME },
164	{ "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER },
165
166	{ NULL, 0, NULL, 0 }
167};
168
169/*
170 * Signal handler
171 */
172static void
173sig_handler(int sig)
174{
175	switch (sig) {
176	case SIGALRM:
177		sigalrm = 1;
178		break;
179	case SIGINFO:
180		siginfo = 1;
181		break;
182	case SIGINT:
183		sigint = 1;
184		break;
185	}
186}
187
188struct xferstat {
189	char		 name[64];
190	struct timeval	 start;		/* start of transfer */
191	struct timeval	 last;		/* time of last update */
192	struct timeval	 last2;		/* time of previous last update */
193	off_t		 size;		/* size of file per HTTP hdr */
194	off_t		 offset;	/* starting offset in file */
195	off_t		 rcvd;		/* bytes already received */
196	off_t		 lastrcvd;	/* bytes received since last update */
197};
198
199/*
200 * Compute and display ETA
201 */
202static const char *
203stat_eta(struct xferstat *xs)
204{
205	static char str[16];
206	long elapsed, eta;
207	off_t received, expected;
208
209	elapsed = xs->last.tv_sec - xs->start.tv_sec;
210	received = xs->rcvd - xs->offset;
211	expected = xs->size - xs->rcvd;
212	eta = (long)((double)elapsed * expected / received);
213	if (eta > 3600)
214		snprintf(str, sizeof str, "%02ldh%02ldm",
215		    eta / 3600, (eta % 3600) / 60);
216	else if (eta > 0)
217		snprintf(str, sizeof str, "%02ldm%02lds",
218		    eta / 60, eta % 60);
219	else
220		snprintf(str, sizeof str, "%02ldm%02lds",
221		    elapsed / 60, elapsed % 60);
222	return (str);
223}
224
225/*
226 * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
227 */
228static const char *prefixes = " kMGTP";
229static const char *
230stat_bytes(off_t bytes)
231{
232	static char str[16];
233	const char *prefix = prefixes;
234
235	while (bytes > 9999 && prefix[1] != '\0') {
236		bytes /= 1024;
237		prefix++;
238	}
239	snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix);
240	return (str);
241}
242
243/*
244 * Compute and display transfer rate
245 */
246static const char *
247stat_bps(struct xferstat *xs)
248{
249	static char str[16];
250	double delta, bps;
251
252	delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
253	    - (xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6));
254
255	if (delta == 0.0) {
256		snprintf(str, sizeof str, "?? Bps");
257	} else {
258		bps = (xs->rcvd - xs->lastrcvd) / delta;
259		snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps));
260	}
261	return (str);
262}
263
264/*
265 * Update the stats display
266 */
267static void
268stat_display(struct xferstat *xs, int force)
269{
270	struct timeval now;
271	int ctty_pgrp;
272
273	/* check if we're the foreground process */
274	if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 ||
275	    (pid_t)ctty_pgrp != pgrp)
276		return;
277
278	gettimeofday(&now, NULL);
279	if (!force && now.tv_sec <= xs->last.tv_sec)
280		return;
281	xs->last2 = xs->last;
282	xs->last = now;
283
284	fprintf(stderr, "\r%-46.46s", xs->name);
285	if (xs->size <= 0) {
286		setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd));
287		fprintf(stderr, "        %s", stat_bytes(xs->rcvd));
288	} else {
289		setproctitle("%s [%d%% of %s]", xs->name,
290		    (int)((100.0 * xs->rcvd) / xs->size),
291		    stat_bytes(xs->size));
292		fprintf(stderr, "%3d%% of %s",
293		    (int)((100.0 * xs->rcvd) / xs->size),
294		    stat_bytes(xs->size));
295	}
296	if (force == 2) {
297		xs->lastrcvd = xs->offset;
298		xs->last2 = xs->start;
299	}
300	fprintf(stderr, " %s", stat_bps(xs));
301	if ((xs->size > 0 && xs->rcvd > 0 &&
302	     xs->last.tv_sec >= xs->start.tv_sec + 3) ||
303	    force == 2)
304		fprintf(stderr, " %s", stat_eta(xs));
305	xs->lastrcvd = xs->rcvd;
306}
307
308/*
309 * Initialize the transfer statistics
310 */
311static void
312stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
313{
314	snprintf(xs->name, sizeof xs->name, "%s", name);
315	gettimeofday(&xs->start, NULL);
316	xs->last.tv_sec = xs->last.tv_usec = 0;
317	xs->size = size;
318	xs->offset = offset;
319	xs->rcvd = offset;
320	xs->lastrcvd = offset;
321	if (v_tty && v_level > 0)
322		stat_display(xs, 1);
323	else if (v_level > 0)
324		fprintf(stderr, "%-46s", xs->name);
325}
326
327/*
328 * Update the transfer statistics
329 */
330static void
331stat_update(struct xferstat *xs, off_t rcvd)
332{
333	xs->rcvd = rcvd;
334	if (v_tty && v_level > 0)
335		stat_display(xs, 0);
336}
337
338/*
339 * Finalize the transfer statistics
340 */
341static void
342stat_end(struct xferstat *xs)
343{
344	gettimeofday(&xs->last, NULL);
345	if (v_tty && v_level > 0) {
346		stat_display(xs, 2);
347		putc('\n', stderr);
348	} else if (v_level > 0) {
349		fprintf(stderr, "        %s %s\n",
350		    stat_bytes(xs->size), stat_bps(xs));
351	}
352}
353
354/*
355 * Ask the user for authentication details
356 */
357static int
358query_auth(struct url *URL)
359{
360	struct termios tios;
361	tcflag_t saved_flags;
362	int i, nopwd;
363
364	fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n",
365	    URL->scheme, URL->host, URL->port);
366
367	fprintf(stderr, "Login: ");
368	if (fgets(URL->user, sizeof URL->user, stdin) == NULL)
369		return (-1);
370	for (i = strlen(URL->user); i >= 0; --i)
371		if (URL->user[i] == '\r' || URL->user[i] == '\n')
372			URL->user[i] = '\0';
373
374	fprintf(stderr, "Password: ");
375	if (tcgetattr(STDIN_FILENO, &tios) == 0) {
376		saved_flags = tios.c_lflag;
377		tios.c_lflag &= ~ECHO;
378		tios.c_lflag |= ECHONL|ICANON;
379		tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios);
380		nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
381		tios.c_lflag = saved_flags;
382		tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios);
383	} else {
384		nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
385	}
386	if (nopwd)
387		return (-1);
388	for (i = strlen(URL->pwd); i >= 0; --i)
389		if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n')
390			URL->pwd[i] = '\0';
391
392	return (0);
393}
394
395/*
396 * Fetch a file
397 */
398static int
399fetch(char *URL, const char *path)
400{
401	struct url *url;
402	struct url_stat us;
403	struct stat sb, nsb;
404	struct xferstat xs;
405	FILE *f, *of;
406	size_t size, readcnt, wr;
407	off_t count;
408	char flags[8];
409	const char *slash;
410	char *tmppath;
411	int r;
412	unsigned timeout;
413	char *ptr;
414
415	f = of = NULL;
416	tmppath = NULL;
417
418	timeout = 0;
419	*flags = 0;
420	count = 0;
421
422	/* set verbosity level */
423	if (v_level > 1)
424		strcat(flags, "v");
425	if (v_level > 2)
426		fetchDebug = 1;
427
428	/* parse URL */
429	url = NULL;
430	if (*URL == '\0') {
431		warnx("empty URL");
432		goto failure;
433	}
434	if ((url = fetchParseURL(URL)) == NULL) {
435		warnx("%s: parse error", URL);
436		goto failure;
437	}
438
439	/* if no scheme was specified, take a guess */
440	if (!*url->scheme) {
441		if (!*url->host)
442			strcpy(url->scheme, SCHEME_FILE);
443		else if (strncasecmp(url->host, "ftp.", 4) == 0)
444			strcpy(url->scheme, SCHEME_FTP);
445		else if (strncasecmp(url->host, "www.", 4) == 0)
446			strcpy(url->scheme, SCHEME_HTTP);
447	}
448
449	/* common flags */
450	switch (family) {
451	case PF_INET:
452		strcat(flags, "4");
453		break;
454	case PF_INET6:
455		strcat(flags, "6");
456		break;
457	}
458
459	/* FTP specific flags */
460	if (strcmp(url->scheme, SCHEME_FTP) == 0) {
461		if (p_flag)
462			strcat(flags, "p");
463		if (d_flag)
464			strcat(flags, "d");
465		if (U_flag)
466			strcat(flags, "l");
467		timeout = T_secs ? T_secs : ftp_timeout;
468	}
469
470	/* HTTP specific flags */
471	if (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
472	    strcmp(url->scheme, SCHEME_HTTPS) == 0) {
473		if (d_flag)
474			strcat(flags, "d");
475		if (A_flag)
476			strcat(flags, "A");
477		timeout = T_secs ? T_secs : http_timeout;
478		if (i_flag) {
479			if (stat(i_filename, &sb)) {
480				warn("%s: stat()", i_filename);
481				goto failure;
482			}
483			url->ims_time = sb.st_mtime;
484			strcat(flags, "i");
485		}
486	}
487
488	/* set the protocol timeout. */
489	fetchTimeout = timeout;
490
491	/* just print size */
492	if (s_flag) {
493		if (timeout)
494			alarm(timeout);
495		r = fetchStat(url, &us, flags);
496		if (timeout)
497			alarm(0);
498		if (sigalrm || sigint)
499			goto signal;
500		if (r == -1) {
501			warnx("%s", fetchLastErrString);
502			goto failure;
503		}
504		if (us.size == -1)
505			printf("Unknown\n");
506		else
507			printf("%jd\n", (intmax_t)us.size);
508		goto success;
509	}
510
511	/*
512	 * If the -r flag was specified, we have to compare the local
513	 * and remote files, so we should really do a fetchStat()
514	 * first, but I know of at least one HTTP server that only
515	 * sends the content size in response to GET requests, and
516	 * leaves it out of replies to HEAD requests.  Also, in the
517	 * (frequent) case that the local and remote files match but
518	 * the local file is truncated, we have sufficient information
519	 * before the compare to issue a correct request.  Therefore,
520	 * we always issue a GET request as if we were sure the local
521	 * file was a truncated copy of the remote file; we can drop
522	 * the connection later if we change our minds.
523	 */
524	sb.st_size = -1;
525	if (!o_stdout) {
526		r = stat(path, &sb);
527		if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
528			url->offset = sb.st_size;
529		} else if (r == -1 || !S_ISREG(sb.st_mode)) {
530			/*
531			 * Whatever value sb.st_size has now is either
532			 * wrong (if stat(2) failed) or irrelevant (if the
533			 * path does not refer to a regular file)
534			 */
535			sb.st_size = -1;
536		}
537		if (r == -1 && errno != ENOENT) {
538			warnx("%s: stat()", path);
539			goto failure;
540		}
541	}
542
543	/* start the transfer */
544	if (timeout)
545		alarm(timeout);
546	f = fetchXGet(url, &us, flags);
547	if (timeout)
548		alarm(0);
549	if (sigalrm || sigint)
550		goto signal;
551	if (f == NULL) {
552		warnx("%s: %s", URL, fetchLastErrString);
553		if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
554		    && fetchLastErrCode == FETCH_OK
555		    && strcmp(fetchLastErrString, "Not Modified") == 0) {
556			/* HTTP Not Modified Response, return OK. */
557			r = 0;
558			goto done;
559		} else
560			goto failure;
561	}
562	if (sigint)
563		goto signal;
564
565	/* check that size is as expected */
566	if (S_size) {
567		if (us.size == -1) {
568			warnx("%s: size unknown", URL);
569		} else if (us.size != S_size) {
570			warnx("%s: size mismatch: expected %jd, actual %jd",
571			    URL, (intmax_t)S_size, (intmax_t)us.size);
572			goto failure;
573		}
574	}
575
576	/* symlink instead of copy */
577	if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) {
578		if (symlink(url->doc, path) == -1) {
579			warn("%s: symlink()", path);
580			goto failure;
581		}
582		goto success;
583	}
584
585	if (us.size == -1 && !o_stdout && v_level > 0)
586		warnx("%s: size of remote file is not known", URL);
587	if (v_level > 1) {
588		if (sb.st_size != -1)
589			fprintf(stderr, "local size / mtime: %jd / %ld\n",
590			    (intmax_t)sb.st_size, (long)sb.st_mtime);
591		if (us.size != -1)
592			fprintf(stderr, "remote size / mtime: %jd / %ld\n",
593			    (intmax_t)us.size, (long)us.mtime);
594	}
595
596	/* open output file */
597	if (o_stdout) {
598		/* output to stdout */
599		of = stdout;
600	} else if (r_flag && sb.st_size != -1) {
601		/* resume mode, local file exists */
602		if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
603			/* no match! have to refetch */
604			fclose(f);
605			/* if precious, warn the user and give up */
606			if (R_flag) {
607				warnx("%s: local modification time "
608				    "does not match remote", path);
609				goto failure_keep;
610			}
611		} else if (url->offset > sb.st_size) {
612			/* gap between what we asked for and what we got */
613			warnx("%s: gap in resume mode", URL);
614			fclose(of);
615			of = NULL;
616			/* picked up again later */
617		} else if (us.size != -1) {
618			if (us.size == sb.st_size)
619				/* nothing to do */
620				goto success;
621			if (sb.st_size > us.size) {
622				/* local file too long! */
623				warnx("%s: local file (%jd bytes) is longer "
624				    "than remote file (%jd bytes)", path,
625				    (intmax_t)sb.st_size, (intmax_t)us.size);
626				goto failure;
627			}
628			/* we got it, open local file */
629			if ((of = fopen(path, "r+")) == NULL) {
630				warn("%s: fopen()", path);
631				goto failure;
632			}
633			/* check that it didn't move under our feet */
634			if (fstat(fileno(of), &nsb) == -1) {
635				/* can't happen! */
636				warn("%s: fstat()", path);
637				goto failure;
638			}
639			if (nsb.st_dev != sb.st_dev ||
640			    nsb.st_ino != sb.st_ino ||
641			    nsb.st_size != sb.st_size) {
642				warnx("%s: file has changed", URL);
643				fclose(of);
644				of = NULL;
645				sb = nsb;
646				/* picked up again later */
647			}
648		}
649		/* seek to where we left off */
650		if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) {
651			warn("%s: fseeko()", path);
652			fclose(of);
653			of = NULL;
654			/* picked up again later */
655		}
656	} else if (m_flag && sb.st_size != -1) {
657		/* mirror mode, local file exists */
658		if (sb.st_size == us.size && sb.st_mtime == us.mtime)
659			goto success;
660	}
661
662	if (of == NULL) {
663		/*
664		 * We don't yet have an output file; either this is a
665		 * vanilla run with no special flags, or the local and
666		 * remote files didn't match.
667		 */
668
669		if (url->offset > 0) {
670			/*
671			 * We tried to restart a transfer, but for
672			 * some reason gave up - so we have to restart
673			 * from scratch if we want the whole file
674			 */
675			url->offset = 0;
676			if ((f = fetchXGet(url, &us, flags)) == NULL) {
677				warnx("%s: %s", URL, fetchLastErrString);
678				goto failure;
679			}
680			if (sigint)
681				goto signal;
682		}
683
684		/* construct a temp file name */
685		if (sb.st_size != -1 && S_ISREG(sb.st_mode)) {
686			if ((slash = strrchr(path, '/')) == NULL)
687				slash = path;
688			else
689				++slash;
690			asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
691			    (int)(slash - path), path, slash);
692			if (tmppath != NULL) {
693				if (mkstemps(tmppath, strlen(slash) + 1) == -1) {
694					warn("%s: mkstemps()", path);
695					goto failure;
696				}
697				of = fopen(tmppath, "w");
698				chown(tmppath, sb.st_uid, sb.st_gid);
699				chmod(tmppath, sb.st_mode & ALLPERMS);
700			}
701		}
702		if (of == NULL)
703			of = fopen(path, "w");
704		if (of == NULL) {
705			warn("%s: open()", path);
706			goto failure;
707		}
708	}
709	count = url->offset;
710
711	/* start the counter */
712	stat_start(&xs, path, us.size, count);
713
714	sigalrm = siginfo = sigint = 0;
715
716	/* suck in the data */
717	setvbuf(f, NULL, _IOFBF, B_size);
718	signal(SIGINFO, sig_handler);
719	while (!sigint) {
720		if (us.size != -1 && us.size - count < B_size &&
721		    us.size - count >= 0)
722			size = us.size - count;
723		else
724			size = B_size;
725		if (siginfo) {
726			stat_end(&xs);
727			siginfo = 0;
728		}
729
730		if (size == 0)
731			break;
732
733		if ((readcnt = fread(buf, 1, size, f)) < size) {
734			if (ferror(f) && errno == EINTR && !sigint)
735				clearerr(f);
736			else if (readcnt == 0)
737				break;
738		}
739
740		stat_update(&xs, count += readcnt);
741		for (ptr = buf; readcnt > 0; ptr += wr, readcnt -= wr)
742			if ((wr = fwrite(ptr, 1, readcnt, of)) < readcnt) {
743				if (ferror(of) && errno == EINTR && !sigint)
744					clearerr(of);
745				else
746					break;
747			}
748		if (readcnt != 0)
749			break;
750	}
751	if (!sigalrm)
752		sigalrm = ferror(f) && errno == ETIMEDOUT;
753	signal(SIGINFO, SIG_DFL);
754
755	stat_end(&xs);
756
757	/*
758	 * If the transfer timed out or was interrupted, we still want to
759	 * set the mtime in case the file is not removed (-r or -R) and
760	 * the user later restarts the transfer.
761	 */
762 signal:
763	/* set mtime of local file */
764	if (!n_flag && us.mtime && !o_stdout && of != NULL &&
765	    (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) {
766		struct timeval tv[2];
767
768		fflush(of);
769		tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime);
770		tv[1].tv_sec = (long)us.mtime;
771		tv[0].tv_usec = tv[1].tv_usec = 0;
772		if (utimes(tmppath ? tmppath : path, tv))
773			warn("%s: utimes()", tmppath ? tmppath : path);
774	}
775
776	/* timed out or interrupted? */
777	if (sigalrm)
778		warnx("transfer timed out");
779	if (sigint) {
780		warnx("transfer interrupted");
781		goto failure;
782	}
783
784	/* timeout / interrupt before connection completley established? */
785	if (f == NULL)
786		goto failure;
787
788	if (!sigalrm) {
789		/* check the status of our files */
790		if (ferror(f))
791			warn("%s", URL);
792		if (ferror(of))
793			warn("%s", path);
794		if (ferror(f) || ferror(of))
795			goto failure;
796	}
797
798	/* did the transfer complete normally? */
799	if (us.size != -1 && count < us.size) {
800		warnx("%s appears to be truncated: %jd/%jd bytes",
801		    path, (intmax_t)count, (intmax_t)us.size);
802		goto failure_keep;
803	}
804
805	/*
806	 * If the transfer timed out and we didn't know how much to
807	 * expect, assume the worst (i.e. we didn't get all of it)
808	 */
809	if (sigalrm && us.size == -1) {
810		warnx("%s may be truncated", path);
811		goto failure_keep;
812	}
813
814 success:
815	r = 0;
816	if (tmppath != NULL && rename(tmppath, path) == -1) {
817		warn("%s: rename()", path);
818		goto failure_keep;
819	}
820	goto done;
821 failure:
822	if (of && of != stdout && !R_flag && !r_flag)
823		if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG))
824			unlink(tmppath ? tmppath : path);
825	if (R_flag && tmppath != NULL && sb.st_size == -1)
826		rename(tmppath, path); /* ignore errors here */
827 failure_keep:
828	r = -1;
829	goto done;
830 done:
831	if (f)
832		fclose(f);
833	if (of && of != stdout)
834		fclose(of);
835	if (url)
836		fetchFreeURL(url);
837	if (tmppath != NULL)
838		free(tmppath);
839	return (r);
840}
841
842static void
843usage(void)
844{
845	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
846"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
847"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
848"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
849"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
850"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
851"       [--user-agent=agent-string] [-w seconds] URL ...",
852"       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
853"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
854"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
855"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
856"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
857"       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
858}
859
860
861/*
862 * Entry point
863 */
864int
865main(int argc, char *argv[])
866{
867	struct stat sb;
868	struct sigaction sa;
869	const char *p, *s;
870	char *end, *q;
871	int c, e, r;
872
873
874	while ((c = getopt_long(argc, argv,
875	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:",
876	    longopts, NULL)) != -1)
877		switch (c) {
878		case '1':
879			once_flag = 1;
880			break;
881		case '4':
882			family = PF_INET;
883			break;
884		case '6':
885			family = PF_INET6;
886			break;
887		case 'A':
888			A_flag = 1;
889			break;
890		case 'a':
891			a_flag = 1;
892			break;
893		case 'B':
894			B_size = (off_t)strtol(optarg, &end, 10);
895			if (*optarg == '\0' || *end != '\0')
896				errx(1, "invalid buffer size (%s)", optarg);
897			break;
898		case 'b':
899			warnx("warning: the -b option is deprecated");
900			b_flag = 1;
901			break;
902		case 'c':
903			c_dirname = optarg;
904			break;
905		case 'd':
906			d_flag = 1;
907			break;
908		case 'F':
909			F_flag = 1;
910			break;
911		case 'f':
912			f_filename = optarg;
913			break;
914		case 'H':
915			warnx("the -H option is now implicit, "
916			    "use -U to disable");
917			break;
918		case 'h':
919			h_hostname = optarg;
920			break;
921		case 'i':
922			i_flag = 1;
923			i_filename = optarg;
924			break;
925		case 'l':
926			l_flag = 1;
927			break;
928		case 'o':
929			o_flag = 1;
930			o_filename = optarg;
931			break;
932		case 'M':
933		case 'm':
934			if (r_flag)
935				errx(1, "the -m and -r flags "
936				    "are mutually exclusive");
937			m_flag = 1;
938			break;
939		case 'N':
940			N_filename = optarg;
941			break;
942		case 'n':
943			n_flag = 1;
944			break;
945		case 'P':
946		case 'p':
947			p_flag = 1;
948			break;
949		case 'q':
950			v_level = 0;
951			break;
952		case 'R':
953			R_flag = 1;
954			break;
955		case 'r':
956			if (m_flag)
957				errx(1, "the -m and -r flags "
958				    "are mutually exclusive");
959			r_flag = 1;
960			break;
961		case 'S':
962			S_size = (off_t)strtol(optarg, &end, 10);
963			if (*optarg == '\0' || *end != '\0')
964				errx(1, "invalid size (%s)", optarg);
965			break;
966		case 's':
967			s_flag = 1;
968			break;
969		case 'T':
970			T_secs = strtol(optarg, &end, 10);
971			if (*optarg == '\0' || *end != '\0')
972				errx(1, "invalid timeout (%s)", optarg);
973			break;
974		case 't':
975			t_flag = 1;
976			warnx("warning: the -t option is deprecated");
977			break;
978		case 'U':
979			U_flag = 1;
980			break;
981		case 'v':
982			v_level++;
983			break;
984		case 'w':
985			a_flag = 1;
986			w_secs = strtol(optarg, &end, 10);
987			if (*optarg == '\0' || *end != '\0')
988				errx(1, "invalid delay (%s)", optarg);
989			break;
990		case OPTION_BIND_ADDRESS:
991			setenv("FETCH_BIND_ADDRESS", optarg, 1);
992			break;
993		case OPTION_NO_FTP_PASSIVE_MODE:
994			setenv("FTP_PASSIVE_MODE", "no", 1);
995			break;
996		case OPTION_HTTP_REFERER:
997			setenv("HTTP_REFERER", optarg, 1);
998			break;
999		case OPTION_HTTP_USER_AGENT:
1000			setenv("HTTP_USER_AGENT", optarg, 1);
1001			break;
1002		case OPTION_NO_PROXY:
1003			setenv("NO_PROXY", optarg, 1);
1004			break;
1005		case OPTION_SSL_CA_CERT_FILE:
1006			setenv("SSL_CA_CERT_FILE", optarg, 1);
1007			break;
1008		case OPTION_SSL_CA_CERT_PATH:
1009			setenv("SSL_CA_CERT_PATH", optarg, 1);
1010			break;
1011		case OPTION_SSL_CLIENT_CERT_FILE:
1012			setenv("SSL_CLIENT_CERT_FILE", optarg, 1);
1013			break;
1014		case OPTION_SSL_CLIENT_KEY_FILE:
1015			setenv("SSL_CLIENT_KEY_FILE", optarg, 1);
1016			break;
1017		case OPTION_SSL_CRL_FILE:
1018			setenv("SSL_CLIENT_CRL_FILE", optarg, 1);
1019			break;
1020		case OPTION_SSL_NO_SSL3:
1021			setenv("SSL_NO_SSL3", "", 1);
1022			break;
1023		case OPTION_SSL_NO_TLS1:
1024			setenv("SSL_NO_TLS1", "", 1);
1025			break;
1026		case OPTION_SSL_NO_VERIFY_HOSTNAME:
1027			setenv("SSL_NO_VERIFY_HOSTNAME", "", 1);
1028			break;
1029		case OPTION_SSL_NO_VERIFY_PEER:
1030			setenv("SSL_NO_VERIFY_PEER", "", 1);
1031			break;
1032		default:
1033			usage();
1034			exit(1);
1035		}
1036
1037	argc -= optind;
1038	argv += optind;
1039
1040	if (h_hostname || f_filename || c_dirname) {
1041		if (!h_hostname || !f_filename || argc) {
1042			usage();
1043			exit(1);
1044		}
1045		/* XXX this is a hack. */
1046		if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
1047			errx(1, "invalid hostname");
1048		if (asprintf(argv, "ftp://%s/%s/%s", h_hostname,
1049		    c_dirname ? c_dirname : "", f_filename) == -1)
1050			errx(1, "%s", strerror(ENOMEM));
1051		argc++;
1052	}
1053
1054	if (!argc) {
1055		usage();
1056		exit(1);
1057	}
1058
1059	/* allocate buffer */
1060	if (B_size < MINBUFSIZE)
1061		B_size = MINBUFSIZE;
1062	if ((buf = malloc(B_size)) == NULL)
1063		errx(1, "%s", strerror(ENOMEM));
1064
1065	/* timeouts */
1066	if ((s = getenv("FTP_TIMEOUT")) != NULL) {
1067		ftp_timeout = strtol(s, &end, 10);
1068		if (*s == '\0' || *end != '\0' || ftp_timeout < 0) {
1069			warnx("FTP_TIMEOUT (%s) is not a positive integer", s);
1070			ftp_timeout = 0;
1071		}
1072	}
1073	if ((s = getenv("HTTP_TIMEOUT")) != NULL) {
1074		http_timeout = strtol(s, &end, 10);
1075		if (*s == '\0' || *end != '\0' || http_timeout < 0) {
1076			warnx("HTTP_TIMEOUT (%s) is not a positive integer", s);
1077			http_timeout = 0;
1078		}
1079	}
1080
1081	/* signal handling */
1082	sa.sa_flags = 0;
1083	sa.sa_handler = sig_handler;
1084	sigemptyset(&sa.sa_mask);
1085	sigaction(SIGALRM, &sa, NULL);
1086	sa.sa_flags = SA_RESETHAND;
1087	sigaction(SIGINT, &sa, NULL);
1088	fetchRestartCalls = 0;
1089
1090	/* output file */
1091	if (o_flag) {
1092		if (strcmp(o_filename, "-") == 0) {
1093			o_stdout = 1;
1094		} else if (stat(o_filename, &sb) == -1) {
1095			if (errno == ENOENT) {
1096				if (argc > 1)
1097					errx(1, "%s is not a directory",
1098					    o_filename);
1099			} else {
1100				err(1, "%s", o_filename);
1101			}
1102		} else {
1103			if (sb.st_mode & S_IFDIR)
1104				o_directory = 1;
1105		}
1106	}
1107
1108	/* check if output is to a tty (for progress report) */
1109	v_tty = isatty(STDERR_FILENO);
1110	if (v_tty)
1111		pgrp = getpgrp();
1112
1113	r = 0;
1114
1115	/* authentication */
1116	if (v_tty)
1117		fetchAuthMethod = query_auth;
1118	if (N_filename != NULL)
1119		if (setenv("NETRC", N_filename, 1) == -1)
1120			err(1, "setenv: cannot set NETRC=%s", N_filename);
1121
1122	while (argc) {
1123		if ((p = strrchr(*argv, '/')) == NULL)
1124			p = *argv;
1125		else
1126			p++;
1127
1128		if (!*p)
1129			p = "fetch.out";
1130
1131		fetchLastErrCode = 0;
1132
1133		if (o_flag) {
1134			if (o_stdout) {
1135				e = fetch(*argv, "-");
1136			} else if (o_directory) {
1137				asprintf(&q, "%s/%s", o_filename, p);
1138				e = fetch(*argv, q);
1139				free(q);
1140			} else {
1141				e = fetch(*argv, o_filename);
1142			}
1143		} else {
1144			e = fetch(*argv, p);
1145		}
1146
1147		if (sigint)
1148			kill(getpid(), SIGINT);
1149
1150		if (e == 0 && once_flag)
1151			exit(0);
1152
1153		if (e) {
1154			r = 1;
1155			if ((fetchLastErrCode
1156			    && fetchLastErrCode != FETCH_UNAVAIL
1157			    && fetchLastErrCode != FETCH_MOVED
1158			    && fetchLastErrCode != FETCH_URL
1159			    && fetchLastErrCode != FETCH_RESOLV
1160			    && fetchLastErrCode != FETCH_UNKNOWN)) {
1161				if (w_secs && v_level)
1162					fprintf(stderr, "Waiting %ld seconds "
1163					    "before retrying\n", w_secs);
1164				if (w_secs)
1165					sleep(w_secs);
1166				if (a_flag)
1167					continue;
1168			}
1169		}
1170
1171		argc--, argv++;
1172	}
1173
1174	exit(r);
1175}
1176