1223328Sgavin/* $NetBSD: fetch.c,v 1.18 2009/11/15 10:12:37 lukem Exp $ */ 2223328Sgavin/* from NetBSD: fetch.c,v 1.191 2009/08/17 09:08:16 christos Exp */ 379971Sobrien 479971Sobrien/*- 5223328Sgavin * Copyright (c) 1997-2009 The NetBSD Foundation, Inc. 679971Sobrien * All rights reserved. 779971Sobrien * 879971Sobrien * This code is derived from software contributed to The NetBSD Foundation 979971Sobrien * by Luke Mewburn. 1079971Sobrien * 1179971Sobrien * This code is derived from software contributed to The NetBSD Foundation 1279971Sobrien * by Scott Aaron Bamford. 1379971Sobrien * 1479971Sobrien * Redistribution and use in source and binary forms, with or without 1579971Sobrien * modification, are permitted provided that the following conditions 1679971Sobrien * are met: 1779971Sobrien * 1. Redistributions of source code must retain the above copyright 1879971Sobrien * notice, this list of conditions and the following disclaimer. 1979971Sobrien * 2. Redistributions in binary form must reproduce the above copyright 2079971Sobrien * notice, this list of conditions and the following disclaimer in the 2179971Sobrien * documentation and/or other materials provided with the distribution. 2279971Sobrien * 2379971Sobrien * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 2479971Sobrien * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 2579971Sobrien * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 2679971Sobrien * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 2779971Sobrien * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2879971Sobrien * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2979971Sobrien * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 3079971Sobrien * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 3179971Sobrien * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 3279971Sobrien * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3379971Sobrien * POSSIBILITY OF SUCH DAMAGE. 3479971Sobrien */ 3579971Sobrien 36223328Sgavin#include "tnftp.h" 37223328Sgavin 38223328Sgavin#if 0 /* tnftp */ 39223328Sgavin 40116424Smikeh#include <sys/cdefs.h> 41116424Smikeh#ifndef lint 42223328Sgavin__RCSID(" NetBSD: fetch.c,v 1.191 2009/08/17 09:08:16 christos Exp "); 43116424Smikeh#endif /* not lint */ 44116424Smikeh 4579971Sobrien/* 4679971Sobrien * FTP User Program -- Command line file retrieval 4779971Sobrien */ 4879971Sobrien 49116424Smikeh#include <sys/types.h> 50116424Smikeh#include <sys/param.h> 51116424Smikeh#include <sys/socket.h> 52116424Smikeh#include <sys/stat.h> 53116424Smikeh#include <sys/time.h> 5479971Sobrien 55116424Smikeh#include <netinet/in.h> 56116424Smikeh 57116424Smikeh#include <arpa/ftp.h> 58116424Smikeh#include <arpa/inet.h> 59116424Smikeh 60116424Smikeh#include <ctype.h> 61116424Smikeh#include <err.h> 62116424Smikeh#include <errno.h> 63116424Smikeh#include <netdb.h> 64116424Smikeh#include <fcntl.h> 65116424Smikeh#include <stdio.h> 66116424Smikeh#include <stdlib.h> 67116424Smikeh#include <string.h> 68116424Smikeh#include <unistd.h> 69116424Smikeh#include <time.h> 70116424Smikeh 71223328Sgavin#endif /* tnftp */ 72223328Sgavin 7379971Sobrien#include "ftp_var.h" 7479971Sobrien#include "version.h" 7579971Sobrien 7679971Sobrientypedef enum { 7779971Sobrien UNKNOWN_URL_T=-1, 7879971Sobrien HTTP_URL_T, 7979971Sobrien FTP_URL_T, 8079971Sobrien FILE_URL_T, 8179971Sobrien CLASSIC_URL_T 8279971Sobrien} url_t; 8379971Sobrien 8479971Sobrienvoid aborthttp(int); 85142129Smikeh#ifndef NO_AUTH 8679971Sobrienstatic int auth_url(const char *, char **, const char *, const char *); 87142129Smikehstatic void base64_encode(const unsigned char *, size_t, unsigned char *); 88142129Smikeh#endif 8979971Sobrienstatic int go_fetch(const char *); 9079971Sobrienstatic int fetch_ftp(const char *); 9179971Sobrienstatic int fetch_url(const char *, const char *, char *, char *); 92142129Smikehstatic const char *match_token(const char **, const char *); 9379971Sobrienstatic int parse_url(const char *, const char *, url_t *, char **, 9479971Sobrien char **, char **, char **, in_port_t *, char **); 9579971Sobrienstatic void url_decode(char *); 9679971Sobrien 9779971Sobrienstatic int redirect_loop; 9879971Sobrien 9979971Sobrien 100142129Smikeh#define STRNEQUAL(a,b) (strncasecmp((a), (b), sizeof((b))-1) == 0) 101142129Smikeh#define ISLWS(x) ((x)=='\r' || (x)=='\n' || (x)==' ' || (x)=='\t') 102142129Smikeh#define SKIPLWS(x) do { while (ISLWS((*x))) x++; } while (0) 103142129Smikeh 104142129Smikeh 10579971Sobrien#define ABOUT_URL "about:" /* propaganda */ 10679971Sobrien#define FILE_URL "file://" /* file URL prefix */ 10779971Sobrien#define FTP_URL "ftp://" /* ftp URL prefix */ 10879971Sobrien#define HTTP_URL "http://" /* http URL prefix */ 10979971Sobrien 11079971Sobrien 11179971Sobrien/* 112142129Smikeh * Determine if token is the next word in buf (case insensitive). 113142129Smikeh * If so, advance buf past the token and any trailing LWS, and 114142129Smikeh * return a pointer to the token (in buf). Otherwise, return NULL. 115223328Sgavin * token may be preceded by LWS. 116142129Smikeh * token must be followed by LWS or NUL. (I.e, don't partial match). 117142129Smikeh */ 118142129Smikehstatic const char * 119142129Smikehmatch_token(const char **buf, const char *token) 120142129Smikeh{ 121142129Smikeh const char *p, *orig; 122142129Smikeh size_t tlen; 123142129Smikeh 124142129Smikeh tlen = strlen(token); 125142129Smikeh p = *buf; 126142129Smikeh SKIPLWS(p); 127142129Smikeh orig = p; 128142129Smikeh if (strncasecmp(p, token, tlen) != 0) 129142129Smikeh return NULL; 130142129Smikeh p += tlen; 131142129Smikeh if (*p != '\0' && !ISLWS(*p)) 132142129Smikeh return NULL; 133142129Smikeh SKIPLWS(p); 134142129Smikeh orig = *buf; 135142129Smikeh *buf = p; 136142129Smikeh return orig; 137142129Smikeh} 138142129Smikeh 139142129Smikeh#ifndef NO_AUTH 140142129Smikeh/* 14179971Sobrien * Generate authorization response based on given authentication challenge. 14279971Sobrien * Returns -1 if an error occurred, otherwise 0. 14379971Sobrien * Sets response to a malloc(3)ed string; caller should free. 14479971Sobrien */ 14579971Sobrienstatic int 14679971Sobrienauth_url(const char *challenge, char **response, const char *guser, 14779971Sobrien const char *gpass) 14879971Sobrien{ 149223328Sgavin const char *cp, *scheme, *errormsg; 150142129Smikeh char *ep, *clear, *realm; 151223328Sgavin char uuser[BUFSIZ], *gotpass; 152223328Sgavin const char *upass; 15379971Sobrien int rval; 15479971Sobrien size_t len, clen, rlen; 15579971Sobrien 15679971Sobrien *response = NULL; 157142129Smikeh clear = realm = NULL; 15879971Sobrien rval = -1; 159142129Smikeh cp = challenge; 160142129Smikeh scheme = "Basic"; /* only support Basic authentication */ 161223328Sgavin gotpass = NULL; 16279971Sobrien 163223328Sgavin DPRINTF("auth_url: challenge `%s'\n", challenge); 16479971Sobrien 165142129Smikeh if (! match_token(&cp, scheme)) { 166223328Sgavin warnx("Unsupported authentication challenge `%s'", 16779971Sobrien challenge); 16879971Sobrien goto cleanup_auth_url; 16979971Sobrien } 17079971Sobrien 17179971Sobrien#define REALM "realm=\"" 172142129Smikeh if (STRNEQUAL(cp, REALM)) 17379971Sobrien cp += sizeof(REALM) - 1; 17479971Sobrien else { 175223328Sgavin warnx("Unsupported authentication challenge `%s'", 17679971Sobrien challenge); 17779971Sobrien goto cleanup_auth_url; 17879971Sobrien } 179142129Smikeh/* XXX: need to improve quoted-string parsing to support \ quoting, etc. */ 18079971Sobrien if ((ep = strchr(cp, '\"')) != NULL) { 181223328Sgavin len = ep - cp; 182223328Sgavin realm = (char *)ftp_malloc(len + 1); 18379971Sobrien (void)strlcpy(realm, cp, len + 1); 18479971Sobrien } else { 185223328Sgavin warnx("Unsupported authentication challenge `%s'", 18679971Sobrien challenge); 18779971Sobrien goto cleanup_auth_url; 18879971Sobrien } 18979971Sobrien 190142129Smikeh fprintf(ttyout, "Username for `%s': ", realm); 191142129Smikeh if (guser != NULL) { 192223328Sgavin (void)strlcpy(uuser, guser, sizeof(uuser)); 193223328Sgavin fprintf(ttyout, "%s\n", uuser); 194142129Smikeh } else { 19579971Sobrien (void)fflush(ttyout); 196223328Sgavin if (get_line(stdin, uuser, sizeof(uuser), &errormsg) < 0) { 197223328Sgavin warnx("%s; can't authenticate", errormsg); 19879971Sobrien goto cleanup_auth_url; 19979971Sobrien } 20079971Sobrien } 20179971Sobrien if (gpass != NULL) 202223328Sgavin upass = gpass; 203223328Sgavin else { 204223328Sgavin gotpass = getpass("Password: "); 205223328Sgavin if (gotpass == NULL) { 206223328Sgavin warnx("Can't read password"); 207223328Sgavin goto cleanup_auth_url; 208223328Sgavin } 209223328Sgavin upass = gotpass; 210223328Sgavin } 21179971Sobrien 212223328Sgavin clen = strlen(uuser) + strlen(upass) + 2; /* user + ":" + pass + "\0" */ 213223328Sgavin clear = (char *)ftp_malloc(clen); 214223328Sgavin (void)strlcpy(clear, uuser, clen); 21579971Sobrien (void)strlcat(clear, ":", clen); 216223328Sgavin (void)strlcat(clear, upass, clen); 217223328Sgavin if (gotpass) 218223328Sgavin memset(gotpass, 0, strlen(gotpass)); 21979971Sobrien 22079971Sobrien /* scheme + " " + enc + "\0" */ 22179971Sobrien rlen = strlen(scheme) + 1 + (clen + 2) * 4 / 3 + 1; 222223328Sgavin *response = (char *)ftp_malloc(rlen); 22379971Sobrien (void)strlcpy(*response, scheme, rlen); 22479971Sobrien len = strlcat(*response, " ", rlen); 225142129Smikeh /* use `clen - 1' to not encode the trailing NUL */ 226146309Smikeh base64_encode((unsigned char *)clear, clen - 1, 227146309Smikeh (unsigned char *)*response + len); 22879971Sobrien memset(clear, 0, clen); 22979971Sobrien rval = 0; 23079971Sobrien 23179971Sobrien cleanup_auth_url: 23279971Sobrien FREEPTR(clear); 23379971Sobrien FREEPTR(realm); 23479971Sobrien return (rval); 23579971Sobrien} 23679971Sobrien 23779971Sobrien/* 23879971Sobrien * Encode len bytes starting at clear using base64 encoding into encoded, 23979971Sobrien * which should be at least ((len + 2) * 4 / 3 + 1) in size. 24079971Sobrien */ 24179971Sobrienstatic void 242142129Smikehbase64_encode(const unsigned char *clear, size_t len, unsigned char *encoded) 24379971Sobrien{ 244142129Smikeh static const unsigned char enc[] = 24579971Sobrien "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 246142129Smikeh unsigned char *cp; 247223328Sgavin size_t i; 24879971Sobrien 24979971Sobrien cp = encoded; 25079971Sobrien for (i = 0; i < len; i += 3) { 25179971Sobrien *(cp++) = enc[((clear[i + 0] >> 2))]; 25279971Sobrien *(cp++) = enc[((clear[i + 0] << 4) & 0x30) 25379971Sobrien | ((clear[i + 1] >> 4) & 0x0f)]; 25479971Sobrien *(cp++) = enc[((clear[i + 1] << 2) & 0x3c) 25579971Sobrien | ((clear[i + 2] >> 6) & 0x03)]; 25679971Sobrien *(cp++) = enc[((clear[i + 2] ) & 0x3f)]; 25779971Sobrien } 25879971Sobrien *cp = '\0'; 25979971Sobrien while (i-- > len) 26079971Sobrien *(--cp) = '='; 26179971Sobrien} 262142129Smikeh#endif 26379971Sobrien 26479971Sobrien/* 26579971Sobrien * Decode %xx escapes in given string, `in-place'. 26679971Sobrien */ 26779971Sobrienstatic void 26879971Sobrienurl_decode(char *url) 26979971Sobrien{ 27079971Sobrien unsigned char *p, *q; 27179971Sobrien 27279971Sobrien if (EMPTYSTRING(url)) 27379971Sobrien return; 27479971Sobrien p = q = (unsigned char *)url; 27579971Sobrien 27679971Sobrien#define HEXTOINT(x) (x - (isdigit(x) ? '0' : (islower(x) ? 'a' : 'A') - 10)) 27779971Sobrien while (*p) { 27879971Sobrien if (p[0] == '%' 27979971Sobrien && p[1] && isxdigit((unsigned char)p[1]) 28079971Sobrien && p[2] && isxdigit((unsigned char)p[2])) { 28179971Sobrien *q++ = HEXTOINT(p[1]) * 16 + HEXTOINT(p[2]); 28279971Sobrien p+=3; 28379971Sobrien } else 28479971Sobrien *q++ = *p++; 28579971Sobrien } 28679971Sobrien *q = '\0'; 28779971Sobrien} 28879971Sobrien 28979971Sobrien 29079971Sobrien/* 291223328Sgavin * Parse URL of form (per RFC3986): 292128671Smikeh * <type>://[<user>[:<password>]@]<host>[:<port>][/<path>] 29379971Sobrien * Returns -1 if a parse error occurred, otherwise 0. 29479971Sobrien * It's the caller's responsibility to url_decode() the returned 29579971Sobrien * user, pass and path. 29679971Sobrien * 29779971Sobrien * Sets type to url_t, each of the given char ** pointers to a 29879971Sobrien * malloc(3)ed strings of the relevant section, and port to 29979971Sobrien * the number given, or ftpport if ftp://, or httpport if http://. 30079971Sobrien * 301223328Sgavin * XXX: this is not totally RFC3986 compliant; <path> will have the 30279971Sobrien * leading `/' unless it's an ftp:// URL, as this makes things easier 303223328Sgavin * for file:// and http:// URLs. ftp:// URLs have the `/' between the 304116424Smikeh * host and the URL-path removed, but any additional leading slashes 305116424Smikeh * in the URL-path are retained (because they imply that we should 30679971Sobrien * later do "CWD" with a null argument). 30779971Sobrien * 30879971Sobrien * Examples: 309116424Smikeh * input URL output path 31079971Sobrien * --------- ----------- 311223328Sgavin * "http://host" "/" 312223328Sgavin * "http://host/" "/" 313223328Sgavin * "http://host/path" "/path" 31479971Sobrien * "file://host/dir/file" "dir/file" 315223328Sgavin * "ftp://host" "" 31679971Sobrien * "ftp://host/" "" 317223328Sgavin * "ftp://host//" "/" 318223328Sgavin * "ftp://host/dir/file" "dir/file" 31979971Sobrien * "ftp://host//dir/file" "/dir/file" 32079971Sobrien */ 32179971Sobrienstatic int 322223328Sgavinparse_url(const char *url, const char *desc, url_t *utype, 323223328Sgavin char **uuser, char **pass, char **host, char **port, 32479971Sobrien in_port_t *portnum, char **path) 32579971Sobrien{ 326223328Sgavin const char *origurl, *tport; 327223328Sgavin char *cp, *ep, *thost; 32879971Sobrien size_t len; 32979971Sobrien 330223328Sgavin if (url == NULL || desc == NULL || utype == NULL || uuser == NULL 33179971Sobrien || pass == NULL || host == NULL || port == NULL || portnum == NULL 33279971Sobrien || path == NULL) 33379971Sobrien errx(1, "parse_url: invoked with NULL argument!"); 334223328Sgavin DPRINTF("parse_url: %s `%s'\n", desc, url); 33579971Sobrien 33679971Sobrien origurl = url; 337223328Sgavin *utype = UNKNOWN_URL_T; 338223328Sgavin *uuser = *pass = *host = *port = *path = NULL; 33979971Sobrien *portnum = 0; 34079971Sobrien tport = NULL; 34179971Sobrien 342142129Smikeh if (STRNEQUAL(url, HTTP_URL)) { 34379971Sobrien url += sizeof(HTTP_URL) - 1; 344223328Sgavin *utype = HTTP_URL_T; 34579971Sobrien *portnum = HTTP_PORT; 34679971Sobrien tport = httpport; 347142129Smikeh } else if (STRNEQUAL(url, FTP_URL)) { 34879971Sobrien url += sizeof(FTP_URL) - 1; 349223328Sgavin *utype = FTP_URL_T; 35079971Sobrien *portnum = FTP_PORT; 35179971Sobrien tport = ftpport; 352142129Smikeh } else if (STRNEQUAL(url, FILE_URL)) { 35379971Sobrien url += sizeof(FILE_URL) - 1; 354223328Sgavin *utype = FILE_URL_T; 35579971Sobrien } else { 35679971Sobrien warnx("Invalid %s `%s'", desc, url); 35779971Sobrien cleanup_parse_url: 358223328Sgavin FREEPTR(*uuser); 359223328Sgavin if (*pass != NULL) 360223328Sgavin memset(*pass, 0, strlen(*pass)); 36179971Sobrien FREEPTR(*pass); 36279971Sobrien FREEPTR(*host); 36379971Sobrien FREEPTR(*port); 36479971Sobrien FREEPTR(*path); 36579971Sobrien return (-1); 36679971Sobrien } 36779971Sobrien 36879971Sobrien if (*url == '\0') 36979971Sobrien return (0); 37079971Sobrien 37179971Sobrien /* find [user[:pass]@]host[:port] */ 37279971Sobrien ep = strchr(url, '/'); 37379971Sobrien if (ep == NULL) 374223328Sgavin thost = ftp_strdup(url); 37579971Sobrien else { 37679971Sobrien len = ep - url; 377223328Sgavin thost = (char *)ftp_malloc(len + 1); 37879971Sobrien (void)strlcpy(thost, url, len + 1); 379223328Sgavin if (*utype == FTP_URL_T) /* skip first / for ftp URLs */ 38079971Sobrien ep++; 381223328Sgavin *path = ftp_strdup(ep); 38279971Sobrien } 38379971Sobrien 38479971Sobrien cp = strchr(thost, '@'); /* look for user[:pass]@ in URLs */ 38579971Sobrien if (cp != NULL) { 386223328Sgavin if (*utype == FTP_URL_T) 38779971Sobrien anonftp = 0; /* disable anonftp */ 388223328Sgavin *uuser = thost; 38979971Sobrien *cp = '\0'; 390223328Sgavin thost = ftp_strdup(cp + 1); 391223328Sgavin cp = strchr(*uuser, ':'); 39279971Sobrien if (cp != NULL) { 39379971Sobrien *cp = '\0'; 394223328Sgavin *pass = ftp_strdup(cp + 1); 39579971Sobrien } 396223328Sgavin url_decode(*uuser); 397142129Smikeh if (*pass) 398142129Smikeh url_decode(*pass); 39979971Sobrien } 40079971Sobrien 40179971Sobrien#ifdef INET6 40279971Sobrien /* 40379971Sobrien * Check if thost is an encoded IPv6 address, as per 404223328Sgavin * RFC3986: 40579971Sobrien * `[' ipv6-address ']' 40679971Sobrien */ 40779971Sobrien if (*thost == '[') { 40879971Sobrien cp = thost + 1; 40979971Sobrien if ((ep = strchr(cp, ']')) == NULL || 41079971Sobrien (ep[1] != '\0' && ep[1] != ':')) { 41179971Sobrien warnx("Invalid address `%s' in %s `%s'", 41279971Sobrien thost, desc, origurl); 41379971Sobrien goto cleanup_parse_url; 41479971Sobrien } 41579971Sobrien len = ep - cp; /* change `[xyz]' -> `xyz' */ 41679971Sobrien memmove(thost, thost + 1, len); 41779971Sobrien thost[len] = '\0'; 41879971Sobrien if (! isipv6addr(thost)) { 41979971Sobrien warnx("Invalid IPv6 address `%s' in %s `%s'", 42079971Sobrien thost, desc, origurl); 42179971Sobrien goto cleanup_parse_url; 42279971Sobrien } 42379971Sobrien cp = ep + 1; 42479971Sobrien if (*cp == ':') 42579971Sobrien cp++; 42679971Sobrien else 42779971Sobrien cp = NULL; 42879971Sobrien } else 42979971Sobrien#endif /* INET6 */ 430223328Sgavin if ((cp = strchr(thost, ':')) != NULL) 431223328Sgavin *cp++ = '\0'; 43279971Sobrien *host = thost; 43379971Sobrien 43479971Sobrien /* look for [:port] */ 43579971Sobrien if (cp != NULL) { 436223328Sgavin unsigned long nport; 43779971Sobrien 438223328Sgavin nport = strtoul(cp, &ep, 10); 439223328Sgavin if (*cp == '\0' || *ep != '\0' || 440223328Sgavin nport < 1 || nport > MAX_IN_PORT_T) { 44179971Sobrien warnx("Unknown port `%s' in %s `%s'", 44279971Sobrien cp, desc, origurl); 44379971Sobrien goto cleanup_parse_url; 44479971Sobrien } 44579971Sobrien *portnum = nport; 44679971Sobrien tport = cp; 44779971Sobrien } 44879971Sobrien 44979971Sobrien if (tport != NULL) 450223328Sgavin *port = ftp_strdup(tport); 451223328Sgavin if (*path == NULL) { 452223328Sgavin const char *emptypath = "/"; 453223328Sgavin if (*utype == FTP_URL_T) /* skip first / for ftp URLs */ 454223328Sgavin emptypath++; 455223328Sgavin *path = ftp_strdup(emptypath); 456223328Sgavin } 45779971Sobrien 458223328Sgavin DPRINTF("parse_url: user `%s' pass `%s' host %s port %s(%d) " 459223328Sgavin "path `%s'\n", 460223328Sgavin STRorNULL(*uuser), STRorNULL(*pass), 461223328Sgavin STRorNULL(*host), STRorNULL(*port), 462223328Sgavin *portnum ? *portnum : -1, STRorNULL(*path)); 46379971Sobrien 46479971Sobrien return (0); 46579971Sobrien} 46679971Sobrien 46779971Sobriensigjmp_buf httpabort; 46879971Sobrien 46979971Sobrien/* 47079971Sobrien * Retrieve URL, via a proxy if necessary, using HTTP. 47179971Sobrien * If proxyenv is set, use that for the proxy, otherwise try ftp_proxy or 47279971Sobrien * http_proxy as appropriate. 47379971Sobrien * Supports HTTP redirects. 47498247Smikeh * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection 47579971Sobrien * is still open (e.g, ftp xfer with trailing /) 47679971Sobrien */ 47779971Sobrienstatic int 47879971Sobrienfetch_url(const char *url, const char *proxyenv, char *proxyauth, char *wwwauth) 47979971Sobrien{ 48079971Sobrien struct addrinfo hints, *res, *res0 = NULL; 48179971Sobrien int error; 482223328Sgavin sigfunc volatile oldintr; 483223328Sgavin sigfunc volatile oldintp; 484223328Sgavin int volatile s; 48579971Sobrien struct stat sb; 486223328Sgavin int volatile ischunked; 487223328Sgavin int volatile isproxy; 488223328Sgavin int volatile rval; 489223328Sgavin int volatile hcode; 490223328Sgavin int len; 491223328Sgavin size_t flen; 49279971Sobrien static size_t bufsize; 49379971Sobrien static char *xferbuf; 494142129Smikeh const char *cp, *token; 495223328Sgavin char *ep; 496223328Sgavin char buf[FTPBUFLEN]; 497223328Sgavin const char *errormsg; 498223328Sgavin char *volatile savefile; 499223328Sgavin char *volatile auth; 500223328Sgavin char *volatile location; 501223328Sgavin char *volatile message; 502223328Sgavin char *uuser, *pass, *host, *port, *path; 503223328Sgavin char *volatile decodedpath; 504121966Smikeh char *puser, *ppass, *useragent; 50579971Sobrien off_t hashbytes, rangestart, rangeend, entitylen; 506223328Sgavin int (*volatile closefunc)(FILE *); 507223328Sgavin FILE *volatile fin; 508223328Sgavin FILE *volatile fout; 50979971Sobrien time_t mtime; 51079971Sobrien url_t urltype; 51179971Sobrien in_port_t portnum; 51279971Sobrien 513223328Sgavin DPRINTF("fetch_url: `%s' proxyenv `%s'\n", url, STRorNULL(proxyenv)); 514223328Sgavin 51579971Sobrien oldintr = oldintp = NULL; 51679971Sobrien closefunc = NULL; 51779971Sobrien fin = fout = NULL; 51879971Sobrien s = -1; 519223328Sgavin savefile = NULL; 52079971Sobrien auth = location = message = NULL; 52179971Sobrien ischunked = isproxy = hcode = 0; 52279971Sobrien rval = 1; 523223328Sgavin uuser = pass = host = path = decodedpath = puser = ppass = NULL; 52479971Sobrien 525223328Sgavin if (parse_url(url, "URL", &urltype, &uuser, &pass, &host, &port, 52679971Sobrien &portnum, &path) == -1) 52779971Sobrien goto cleanup_fetch_url; 52879971Sobrien 52979971Sobrien if (urltype == FILE_URL_T && ! EMPTYSTRING(host) 53079971Sobrien && strcasecmp(host, "localhost") != 0) { 53179971Sobrien warnx("No support for non local file URL `%s'", url); 53279971Sobrien goto cleanup_fetch_url; 53379971Sobrien } 53479971Sobrien 53579971Sobrien if (EMPTYSTRING(path)) { 53679971Sobrien if (urltype == FTP_URL_T) { 53779971Sobrien rval = fetch_ftp(url); 53879971Sobrien goto cleanup_fetch_url; 53979971Sobrien } 54079971Sobrien if (urltype != HTTP_URL_T || outfile == NULL) { 54179971Sobrien warnx("Invalid URL (no file after host) `%s'", url); 54279971Sobrien goto cleanup_fetch_url; 54379971Sobrien } 54479971Sobrien } 54579971Sobrien 546223328Sgavin decodedpath = ftp_strdup(path); 54779971Sobrien url_decode(decodedpath); 54879971Sobrien 54979971Sobrien if (outfile) 550274110Sdes savefile = outfile; 55179971Sobrien else { 55279971Sobrien cp = strrchr(decodedpath, '/'); /* find savefile */ 55379971Sobrien if (cp != NULL) 554223328Sgavin savefile = ftp_strdup(cp + 1); 55579971Sobrien else 556223328Sgavin savefile = ftp_strdup(decodedpath); 55779971Sobrien } 558223328Sgavin DPRINTF("fetch_url: savefile `%s'\n", savefile); 55979971Sobrien if (EMPTYSTRING(savefile)) { 56079971Sobrien if (urltype == FTP_URL_T) { 56179971Sobrien rval = fetch_ftp(url); 56279971Sobrien goto cleanup_fetch_url; 56379971Sobrien } 564223328Sgavin warnx("No file after directory (you must specify an " 565116424Smikeh "output file) `%s'", url); 56679971Sobrien goto cleanup_fetch_url; 56779971Sobrien } 56879971Sobrien 56979971Sobrien restart_point = 0; 57079971Sobrien filesize = -1; 57179971Sobrien rangestart = rangeend = entitylen = -1; 57279971Sobrien mtime = -1; 57379971Sobrien if (restartautofetch) { 574274110Sdes if (stat(savefile, &sb) == 0) 57579971Sobrien restart_point = sb.st_size; 57679971Sobrien } 57779971Sobrien if (urltype == FILE_URL_T) { /* file:// URLs */ 57879971Sobrien direction = "copied"; 57979971Sobrien fin = fopen(decodedpath, "r"); 58079971Sobrien if (fin == NULL) { 581223328Sgavin warn("Can't open `%s'", decodedpath); 58279971Sobrien goto cleanup_fetch_url; 58379971Sobrien } 58479971Sobrien if (fstat(fileno(fin), &sb) == 0) { 58579971Sobrien mtime = sb.st_mtime; 58679971Sobrien filesize = sb.st_size; 58779971Sobrien } 58879971Sobrien if (restart_point) { 58979971Sobrien if (lseek(fileno(fin), restart_point, SEEK_SET) < 0) { 590223328Sgavin warn("Can't seek to restart `%s'", 59179971Sobrien decodedpath); 59279971Sobrien goto cleanup_fetch_url; 59379971Sobrien } 59479971Sobrien } 59579971Sobrien if (verbose) { 59679971Sobrien fprintf(ttyout, "Copying %s", decodedpath); 59779971Sobrien if (restart_point) 59879971Sobrien fprintf(ttyout, " (restarting at " LLF ")", 59979971Sobrien (LLT)restart_point); 60079971Sobrien fputs("\n", ttyout); 60179971Sobrien } 60279971Sobrien } else { /* ftp:// or http:// URLs */ 603223328Sgavin const char *leading; 60479971Sobrien int hasleading; 60579971Sobrien 60679971Sobrien if (proxyenv == NULL) { 60779971Sobrien if (urltype == HTTP_URL_T) 60879971Sobrien proxyenv = getoptionvalue("http_proxy"); 60979971Sobrien else if (urltype == FTP_URL_T) 61079971Sobrien proxyenv = getoptionvalue("ftp_proxy"); 61179971Sobrien } 61279971Sobrien direction = "retrieved"; 61379971Sobrien if (! EMPTYSTRING(proxyenv)) { /* use proxy */ 61479971Sobrien url_t purltype; 61579971Sobrien char *phost, *ppath; 61679971Sobrien char *pport, *no_proxy; 617223328Sgavin in_port_t pportnum; 61879971Sobrien 61979971Sobrien isproxy = 1; 62079971Sobrien 62179971Sobrien /* check URL against list of no_proxied sites */ 62279971Sobrien no_proxy = getoptionvalue("no_proxy"); 62379971Sobrien if (! EMPTYSTRING(no_proxy)) { 624223328Sgavin char *np, *np_copy, *np_iter; 625223328Sgavin unsigned long np_port; 62679971Sobrien size_t hlen, plen; 62779971Sobrien 628223328Sgavin np_iter = np_copy = ftp_strdup(no_proxy); 62979971Sobrien hlen = strlen(host); 630223328Sgavin while ((cp = strsep(&np_iter, " ,")) != NULL) { 63179971Sobrien if (*cp == '\0') 63279971Sobrien continue; 63379971Sobrien if ((np = strrchr(cp, ':')) != NULL) { 634223328Sgavin *np++ = '\0'; 635223328Sgavin np_port = strtoul(np, &ep, 10); 636223328Sgavin if (*np == '\0' || *ep != '\0') 63779971Sobrien continue; 63879971Sobrien if (np_port != portnum) 63979971Sobrien continue; 64079971Sobrien } 64179971Sobrien plen = strlen(cp); 64279971Sobrien if (hlen < plen) 64379971Sobrien continue; 64479971Sobrien if (strncasecmp(host + hlen - plen, 64579971Sobrien cp, plen) == 0) { 64679971Sobrien isproxy = 0; 64779971Sobrien break; 64879971Sobrien } 64979971Sobrien } 65079971Sobrien FREEPTR(np_copy); 65198247Smikeh if (isproxy == 0 && urltype == FTP_URL_T) { 65298247Smikeh rval = fetch_ftp(url); 65398247Smikeh goto cleanup_fetch_url; 65498247Smikeh } 65579971Sobrien } 65679971Sobrien 65779971Sobrien if (isproxy) { 658223328Sgavin if (restart_point) { 659223328Sgavin warnx("Can't restart via proxy URL `%s'", 660223328Sgavin proxyenv); 661223328Sgavin goto cleanup_fetch_url; 662223328Sgavin } 66379971Sobrien if (parse_url(proxyenv, "proxy URL", &purltype, 664223328Sgavin &puser, &ppass, &phost, &pport, &pportnum, 66579971Sobrien &ppath) == -1) 66679971Sobrien goto cleanup_fetch_url; 66779971Sobrien 66879971Sobrien if ((purltype != HTTP_URL_T 66979971Sobrien && purltype != FTP_URL_T) || 67079971Sobrien EMPTYSTRING(phost) || 67179971Sobrien (! EMPTYSTRING(ppath) 67279971Sobrien && strcmp(ppath, "/") != 0)) { 67379971Sobrien warnx("Malformed proxy URL `%s'", 67479971Sobrien proxyenv); 67579971Sobrien FREEPTR(phost); 67679971Sobrien FREEPTR(pport); 67779971Sobrien FREEPTR(ppath); 67879971Sobrien goto cleanup_fetch_url; 67979971Sobrien } 68079971Sobrien if (isipv6addr(host) && 68179971Sobrien strchr(host, '%') != NULL) { 68279971Sobrien warnx( 68379971Sobrien"Scoped address notation `%s' disallowed via web proxy", 68479971Sobrien host); 68579971Sobrien FREEPTR(phost); 68679971Sobrien FREEPTR(pport); 68779971Sobrien FREEPTR(ppath); 68879971Sobrien goto cleanup_fetch_url; 68979971Sobrien } 69079971Sobrien 69179971Sobrien FREEPTR(host); 69279971Sobrien host = phost; 69379971Sobrien FREEPTR(port); 69479971Sobrien port = pport; 69579971Sobrien FREEPTR(path); 696223328Sgavin path = ftp_strdup(url); 69779971Sobrien FREEPTR(ppath); 69879971Sobrien } 69979971Sobrien } /* ! EMPTYSTRING(proxyenv) */ 70079971Sobrien 70179971Sobrien memset(&hints, 0, sizeof(hints)); 70279971Sobrien hints.ai_flags = 0; 70395504Smikeh hints.ai_family = family; 70479971Sobrien hints.ai_socktype = SOCK_STREAM; 70579971Sobrien hints.ai_protocol = 0; 706223328Sgavin error = getaddrinfo(host, port, &hints, &res0); 70779971Sobrien if (error) { 708223328Sgavin warnx("Can't lookup `%s:%s': %s", host, port, 709223328Sgavin (error == EAI_SYSTEM) ? strerror(errno) 710223328Sgavin : gai_strerror(error)); 71179971Sobrien goto cleanup_fetch_url; 71279971Sobrien } 71379971Sobrien if (res0->ai_canonname) 71479971Sobrien host = res0->ai_canonname; 71579971Sobrien 71679971Sobrien s = -1; 71779971Sobrien for (res = res0; res; res = res->ai_next) { 718223328Sgavin char hname[NI_MAXHOST], sname[NI_MAXSERV]; 719223328Sgavin 72079971Sobrien ai_unmapped(res); 72179971Sobrien if (getnameinfo(res->ai_addr, res->ai_addrlen, 722223328Sgavin hname, sizeof(hname), sname, sizeof(sname), 723223328Sgavin NI_NUMERICHOST | NI_NUMERICSERV) != 0) { 724223328Sgavin strlcpy(hname, "?", sizeof(hname)); 725223328Sgavin strlcpy(sname, "?", sizeof(sname)); 726223328Sgavin } 72779971Sobrien 728223328Sgavin if (verbose && res0->ai_next) { 729223328Sgavin fprintf(ttyout, "Trying %s:%s ...\n", 730223328Sgavin hname, sname); 731223328Sgavin } 73279971Sobrien 73379971Sobrien s = socket(res->ai_family, SOCK_STREAM, 73479971Sobrien res->ai_protocol); 73579971Sobrien if (s < 0) { 736223328Sgavin warn( 737223328Sgavin "Can't create socket for connection to " 738223328Sgavin "`%s:%s'", hname, sname); 73979971Sobrien continue; 74079971Sobrien } 74179971Sobrien 742223328Sgavin if (ftp_connect(s, res->ai_addr, res->ai_addrlen) < 0) { 74379971Sobrien close(s); 74479971Sobrien s = -1; 74579971Sobrien continue; 74679971Sobrien } 74779971Sobrien 74879971Sobrien /* success */ 74979971Sobrien break; 75079971Sobrien } 75179971Sobrien 75279971Sobrien if (s < 0) { 753223328Sgavin warnx("Can't connect to `%s:%s'", host, port); 75479971Sobrien goto cleanup_fetch_url; 75579971Sobrien } 75679971Sobrien 75779971Sobrien fin = fdopen(s, "r+"); 75879971Sobrien /* 75979971Sobrien * Construct and send the request. 76079971Sobrien */ 76179971Sobrien if (verbose) 76279971Sobrien fprintf(ttyout, "Requesting %s\n", url); 76379971Sobrien leading = " ("; 76479971Sobrien hasleading = 0; 76579971Sobrien if (isproxy) { 76679971Sobrien if (verbose) { 76779971Sobrien fprintf(ttyout, "%svia %s:%s", leading, 76879971Sobrien host, port); 76979971Sobrien leading = ", "; 77079971Sobrien hasleading++; 77179971Sobrien } 77279971Sobrien fprintf(fin, "GET %s HTTP/1.0\r\n", path); 77379971Sobrien if (flushcache) 77479971Sobrien fprintf(fin, "Pragma: no-cache\r\n"); 77579971Sobrien } else { 77679971Sobrien fprintf(fin, "GET %s HTTP/1.1\r\n", path); 77779971Sobrien if (strchr(host, ':')) { 77879971Sobrien char *h, *p; 77979971Sobrien 78079971Sobrien /* 78179971Sobrien * strip off IPv6 scope identifier, since it is 78279971Sobrien * local to the node 78379971Sobrien */ 784223328Sgavin h = ftp_strdup(host); 78579971Sobrien if (isipv6addr(h) && 78679971Sobrien (p = strchr(h, '%')) != NULL) { 78779971Sobrien *p = '\0'; 78879971Sobrien } 78998247Smikeh fprintf(fin, "Host: [%s]", h); 79079971Sobrien free(h); 79179971Sobrien } else 79298247Smikeh fprintf(fin, "Host: %s", host); 79398247Smikeh if (portnum != HTTP_PORT) 79498247Smikeh fprintf(fin, ":%u", portnum); 79598247Smikeh fprintf(fin, "\r\n"); 79679971Sobrien fprintf(fin, "Accept: */*\r\n"); 79779971Sobrien fprintf(fin, "Connection: close\r\n"); 79879971Sobrien if (restart_point) { 79979971Sobrien fputs(leading, ttyout); 80079971Sobrien fprintf(fin, "Range: bytes=" LLF "-\r\n", 80179971Sobrien (LLT)restart_point); 80279971Sobrien fprintf(ttyout, "restarting at " LLF, 80379971Sobrien (LLT)restart_point); 80479971Sobrien leading = ", "; 80579971Sobrien hasleading++; 80679971Sobrien } 80779971Sobrien if (flushcache) 80879971Sobrien fprintf(fin, "Cache-Control: no-cache\r\n"); 80979971Sobrien } 810121966Smikeh if ((useragent=getenv("FTPUSERAGENT")) != NULL) { 811121966Smikeh fprintf(fin, "User-Agent: %s\r\n", useragent); 812121966Smikeh } else { 813121966Smikeh fprintf(fin, "User-Agent: %s/%s\r\n", 814121966Smikeh FTP_PRODUCT, FTP_VERSION); 815121966Smikeh } 81679971Sobrien if (wwwauth) { 81779971Sobrien if (verbose) { 81879971Sobrien fprintf(ttyout, "%swith authorization", 81979971Sobrien leading); 82079971Sobrien leading = ", "; 82179971Sobrien hasleading++; 82279971Sobrien } 82379971Sobrien fprintf(fin, "Authorization: %s\r\n", wwwauth); 82479971Sobrien } 82579971Sobrien if (proxyauth) { 82679971Sobrien if (verbose) { 82779971Sobrien fprintf(ttyout, 82879971Sobrien "%swith proxy authorization", leading); 82979971Sobrien leading = ", "; 83079971Sobrien hasleading++; 83179971Sobrien } 83279971Sobrien fprintf(fin, "Proxy-Authorization: %s\r\n", proxyauth); 83379971Sobrien } 83479971Sobrien if (verbose && hasleading) 83579971Sobrien fputs(")\n", ttyout); 83679971Sobrien fprintf(fin, "\r\n"); 83779971Sobrien if (fflush(fin) == EOF) { 83879971Sobrien warn("Writing HTTP request"); 83979971Sobrien goto cleanup_fetch_url; 84079971Sobrien } 84179971Sobrien 84279971Sobrien /* Read the response */ 843223328Sgavin len = get_line(fin, buf, sizeof(buf), &errormsg); 844223328Sgavin if (len < 0) { 845223328Sgavin if (*errormsg == '\n') 846223328Sgavin errormsg++; 847223328Sgavin warnx("Receiving HTTP reply: %s", errormsg); 84879971Sobrien goto cleanup_fetch_url; 84979971Sobrien } 850142129Smikeh while (len > 0 && (ISLWS(buf[len-1]))) 85179971Sobrien buf[--len] = '\0'; 852223328Sgavin DPRINTF("fetch_url: received `%s'\n", buf); 85379971Sobrien 85479971Sobrien /* Determine HTTP response code */ 85579971Sobrien cp = strchr(buf, ' '); 85679971Sobrien if (cp == NULL) 85779971Sobrien goto improper; 85879971Sobrien else 85979971Sobrien cp++; 86079971Sobrien hcode = strtol(cp, &ep, 10); 86179971Sobrien if (*ep != '\0' && !isspace((unsigned char)*ep)) 86279971Sobrien goto improper; 863223328Sgavin message = ftp_strdup(cp); 86479971Sobrien 86579971Sobrien /* Read the rest of the header. */ 86679971Sobrien while (1) { 867223328Sgavin len = get_line(fin, buf, sizeof(buf), &errormsg); 868223328Sgavin if (len < 0) { 869223328Sgavin if (*errormsg == '\n') 870223328Sgavin errormsg++; 871223328Sgavin warnx("Receiving HTTP reply: %s", errormsg); 87279971Sobrien goto cleanup_fetch_url; 87379971Sobrien } 874142129Smikeh while (len > 0 && (ISLWS(buf[len-1]))) 87579971Sobrien buf[--len] = '\0'; 87679971Sobrien if (len == 0) 87779971Sobrien break; 878223328Sgavin DPRINTF("fetch_url: received `%s'\n", buf); 87979971Sobrien 880142129Smikeh /* 881142129Smikeh * Look for some headers 882142129Smikeh */ 883142129Smikeh 88479971Sobrien cp = buf; 88579971Sobrien 886142129Smikeh if (match_token(&cp, "Content-Length:")) { 88779971Sobrien filesize = STRTOLL(cp, &ep, 10); 88879971Sobrien if (filesize < 0 || *ep != '\0') 88979971Sobrien goto improper; 890223328Sgavin DPRINTF("fetch_url: parsed len as: " LLF "\n", 891223328Sgavin (LLT)filesize); 89279971Sobrien 893142129Smikeh } else if (match_token(&cp, "Content-Range:")) { 894142129Smikeh if (! match_token(&cp, "bytes")) 895142129Smikeh goto improper; 896142129Smikeh 897142129Smikeh if (*cp == '*') 898142129Smikeh cp++; 89998247Smikeh else { 90098247Smikeh rangestart = STRTOLL(cp, &ep, 10); 90198247Smikeh if (rangestart < 0 || *ep != '-') 90298247Smikeh goto improper; 90398247Smikeh cp = ep + 1; 90498247Smikeh rangeend = STRTOLL(cp, &ep, 10); 90598247Smikeh if (rangeend < 0 || rangeend < rangestart) 90698247Smikeh goto improper; 907142129Smikeh cp = ep; 90898247Smikeh } 909142129Smikeh if (*cp != '/') 91079971Sobrien goto improper; 911142129Smikeh cp++; 912142129Smikeh if (*cp == '*') 913142129Smikeh cp++; 91498247Smikeh else { 91598247Smikeh entitylen = STRTOLL(cp, &ep, 10); 91698247Smikeh if (entitylen < 0) 91798247Smikeh goto improper; 918142129Smikeh cp = ep; 91998247Smikeh } 920142129Smikeh if (*cp != '\0') 92179971Sobrien goto improper; 92279971Sobrien 923223328Sgavin#ifndef NO_DEBUG 924223328Sgavin if (ftp_debug) { 92598247Smikeh fprintf(ttyout, "parsed range as: "); 92698247Smikeh if (rangestart == -1) 92798247Smikeh fprintf(ttyout, "*"); 92898247Smikeh else 92998247Smikeh fprintf(ttyout, LLF "-" LLF, 93098247Smikeh (LLT)rangestart, 93198247Smikeh (LLT)rangeend); 93298247Smikeh fprintf(ttyout, "/" LLF "\n", (LLT)entitylen); 93398247Smikeh } 934223328Sgavin#endif 93579971Sobrien if (! restart_point) { 93679971Sobrien warnx( 93779971Sobrien "Received unexpected Content-Range header"); 93879971Sobrien goto cleanup_fetch_url; 93979971Sobrien } 94079971Sobrien 941142129Smikeh } else if (match_token(&cp, "Last-Modified:")) { 94279971Sobrien struct tm parsed; 94379971Sobrien char *t; 94479971Sobrien 945223328Sgavin memset(&parsed, 0, sizeof(parsed)); 946223328Sgavin /* RFC1123 */ 94779971Sobrien if ((t = strptime(cp, 94879971Sobrien "%a, %d %b %Y %H:%M:%S GMT", 94979971Sobrien &parsed)) 950223328Sgavin /* RFC0850 */ 95179971Sobrien || (t = strptime(cp, 95279971Sobrien "%a, %d-%b-%y %H:%M:%S GMT", 95379971Sobrien &parsed)) 95479971Sobrien /* asctime */ 95579971Sobrien || (t = strptime(cp, 95679971Sobrien "%a, %b %d %H:%M:%S %Y", 95779971Sobrien &parsed))) { 95879971Sobrien parsed.tm_isdst = -1; 95979971Sobrien if (*t == '\0') 96079971Sobrien mtime = timegm(&parsed); 961223328Sgavin#ifndef NO_DEBUG 962223328Sgavin if (ftp_debug && mtime != -1) { 96379971Sobrien fprintf(ttyout, 96479971Sobrien "parsed date as: %s", 965223328Sgavin rfc2822time(localtime(&mtime))); 96679971Sobrien } 967223328Sgavin#endif 96879971Sobrien } 96979971Sobrien 970142129Smikeh } else if (match_token(&cp, "Location:")) { 971223328Sgavin location = ftp_strdup(cp); 972223328Sgavin DPRINTF("fetch_url: parsed location as `%s'\n", 973223328Sgavin cp); 97479971Sobrien 975142129Smikeh } else if (match_token(&cp, "Transfer-Encoding:")) { 976142129Smikeh if (match_token(&cp, "binary")) { 97779971Sobrien warnx( 978223328Sgavin "Bogus transfer encoding `binary' (fetching anyway)"); 97979971Sobrien continue; 98079971Sobrien } 981142129Smikeh if (! (token = match_token(&cp, "chunked"))) { 98279971Sobrien warnx( 983223328Sgavin "Unsupported transfer encoding `%s'", 984142129Smikeh token); 98579971Sobrien goto cleanup_fetch_url; 98679971Sobrien } 98779971Sobrien ischunked++; 988223328Sgavin DPRINTF("fetch_url: using chunked encoding\n"); 98979971Sobrien 990142129Smikeh } else if (match_token(&cp, "Proxy-Authenticate:") 991142129Smikeh || match_token(&cp, "WWW-Authenticate:")) { 992142129Smikeh if (! (token = match_token(&cp, "Basic"))) { 993223328Sgavin DPRINTF( 994223328Sgavin "fetch_url: skipping unknown auth scheme `%s'\n", 995142129Smikeh token); 996142129Smikeh continue; 997142129Smikeh } 99879971Sobrien FREEPTR(auth); 999223328Sgavin auth = ftp_strdup(token); 1000223328Sgavin DPRINTF("fetch_url: parsed auth as `%s'\n", cp); 100179971Sobrien } 100279971Sobrien 100379971Sobrien } 100479971Sobrien /* finished parsing header */ 100579971Sobrien 100679971Sobrien switch (hcode) { 100779971Sobrien case 200: 100879971Sobrien break; 100979971Sobrien case 206: 101079971Sobrien if (! restart_point) { 101179971Sobrien warnx("Not expecting partial content header"); 101279971Sobrien goto cleanup_fetch_url; 101379971Sobrien } 101479971Sobrien break; 101579971Sobrien case 300: 101679971Sobrien case 301: 101779971Sobrien case 302: 101879971Sobrien case 303: 101979971Sobrien case 305: 1020223328Sgavin case 307: 102179971Sobrien if (EMPTYSTRING(location)) { 102279971Sobrien warnx( 102379971Sobrien "No redirection Location provided by server"); 102479971Sobrien goto cleanup_fetch_url; 102579971Sobrien } 102679971Sobrien if (redirect_loop++ > 5) { 102779971Sobrien warnx("Too many redirections requested"); 102879971Sobrien goto cleanup_fetch_url; 102979971Sobrien } 103079971Sobrien if (hcode == 305) { 103179971Sobrien if (verbose) 103279971Sobrien fprintf(ttyout, "Redirected via %s\n", 103379971Sobrien location); 103479971Sobrien rval = fetch_url(url, location, 103579971Sobrien proxyauth, wwwauth); 103679971Sobrien } else { 103779971Sobrien if (verbose) 103879971Sobrien fprintf(ttyout, "Redirected to %s\n", 103979971Sobrien location); 104079971Sobrien rval = go_fetch(location); 104179971Sobrien } 104279971Sobrien goto cleanup_fetch_url; 1043142129Smikeh#ifndef NO_AUTH 104479971Sobrien case 401: 104579971Sobrien case 407: 104679971Sobrien { 104779971Sobrien char **authp; 104879971Sobrien char *auser, *apass; 104979971Sobrien 105079971Sobrien if (hcode == 401) { 105179971Sobrien authp = &wwwauth; 1052223328Sgavin auser = uuser; 105379971Sobrien apass = pass; 105479971Sobrien } else { 105579971Sobrien authp = &proxyauth; 105679971Sobrien auser = puser; 105779971Sobrien apass = ppass; 105879971Sobrien } 1059142129Smikeh if (verbose || *authp == NULL || 1060142129Smikeh auser == NULL || apass == NULL) 1061142129Smikeh fprintf(ttyout, "%s\n", message); 1062142129Smikeh if (EMPTYSTRING(auth)) { 1063142129Smikeh warnx( 1064142129Smikeh "No authentication challenge provided by server"); 1065142129Smikeh goto cleanup_fetch_url; 1066142129Smikeh } 106779971Sobrien if (*authp != NULL) { 106879971Sobrien char reply[10]; 106979971Sobrien 107079971Sobrien fprintf(ttyout, 107179971Sobrien "Authorization failed. Retry (y/n)? "); 1072223328Sgavin if (get_line(stdin, reply, sizeof(reply), NULL) 1073223328Sgavin < 0) { 107479971Sobrien goto cleanup_fetch_url; 107579971Sobrien } 1076142129Smikeh if (tolower((unsigned char)reply[0]) != 'y') 1077142129Smikeh goto cleanup_fetch_url; 107879971Sobrien auser = NULL; 107979971Sobrien apass = NULL; 108079971Sobrien } 108179971Sobrien if (auth_url(auth, authp, auser, apass) == 0) { 108279971Sobrien rval = fetch_url(url, proxyenv, 108379971Sobrien proxyauth, wwwauth); 108479971Sobrien memset(*authp, 0, strlen(*authp)); 108579971Sobrien FREEPTR(*authp); 108679971Sobrien } 108779971Sobrien goto cleanup_fetch_url; 108879971Sobrien } 1089142129Smikeh#endif 109079971Sobrien default: 109179971Sobrien if (message) 1092223328Sgavin warnx("Error retrieving file `%s'", message); 109379971Sobrien else 109479971Sobrien warnx("Unknown error retrieving file"); 109579971Sobrien goto cleanup_fetch_url; 109679971Sobrien } 109779971Sobrien } /* end of ftp:// or http:// specific setup */ 109879971Sobrien 109979971Sobrien /* Open the output file. */ 1100274110Sdes 1101274110Sdes /* 1102274110Sdes * Only trust filenames with special meaning if they came from 1103274110Sdes * the command line 1104274110Sdes */ 1105274110Sdes if (outfile == savefile) { 1106274110Sdes if (strcmp(savefile, "-") == 0) { 1107274110Sdes fout = stdout; 1108274110Sdes } else if (*savefile == '|') { 1109274110Sdes oldintp = xsignal(SIGPIPE, SIG_IGN); 1110274110Sdes fout = popen(savefile + 1, "w"); 1111274110Sdes if (fout == NULL) { 1112274110Sdes warn("Can't execute `%s'", savefile + 1); 1113274110Sdes goto cleanup_fetch_url; 1114274110Sdes } 1115274110Sdes closefunc = pclose; 111679971Sobrien } 1117274110Sdes } 1118274110Sdes if (fout == NULL) { 111998247Smikeh if ((rangeend != -1 && rangeend <= restart_point) || 112098247Smikeh (rangestart == -1 && filesize != -1 && filesize <= restart_point)) { 112198247Smikeh /* already done */ 112298247Smikeh if (verbose) 112398247Smikeh fprintf(ttyout, "already done\n"); 112498247Smikeh rval = 0; 112598247Smikeh goto cleanup_fetch_url; 112698247Smikeh } 112798247Smikeh if (restart_point && rangestart != -1) { 112879971Sobrien if (entitylen != -1) 112979971Sobrien filesize = entitylen; 113098247Smikeh if (rangestart != restart_point) { 113179971Sobrien warnx( 113279971Sobrien "Size of `%s' differs from save file `%s'", 113379971Sobrien url, savefile); 113479971Sobrien goto cleanup_fetch_url; 113579971Sobrien } 113679971Sobrien fout = fopen(savefile, "a"); 113779971Sobrien } else 113879971Sobrien fout = fopen(savefile, "w"); 113979971Sobrien if (fout == NULL) { 114079971Sobrien warn("Can't open `%s'", savefile); 114179971Sobrien goto cleanup_fetch_url; 114279971Sobrien } 114379971Sobrien closefunc = fclose; 114479971Sobrien } 114579971Sobrien 114679971Sobrien /* Trap signals */ 114779971Sobrien if (sigsetjmp(httpabort, 1)) 114879971Sobrien goto cleanup_fetch_url; 114979971Sobrien (void)xsignal(SIGQUIT, psummary); 115079971Sobrien oldintr = xsignal(SIGINT, aborthttp); 115179971Sobrien 1152223328Sgavin if ((size_t)rcvbuf_size > bufsize) { 115379971Sobrien if (xferbuf) 115479971Sobrien (void)free(xferbuf); 115579971Sobrien bufsize = rcvbuf_size; 1156223328Sgavin xferbuf = ftp_malloc(bufsize); 115779971Sobrien } 115879971Sobrien 115979971Sobrien bytes = 0; 116079971Sobrien hashbytes = mark; 116179971Sobrien progressmeter(-1); 116279971Sobrien 116379971Sobrien /* Finally, suck down the file. */ 116479971Sobrien do { 116579971Sobrien long chunksize; 1166223328Sgavin short lastchunk; 116779971Sobrien 116879971Sobrien chunksize = 0; 1169223328Sgavin lastchunk = 0; 1170223328Sgavin /* read chunk-size */ 117179971Sobrien if (ischunked) { 117279971Sobrien if (fgets(xferbuf, bufsize, fin) == NULL) { 1173223328Sgavin warnx("Unexpected EOF reading chunk-size"); 117479971Sobrien goto cleanup_fetch_url; 117579971Sobrien } 1176223328Sgavin errno = 0; 117779971Sobrien chunksize = strtol(xferbuf, &ep, 16); 1178223328Sgavin if (ep == xferbuf) { 1179223328Sgavin warnx("Invalid chunk-size"); 1180223328Sgavin goto cleanup_fetch_url; 1181223328Sgavin } 1182223328Sgavin if (errno == ERANGE || chunksize < 0) { 1183223328Sgavin errno = ERANGE; 1184223328Sgavin warn("Chunk-size `%.*s'", 1185223328Sgavin (int)(ep-xferbuf), xferbuf); 1186223328Sgavin goto cleanup_fetch_url; 1187223328Sgavin } 118879971Sobrien 118979971Sobrien /* 119079971Sobrien * XXX: Work around bug in Apache 1.3.9 and 119179971Sobrien * 1.3.11, which incorrectly put trailing 1192223328Sgavin * space after the chunk-size. 119379971Sobrien */ 119479971Sobrien while (*ep == ' ') 119579971Sobrien ep++; 119679971Sobrien 1197223328Sgavin /* skip [ chunk-ext ] */ 1198223328Sgavin if (*ep == ';') { 1199223328Sgavin while (*ep && *ep != '\r') 1200223328Sgavin ep++; 1201223328Sgavin } 1202223328Sgavin 120379971Sobrien if (strcmp(ep, "\r\n") != 0) { 1204223328Sgavin warnx("Unexpected data following chunk-size"); 120579971Sobrien goto cleanup_fetch_url; 120679971Sobrien } 1207223328Sgavin DPRINTF("fetch_url: got chunk-size of " LLF "\n", 1208223328Sgavin (LLT)chunksize); 1209223328Sgavin if (chunksize == 0) { 1210223328Sgavin lastchunk = 1; 1211223328Sgavin goto chunkdone; 1212223328Sgavin } 121379971Sobrien } 121479971Sobrien /* transfer file or chunk */ 121579971Sobrien while (1) { 121679971Sobrien struct timeval then, now, td; 121779971Sobrien off_t bufrem; 121879971Sobrien 121979971Sobrien if (rate_get) 122079971Sobrien (void)gettimeofday(&then, NULL); 1221223328Sgavin bufrem = rate_get ? rate_get : (off_t)bufsize; 122279971Sobrien if (ischunked) 122379971Sobrien bufrem = MIN(chunksize, bufrem); 122479971Sobrien while (bufrem > 0) { 1225223328Sgavin flen = fread(xferbuf, sizeof(char), 1226223328Sgavin MIN((off_t)bufsize, bufrem), fin); 1227223328Sgavin if (flen <= 0) 122879971Sobrien goto chunkdone; 1229223328Sgavin bytes += flen; 1230223328Sgavin bufrem -= flen; 1231223328Sgavin if (fwrite(xferbuf, sizeof(char), flen, fout) 1232223328Sgavin != flen) { 123379971Sobrien warn("Writing `%s'", savefile); 123479971Sobrien goto cleanup_fetch_url; 123579971Sobrien } 123679971Sobrien if (hash && !progress) { 123779971Sobrien while (bytes >= hashbytes) { 123879971Sobrien (void)putc('#', ttyout); 123979971Sobrien hashbytes += mark; 124079971Sobrien } 124179971Sobrien (void)fflush(ttyout); 124279971Sobrien } 124379971Sobrien if (ischunked) { 1244223328Sgavin chunksize -= flen; 124579971Sobrien if (chunksize <= 0) 124679971Sobrien break; 124779971Sobrien } 124879971Sobrien } 124979971Sobrien if (rate_get) { 125079971Sobrien while (1) { 125179971Sobrien (void)gettimeofday(&now, NULL); 125279971Sobrien timersub(&now, &then, &td); 125379971Sobrien if (td.tv_sec > 0) 125479971Sobrien break; 125579971Sobrien usleep(1000000 - td.tv_usec); 125679971Sobrien } 125779971Sobrien } 125879971Sobrien if (ischunked && chunksize <= 0) 125979971Sobrien break; 126079971Sobrien } 126179971Sobrien /* read CRLF after chunk*/ 126279971Sobrien chunkdone: 126379971Sobrien if (ischunked) { 1264223328Sgavin if (fgets(xferbuf, bufsize, fin) == NULL) { 1265223328Sgavin warnx("Unexpected EOF reading chunk CRLF"); 1266223328Sgavin goto cleanup_fetch_url; 1267223328Sgavin } 126879971Sobrien if (strcmp(xferbuf, "\r\n") != 0) { 126979971Sobrien warnx("Unexpected data following chunk"); 127079971Sobrien goto cleanup_fetch_url; 127179971Sobrien } 1272223328Sgavin if (lastchunk) 1273223328Sgavin break; 127479971Sobrien } 127579971Sobrien } while (ischunked); 1276223328Sgavin 1277223328Sgavin/* XXX: deal with optional trailer & CRLF here? */ 1278223328Sgavin 127979971Sobrien if (hash && !progress && bytes > 0) { 128079971Sobrien if (bytes < mark) 128179971Sobrien (void)putc('#', ttyout); 128279971Sobrien (void)putc('\n', ttyout); 128379971Sobrien } 128479971Sobrien if (ferror(fin)) { 128579971Sobrien warn("Reading file"); 128679971Sobrien goto cleanup_fetch_url; 128779971Sobrien } 128879971Sobrien progressmeter(1); 128979971Sobrien (void)fflush(fout); 129079971Sobrien if (closefunc == fclose && mtime != -1) { 129179971Sobrien struct timeval tval[2]; 129279971Sobrien 129379971Sobrien (void)gettimeofday(&tval[0], NULL); 129479971Sobrien tval[1].tv_sec = mtime; 129579971Sobrien tval[1].tv_usec = 0; 129679971Sobrien (*closefunc)(fout); 129779971Sobrien fout = NULL; 129879971Sobrien 129979971Sobrien if (utimes(savefile, tval) == -1) { 130079971Sobrien fprintf(ttyout, 130179971Sobrien "Can't change modification time to %s", 1302223328Sgavin rfc2822time(localtime(&mtime))); 130379971Sobrien } 130479971Sobrien } 130579971Sobrien if (bytes > 0) 130679971Sobrien ptransfer(0); 130798247Smikeh bytes = 0; 130879971Sobrien 130979971Sobrien rval = 0; 131079971Sobrien goto cleanup_fetch_url; 131179971Sobrien 131279971Sobrien improper: 1313223328Sgavin warnx("Improper response from `%s:%s'", host, port); 131479971Sobrien 131579971Sobrien cleanup_fetch_url: 131679971Sobrien if (oldintr) 131779971Sobrien (void)xsignal(SIGINT, oldintr); 131879971Sobrien if (oldintp) 131979971Sobrien (void)xsignal(SIGPIPE, oldintp); 132079971Sobrien if (fin != NULL) 132179971Sobrien fclose(fin); 132279971Sobrien else if (s != -1) 132379971Sobrien close(s); 132479971Sobrien if (closefunc != NULL && fout != NULL) 132579971Sobrien (*closefunc)(fout); 1326146309Smikeh if (res0) 1327146309Smikeh freeaddrinfo(res0); 1328274110Sdes if (savefile != outfile) 1329274110Sdes FREEPTR(savefile); 1330223328Sgavin FREEPTR(uuser); 1331223328Sgavin if (pass != NULL) 1332223328Sgavin memset(pass, 0, strlen(pass)); 133379971Sobrien FREEPTR(pass); 133479971Sobrien FREEPTR(host); 133579971Sobrien FREEPTR(port); 133679971Sobrien FREEPTR(path); 133779971Sobrien FREEPTR(decodedpath); 133879971Sobrien FREEPTR(puser); 1339223328Sgavin if (ppass != NULL) 1340223328Sgavin memset(ppass, 0, strlen(ppass)); 134179971Sobrien FREEPTR(ppass); 134279971Sobrien FREEPTR(auth); 134379971Sobrien FREEPTR(location); 134479971Sobrien FREEPTR(message); 134579971Sobrien return (rval); 134679971Sobrien} 134779971Sobrien 134879971Sobrien/* 134979971Sobrien * Abort a HTTP retrieval 135079971Sobrien */ 135179971Sobrienvoid 135279971Sobrienaborthttp(int notused) 135379971Sobrien{ 135479971Sobrien char msgbuf[100]; 1355223328Sgavin size_t len; 135679971Sobrien 1357142129Smikeh sigint_raised = 1; 135879971Sobrien alarmtimer(0); 135979971Sobrien len = strlcpy(msgbuf, "\nHTTP fetch aborted.\n", sizeof(msgbuf)); 136079971Sobrien write(fileno(ttyout), msgbuf, len); 136179971Sobrien siglongjmp(httpabort, 1); 136279971Sobrien} 136379971Sobrien 136479971Sobrien/* 136579971Sobrien * Retrieve ftp URL or classic ftp argument using FTP. 136679971Sobrien * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection 136779971Sobrien * is still open (e.g, ftp xfer with trailing /) 136879971Sobrien */ 136979971Sobrienstatic int 137079971Sobrienfetch_ftp(const char *url) 137179971Sobrien{ 137279971Sobrien char *cp, *xargv[5], rempath[MAXPATHLEN]; 1373223328Sgavin char *host, *path, *dir, *file, *uuser, *pass; 137479971Sobrien char *port; 1375223328Sgavin char cmdbuf[MAXPATHLEN]; 1376223328Sgavin char dirbuf[4]; 1377223328Sgavin int dirhasglob, filehasglob, rval, transtype, xargc; 1378223328Sgavin int oanonftp, oautologin; 137979971Sobrien in_port_t portnum; 138079971Sobrien url_t urltype; 138179971Sobrien 1382223328Sgavin DPRINTF("fetch_ftp: `%s'\n", url); 1383223328Sgavin host = path = dir = file = uuser = pass = NULL; 138479971Sobrien port = NULL; 138579971Sobrien rval = 1; 1386223328Sgavin transtype = TYPE_I; 138779971Sobrien 1388142129Smikeh if (STRNEQUAL(url, FTP_URL)) { 1389223328Sgavin if ((parse_url(url, "URL", &urltype, &uuser, &pass, 139079971Sobrien &host, &port, &portnum, &path) == -1) || 1391223328Sgavin (uuser != NULL && *uuser == '\0') || 139279971Sobrien EMPTYSTRING(host)) { 139379971Sobrien warnx("Invalid URL `%s'", url); 139479971Sobrien goto cleanup_fetch_ftp; 139579971Sobrien } 139679971Sobrien /* 139779971Sobrien * Note: Don't url_decode(path) here. We need to keep the 139879971Sobrien * distinction between "/" and "%2F" until later. 139979971Sobrien */ 140079971Sobrien 140179971Sobrien /* check for trailing ';type=[aid]' */ 140279971Sobrien if (! EMPTYSTRING(path) && (cp = strrchr(path, ';')) != NULL) { 140379971Sobrien if (strcasecmp(cp, ";type=a") == 0) 1404223328Sgavin transtype = TYPE_A; 140579971Sobrien else if (strcasecmp(cp, ";type=i") == 0) 1406223328Sgavin transtype = TYPE_I; 140779971Sobrien else if (strcasecmp(cp, ";type=d") == 0) { 140879971Sobrien warnx( 140979971Sobrien "Directory listing via a URL is not supported"); 141079971Sobrien goto cleanup_fetch_ftp; 141179971Sobrien } else { 141279971Sobrien warnx("Invalid suffix `%s' in URL `%s'", cp, 141379971Sobrien url); 141479971Sobrien goto cleanup_fetch_ftp; 141579971Sobrien } 141679971Sobrien *cp = 0; 141779971Sobrien } 141879971Sobrien } else { /* classic style `[user@]host:[file]' */ 141979971Sobrien urltype = CLASSIC_URL_T; 1420223328Sgavin host = ftp_strdup(url); 142179971Sobrien cp = strchr(host, '@'); 142279971Sobrien if (cp != NULL) { 142379971Sobrien *cp = '\0'; 1424223328Sgavin uuser = host; 142579971Sobrien anonftp = 0; /* disable anonftp */ 1426223328Sgavin host = ftp_strdup(cp + 1); 142779971Sobrien } 142879971Sobrien cp = strchr(host, ':'); 142979971Sobrien if (cp != NULL) { 143079971Sobrien *cp = '\0'; 1431223328Sgavin path = ftp_strdup(cp + 1); 143279971Sobrien } 143379971Sobrien } 143479971Sobrien if (EMPTYSTRING(host)) 143579971Sobrien goto cleanup_fetch_ftp; 143679971Sobrien 143779971Sobrien /* Extract the file and (if present) directory name. */ 143879971Sobrien dir = path; 143979971Sobrien if (! EMPTYSTRING(dir)) { 144079971Sobrien /* 144179971Sobrien * If we are dealing with classic `[user@]host:[path]' syntax, 144279971Sobrien * then a path of the form `/file' (resulting from input of the 144379971Sobrien * form `host:/file') means that we should do "CWD /" before 144479971Sobrien * retrieving the file. So we set dir="/" and file="file". 144579971Sobrien * 144679971Sobrien * But if we are dealing with URLs like `ftp://host/path' then 144779971Sobrien * a path of the form `/file' (resulting from a URL of the form 144879971Sobrien * `ftp://host//file') means that we should do `CWD ' (with an 144979971Sobrien * empty argument) before retrieving the file. So we set 145079971Sobrien * dir="" and file="file". 145179971Sobrien * 145279971Sobrien * If the path does not contain / at all, we set dir=NULL. 145379971Sobrien * (We get a path without any slashes if we are dealing with 145479971Sobrien * classic `[user@]host:[file]' or URL `ftp://host/file'.) 145579971Sobrien * 145679971Sobrien * In all other cases, we set dir to a string that does not 145779971Sobrien * include the final '/' that separates the dir part from the 145879971Sobrien * file part of the path. (This will be the empty string if 145979971Sobrien * and only if we are dealing with a path of the form `/file' 146079971Sobrien * resulting from an URL of the form `ftp://host//file'.) 146179971Sobrien */ 146279971Sobrien cp = strrchr(dir, '/'); 146379971Sobrien if (cp == dir && urltype == CLASSIC_URL_T) { 146479971Sobrien file = cp + 1; 1465223328Sgavin (void)strlcpy(dirbuf, "/", sizeof(dirbuf)); 1466223328Sgavin dir = dirbuf; 146779971Sobrien } else if (cp != NULL) { 146879971Sobrien *cp++ = '\0'; 146979971Sobrien file = cp; 147079971Sobrien } else { 147179971Sobrien file = dir; 147279971Sobrien dir = NULL; 147379971Sobrien } 147479971Sobrien } else 147579971Sobrien dir = NULL; 147679971Sobrien if (urltype == FTP_URL_T && file != NULL) { 1477146309Smikeh url_decode(file); 147879971Sobrien /* but still don't url_decode(dir) */ 147979971Sobrien } 1480223328Sgavin DPRINTF("fetch_ftp: user `%s' pass `%s' host %s port %s " 1481223328Sgavin "path `%s' dir `%s' file `%s'\n", 1482223328Sgavin STRorNULL(uuser), STRorNULL(pass), 1483223328Sgavin STRorNULL(host), STRorNULL(port), 1484223328Sgavin STRorNULL(path), STRorNULL(dir), STRorNULL(file)); 148579971Sobrien 148679971Sobrien dirhasglob = filehasglob = 0; 148779971Sobrien if (doglob && urltype == CLASSIC_URL_T) { 148879971Sobrien if (! EMPTYSTRING(dir) && strpbrk(dir, "*?[]{}") != NULL) 148979971Sobrien dirhasglob = 1; 149079971Sobrien if (! EMPTYSTRING(file) && strpbrk(file, "*?[]{}") != NULL) 149179971Sobrien filehasglob = 1; 149279971Sobrien } 149379971Sobrien 149479971Sobrien /* Set up the connection */ 1495223328Sgavin oanonftp = anonftp; 149679971Sobrien if (connected) 149779971Sobrien disconnect(0, NULL); 1498223328Sgavin anonftp = oanonftp; 1499223328Sgavin (void)strlcpy(cmdbuf, getprogname(), sizeof(cmdbuf)); 1500223328Sgavin xargv[0] = cmdbuf; 150179971Sobrien xargv[1] = host; 150279971Sobrien xargv[2] = NULL; 150379971Sobrien xargc = 2; 150479971Sobrien if (port) { 150579971Sobrien xargv[2] = port; 150679971Sobrien xargv[3] = NULL; 150779971Sobrien xargc = 3; 150879971Sobrien } 150979971Sobrien oautologin = autologin; 151079971Sobrien /* don't autologin in setpeer(), use ftp_login() below */ 151179971Sobrien autologin = 0; 151279971Sobrien setpeer(xargc, xargv); 151379971Sobrien autologin = oautologin; 151479971Sobrien if ((connected == 0) || 1515223328Sgavin (connected == 1 && !ftp_login(host, uuser, pass))) { 1516223328Sgavin warnx("Can't connect or login to host `%s:%s'", 1517223328Sgavin host, port ? port : "?"); 151879971Sobrien goto cleanup_fetch_ftp; 151979971Sobrien } 152079971Sobrien 1521223328Sgavin switch (transtype) { 152279971Sobrien case TYPE_A: 152379971Sobrien setascii(1, xargv); 152479971Sobrien break; 152579971Sobrien case TYPE_I: 152679971Sobrien setbinary(1, xargv); 152779971Sobrien break; 152879971Sobrien default: 1529223328Sgavin errx(1, "fetch_ftp: unknown transfer type %d", transtype); 153079971Sobrien } 153179971Sobrien 153279971Sobrien /* 153379971Sobrien * Change directories, if necessary. 153479971Sobrien * 153579971Sobrien * Note: don't use EMPTYSTRING(dir) below, because 153679971Sobrien * dir=="" means something different from dir==NULL. 153779971Sobrien */ 153879971Sobrien if (dir != NULL && !dirhasglob) { 153979971Sobrien char *nextpart; 154079971Sobrien 154179971Sobrien /* 154279971Sobrien * If we are dealing with a classic `[user@]host:[path]' 154379971Sobrien * (urltype is CLASSIC_URL_T) then we have a raw directory 154479971Sobrien * name (not encoded in any way) and we can change 154579971Sobrien * directories in one step. 154679971Sobrien * 154779971Sobrien * If we are dealing with an `ftp://host/path' URL 1548223328Sgavin * (urltype is FTP_URL_T), then RFC3986 says we need to 154979971Sobrien * send a separate CWD command for each unescaped "/" 155079971Sobrien * in the path, and we have to interpret %hex escaping 155179971Sobrien * *after* we find the slashes. It's possible to get 155279971Sobrien * empty components here, (from multiple adjacent 1553223328Sgavin * slashes in the path) and RFC3986 says that we should 155479971Sobrien * still do `CWD ' (with a null argument) in such cases. 155579971Sobrien * 155679971Sobrien * Many ftp servers don't support `CWD ', so if there's an 155779971Sobrien * error performing that command, bail out with a descriptive 155879971Sobrien * message. 155979971Sobrien * 156079971Sobrien * Examples: 156179971Sobrien * 156279971Sobrien * host: dir="", urltype=CLASSIC_URL_T 156379971Sobrien * logged in (to default directory) 156479971Sobrien * host:file dir=NULL, urltype=CLASSIC_URL_T 156579971Sobrien * "RETR file" 156679971Sobrien * host:dir/ dir="dir", urltype=CLASSIC_URL_T 156779971Sobrien * "CWD dir", logged in 156879971Sobrien * ftp://host/ dir="", urltype=FTP_URL_T 156979971Sobrien * logged in (to default directory) 157079971Sobrien * ftp://host/dir/ dir="dir", urltype=FTP_URL_T 157179971Sobrien * "CWD dir", logged in 157279971Sobrien * ftp://host/file dir=NULL, urltype=FTP_URL_T 157379971Sobrien * "RETR file" 157479971Sobrien * ftp://host//file dir="", urltype=FTP_URL_T 157579971Sobrien * "CWD ", "RETR file" 157679971Sobrien * host:/file dir="/", urltype=CLASSIC_URL_T 157779971Sobrien * "CWD /", "RETR file" 157879971Sobrien * ftp://host///file dir="/", urltype=FTP_URL_T 157979971Sobrien * "CWD ", "CWD ", "RETR file" 158079971Sobrien * ftp://host/%2F/file dir="%2F", urltype=FTP_URL_T 158179971Sobrien * "CWD /", "RETR file" 158279971Sobrien * ftp://host/foo/file dir="foo", urltype=FTP_URL_T 158379971Sobrien * "CWD foo", "RETR file" 158479971Sobrien * ftp://host/foo/bar/file dir="foo/bar" 158579971Sobrien * "CWD foo", "CWD bar", "RETR file" 158679971Sobrien * ftp://host//foo/bar/file dir="/foo/bar" 158779971Sobrien * "CWD ", "CWD foo", "CWD bar", "RETR file" 158879971Sobrien * ftp://host/foo//bar/file dir="foo//bar" 158979971Sobrien * "CWD foo", "CWD ", "CWD bar", "RETR file" 159079971Sobrien * ftp://host/%2F/foo/bar/file dir="%2F/foo/bar" 159179971Sobrien * "CWD /", "CWD foo", "CWD bar", "RETR file" 159279971Sobrien * ftp://host/%2Ffoo/bar/file dir="%2Ffoo/bar" 159379971Sobrien * "CWD /foo", "CWD bar", "RETR file" 159479971Sobrien * ftp://host/%2Ffoo%2Fbar/file dir="%2Ffoo%2Fbar" 159579971Sobrien * "CWD /foo/bar", "RETR file" 159679971Sobrien * ftp://host/%2Ffoo%2Fbar%2Ffile dir=NULL 159779971Sobrien * "RETR /foo/bar/file" 159879971Sobrien * 159979971Sobrien * Note that we don't need `dir' after this point. 160079971Sobrien */ 160179971Sobrien do { 160279971Sobrien if (urltype == FTP_URL_T) { 160379971Sobrien nextpart = strchr(dir, '/'); 160479971Sobrien if (nextpart) { 160579971Sobrien *nextpart = '\0'; 160679971Sobrien nextpart++; 160779971Sobrien } 160879971Sobrien url_decode(dir); 160979971Sobrien } else 161079971Sobrien nextpart = NULL; 1611223328Sgavin DPRINTF("fetch_ftp: dir `%s', nextpart `%s'\n", 1612223328Sgavin STRorNULL(dir), STRorNULL(nextpart)); 161379971Sobrien if (urltype == FTP_URL_T || *dir != '\0') { 1614223328Sgavin (void)strlcpy(cmdbuf, "cd", sizeof(cmdbuf)); 1615223328Sgavin xargv[0] = cmdbuf; 161679971Sobrien xargv[1] = dir; 161779971Sobrien xargv[2] = NULL; 161879971Sobrien dirchange = 0; 161979971Sobrien cd(2, xargv); 162079971Sobrien if (! dirchange) { 162179971Sobrien if (*dir == '\0' && code == 500) 162279971Sobrien fprintf(stderr, 162379971Sobrien"\n" 162479971Sobrien"ftp: The `CWD ' command (without a directory), which is required by\n" 1625223328Sgavin" RFC3986 to support the empty directory in the URL pathname (`//'),\n" 1626223328Sgavin" conflicts with the server's conformance to RFC0959.\n" 162779971Sobrien" Try the same URL without the `//' in the URL pathname.\n" 162879971Sobrien"\n"); 162979971Sobrien goto cleanup_fetch_ftp; 163079971Sobrien } 163179971Sobrien } 163279971Sobrien dir = nextpart; 163379971Sobrien } while (dir != NULL); 163479971Sobrien } 163579971Sobrien 163679971Sobrien if (EMPTYSTRING(file)) { 163779971Sobrien rval = -1; 163879971Sobrien goto cleanup_fetch_ftp; 163979971Sobrien } 164079971Sobrien 164179971Sobrien if (dirhasglob) { 164279971Sobrien (void)strlcpy(rempath, dir, sizeof(rempath)); 164379971Sobrien (void)strlcat(rempath, "/", sizeof(rempath)); 164479971Sobrien (void)strlcat(rempath, file, sizeof(rempath)); 164579971Sobrien file = rempath; 164679971Sobrien } 164779971Sobrien 164879971Sobrien /* Fetch the file(s). */ 164979971Sobrien xargc = 2; 1650223328Sgavin (void)strlcpy(cmdbuf, "get", sizeof(cmdbuf)); 1651223328Sgavin xargv[0] = cmdbuf; 165279971Sobrien xargv[1] = file; 165379971Sobrien xargv[2] = NULL; 165479971Sobrien if (dirhasglob || filehasglob) { 165579971Sobrien int ointeractive; 165679971Sobrien 165779971Sobrien ointeractive = interactive; 165879971Sobrien interactive = 0; 1659142129Smikeh if (restartautofetch) 1660223328Sgavin (void)strlcpy(cmdbuf, "mreget", sizeof(cmdbuf)); 1661142129Smikeh else 1662223328Sgavin (void)strlcpy(cmdbuf, "mget", sizeof(cmdbuf)); 1663223328Sgavin xargv[0] = cmdbuf; 166479971Sobrien mget(xargc, xargv); 166579971Sobrien interactive = ointeractive; 166679971Sobrien } else { 166779971Sobrien if (outfile == NULL) { 166879971Sobrien cp = strrchr(file, '/'); /* find savefile */ 166979971Sobrien if (cp != NULL) 167079971Sobrien outfile = cp + 1; 167179971Sobrien else 167279971Sobrien outfile = file; 167379971Sobrien } 167479971Sobrien xargv[2] = (char *)outfile; 167579971Sobrien xargv[3] = NULL; 167679971Sobrien xargc++; 167779971Sobrien if (restartautofetch) 167879971Sobrien reget(xargc, xargv); 167979971Sobrien else 168079971Sobrien get(xargc, xargv); 168179971Sobrien } 168279971Sobrien 168379971Sobrien if ((code / 100) == COMPLETE) 168479971Sobrien rval = 0; 168579971Sobrien 168679971Sobrien cleanup_fetch_ftp: 1687223328Sgavin FREEPTR(port); 168879971Sobrien FREEPTR(host); 168979971Sobrien FREEPTR(path); 1690223328Sgavin FREEPTR(uuser); 1691223328Sgavin if (pass) 1692223328Sgavin memset(pass, 0, strlen(pass)); 169379971Sobrien FREEPTR(pass); 169479971Sobrien return (rval); 169579971Sobrien} 169679971Sobrien 169779971Sobrien/* 169879971Sobrien * Retrieve the given file to outfile. 169979971Sobrien * Supports arguments of the form: 170079971Sobrien * "host:path", "ftp://host/path" if $ftpproxy, call fetch_url() else 170179971Sobrien * call fetch_ftp() 170279971Sobrien * "http://host/path" call fetch_url() to use HTTP 170379971Sobrien * "file:///path" call fetch_url() to copy 170479971Sobrien * "about:..." print a message 170579971Sobrien * 170679971Sobrien * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection 170779971Sobrien * is still open (e.g, ftp xfer with trailing /) 170879971Sobrien */ 170979971Sobrienstatic int 171079971Sobriengo_fetch(const char *url) 171179971Sobrien{ 1712223328Sgavin char *proxyenv; 171379971Sobrien 1714142129Smikeh#ifndef NO_ABOUT 171579971Sobrien /* 171679971Sobrien * Check for about:* 171779971Sobrien */ 1718142129Smikeh if (STRNEQUAL(url, ABOUT_URL)) { 171979971Sobrien url += sizeof(ABOUT_URL) -1; 1720121966Smikeh if (strcasecmp(url, "ftp") == 0 || 1721121966Smikeh strcasecmp(url, "tnftp") == 0) { 172279971Sobrien fputs( 1723121966Smikeh"This version of ftp has been enhanced by Luke Mewburn <lukem@NetBSD.org>\n" 172479971Sobrien"for the NetBSD project. Execute `man ftp' for more details.\n", ttyout); 172579971Sobrien } else if (strcasecmp(url, "lukem") == 0) { 172679971Sobrien fputs( 172779971Sobrien"Luke Mewburn is the author of most of the enhancements in this ftp client.\n" 1728121966Smikeh"Please email feedback to <lukem@NetBSD.org>.\n", ttyout); 172979971Sobrien } else if (strcasecmp(url, "netbsd") == 0) { 173079971Sobrien fputs( 173179971Sobrien"NetBSD is a freely available and redistributable UNIX-like operating system.\n" 1732121966Smikeh"For more information, see http://www.NetBSD.org/\n", ttyout); 173379971Sobrien } else if (strcasecmp(url, "version") == 0) { 173479971Sobrien fprintf(ttyout, "Version: %s %s%s\n", 173579971Sobrien FTP_PRODUCT, FTP_VERSION, 173679971Sobrien#ifdef INET6 173779971Sobrien "" 173879971Sobrien#else 173979971Sobrien " (-IPv6)" 174079971Sobrien#endif 174179971Sobrien ); 174279971Sobrien } else { 174379971Sobrien fprintf(ttyout, "`%s' is an interesting topic.\n", url); 174479971Sobrien } 174579971Sobrien fputs("\n", ttyout); 174679971Sobrien return (0); 174779971Sobrien } 1748142129Smikeh#endif 174979971Sobrien 175079971Sobrien /* 175179971Sobrien * Check for file:// and http:// URLs. 175279971Sobrien */ 1753142129Smikeh if (STRNEQUAL(url, HTTP_URL) || STRNEQUAL(url, FILE_URL)) 175479971Sobrien return (fetch_url(url, NULL, NULL, NULL)); 175579971Sobrien 175679971Sobrien /* 175779971Sobrien * Try FTP URL-style and host:file arguments next. 175879971Sobrien * If ftpproxy is set with an FTP URL, use fetch_url() 175979971Sobrien * Othewise, use fetch_ftp(). 176079971Sobrien */ 1761223328Sgavin proxyenv = getoptionvalue("ftp_proxy"); 1762223328Sgavin if (!EMPTYSTRING(proxyenv) && STRNEQUAL(url, FTP_URL)) 176379971Sobrien return (fetch_url(url, NULL, NULL, NULL)); 176479971Sobrien 176579971Sobrien return (fetch_ftp(url)); 176679971Sobrien} 176779971Sobrien 176879971Sobrien/* 176979971Sobrien * Retrieve multiple files from the command line, 177079971Sobrien * calling go_fetch() for each file. 177179971Sobrien * 177279971Sobrien * If an ftp path has a trailing "/", the path will be cd-ed into and 177379971Sobrien * the connection remains open, and the function will return -1 177479971Sobrien * (to indicate the connection is alive). 177579971Sobrien * If an error occurs the return value will be the offset+1 in 177679971Sobrien * argv[] of the file that caused a problem (i.e, argv[x] 177779971Sobrien * returns x+1) 177879971Sobrien * Otherwise, 0 is returned if all files retrieved successfully. 177979971Sobrien */ 178079971Sobrienint 178179971Sobrienauto_fetch(int argc, char *argv[]) 178279971Sobrien{ 1783223328Sgavin volatile int argpos, rval; 178479971Sobrien 1785223328Sgavin argpos = rval = 0; 178679971Sobrien 178779971Sobrien if (sigsetjmp(toplevel, 1)) { 178879971Sobrien if (connected) 178979971Sobrien disconnect(0, NULL); 1790142129Smikeh if (rval > 0) 1791142129Smikeh rval = argpos + 1; 1792142129Smikeh return (rval); 179379971Sobrien } 179479971Sobrien (void)xsignal(SIGINT, intr); 179579971Sobrien (void)xsignal(SIGPIPE, lostpeer); 179679971Sobrien 179779971Sobrien /* 179879971Sobrien * Loop through as long as there's files to fetch. 179979971Sobrien */ 1800223328Sgavin for (; (rval == 0) && (argpos < argc); argpos++) { 180179971Sobrien if (strchr(argv[argpos], ':') == NULL) 180279971Sobrien break; 180379971Sobrien redirect_loop = 0; 180479971Sobrien if (!anonftp) 180579971Sobrien anonftp = 2; /* Handle "automatic" transfers. */ 180679971Sobrien rval = go_fetch(argv[argpos]); 180779971Sobrien if (outfile != NULL && strcmp(outfile, "-") != 0 180879971Sobrien && outfile[0] != '|') 180979971Sobrien outfile = NULL; 181079971Sobrien if (rval > 0) 181179971Sobrien rval = argpos + 1; 181279971Sobrien } 181379971Sobrien 181479971Sobrien if (connected && rval != -1) 181579971Sobrien disconnect(0, NULL); 181679971Sobrien return (rval); 181779971Sobrien} 181879971Sobrien 181979971Sobrien 1820223328Sgavin/* 1821223328Sgavin * Upload multiple files from the command line. 1822223328Sgavin * 1823223328Sgavin * If an error occurs the return value will be the offset+1 in 1824223328Sgavin * argv[] of the file that caused a problem (i.e, argv[x] 1825223328Sgavin * returns x+1) 1826223328Sgavin * Otherwise, 0 is returned if all files uploaded successfully. 1827223328Sgavin */ 182879971Sobrienint 182979971Sobrienauto_put(int argc, char **argv, const char *uploadserver) 183079971Sobrien{ 183179971Sobrien char *uargv[4], *path, *pathsep; 1832223328Sgavin int uargc, rval, argpos; 1833223328Sgavin size_t len; 1834223328Sgavin char cmdbuf[MAX_C_NAME]; 183579971Sobrien 1836223328Sgavin (void)strlcpy(cmdbuf, "mput", sizeof(cmdbuf)); 1837223328Sgavin uargv[0] = cmdbuf; 1838223328Sgavin uargv[1] = argv[0]; 1839223328Sgavin uargc = 2; 184079971Sobrien uargv[2] = uargv[3] = NULL; 184179971Sobrien pathsep = NULL; 184279971Sobrien rval = 1; 184379971Sobrien 1844223328Sgavin DPRINTF("auto_put: target `%s'\n", uploadserver); 184579971Sobrien 1846223328Sgavin path = ftp_strdup(uploadserver); 184779971Sobrien len = strlen(path); 184879971Sobrien if (path[len - 1] != '/' && path[len - 1] != ':') { 184979971Sobrien /* 185079971Sobrien * make sure we always pass a directory to auto_fetch 185179971Sobrien */ 185279971Sobrien if (argc > 1) { /* more than one file to upload */ 185379971Sobrien len = strlen(uploadserver) + 2; /* path + "/" + "\0" */ 185479971Sobrien free(path); 1855223328Sgavin path = (char *)ftp_malloc(len); 185679971Sobrien (void)strlcpy(path, uploadserver, len); 185779971Sobrien (void)strlcat(path, "/", len); 185879971Sobrien } else { /* single file to upload */ 1859223328Sgavin (void)strlcpy(cmdbuf, "put", sizeof(cmdbuf)); 1860223328Sgavin uargv[0] = cmdbuf; 186179971Sobrien pathsep = strrchr(path, '/'); 186279971Sobrien if (pathsep == NULL) { 186379971Sobrien pathsep = strrchr(path, ':'); 186479971Sobrien if (pathsep == NULL) { 186579971Sobrien warnx("Invalid URL `%s'", path); 186679971Sobrien goto cleanup_auto_put; 186779971Sobrien } 186879971Sobrien pathsep++; 1869223328Sgavin uargv[2] = ftp_strdup(pathsep); 187079971Sobrien pathsep[0] = '/'; 1871146309Smikeh } else 1872223328Sgavin uargv[2] = ftp_strdup(pathsep + 1); 187379971Sobrien pathsep[1] = '\0'; 187479971Sobrien uargc++; 187579971Sobrien } 187679971Sobrien } 1877223328Sgavin DPRINTF("auto_put: URL `%s' argv[2] `%s'\n", 1878223328Sgavin path, STRorNULL(uargv[2])); 1879146309Smikeh 1880146309Smikeh /* connect and cwd */ 188179971Sobrien rval = auto_fetch(1, &path); 188279971Sobrien if(rval >= 0) 188379971Sobrien goto cleanup_auto_put; 188479971Sobrien 1885223328Sgavin rval = 0; 1886223328Sgavin 1887223328Sgavin /* target filename provided; upload 1 file */ 188879971Sobrien /* XXX : is this the best way? */ 188979971Sobrien if (uargc == 3) { 189079971Sobrien uargv[1] = argv[0]; 189179971Sobrien put(uargc, uargv); 1892223328Sgavin if ((code / 100) != COMPLETE) 1893223328Sgavin rval = 1; 1894223328Sgavin } else { /* otherwise a target dir: upload all files to it */ 1895223328Sgavin for(argpos = 0; argv[argpos] != NULL; argpos++) { 1896223328Sgavin uargv[1] = argv[argpos]; 1897223328Sgavin mput(uargc, uargv); 1898223328Sgavin if ((code / 100) != COMPLETE) { 1899223328Sgavin rval = argpos + 1; 1900223328Sgavin break; 1901223328Sgavin } 1902223328Sgavin } 190379971Sobrien } 190479971Sobrien 190579971Sobrien cleanup_auto_put: 1906223328Sgavin free(path); 190779971Sobrien FREEPTR(uargv[2]); 190879971Sobrien return (rval); 190979971Sobrien} 1910