Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 1 | /* vi: set sw=4 ts=4: */ |
| 2 | /* |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 3 | * wget - retrieve a file using HTTP or FTP |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 4 | * |
Eric Andersen | 4e573f4 | 2000-11-14 23:29:24 +0000 | [diff] [blame] | 5 | * Chip Rosenthal Covad Communications <chip@laserlink.net> |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 6 | * |
Denis Vlasenko | db12d1d | 2008-12-07 00:52:58 +0000 | [diff] [blame] | 7 | * Licensed under GPLv2, see file LICENSE in this tarball for details. |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 8 | */ |
Denis Vlasenko | b6adbf1 | 2007-05-26 19:00:18 +0000 | [diff] [blame] | 9 | #include "libbb.h" |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 10 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 11 | struct host_info { |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 12 | // May be used if we ever will want to free() all xstrdup()s... |
| 13 | /* char *allocated; */ |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 14 | const char *path; |
| 15 | const char *user; |
| 16 | char *host; |
| 17 | int port; |
| 18 | smallint is_ftp; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 19 | }; |
| 20 | |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 21 | |
| 22 | /* Globals (can be accessed from signal handlers) */ |
| 23 | struct globals { |
| 24 | off_t content_len; /* Content-length of the file */ |
| 25 | off_t beg_range; /* Range at which continue begins */ |
| 26 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 27 | off_t transferred; /* Number of bytes transferred so far */ |
| 28 | const char *curfile; /* Name of current file being transferred */ |
Magnus Damm | f591499 | 2009-11-08 16:34:43 +0100 | [diff] [blame] | 29 | bb_progress_t pmt; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 30 | #endif |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 31 | smallint chunked; /* chunked transfer encoding */ |
| 32 | smallint got_clen; /* got content-length: from server */ |
Denys Vlasenko | 98a4c7c | 2010-02-04 15:00:15 +0100 | [diff] [blame] | 33 | } FIX_ALIASING; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 34 | #define G (*(struct globals*)&bb_common_bufsiz1) |
| 35 | struct BUG_G_too_big { |
Denis Vlasenko | 6b40443 | 2008-01-07 16:13:14 +0000 | [diff] [blame] | 36 | char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1]; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 37 | }; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 38 | #define INIT_G() do { } while (0) |
| 39 | |
| 40 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 41 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 42 | |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 43 | static void progress_meter(int flag) |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 44 | { |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 45 | /* We can be called from signal handler */ |
| 46 | int save_errno = errno; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 47 | |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 48 | if (flag == -1) { /* first call to progress_meter */ |
Magnus Damm | f591499 | 2009-11-08 16:34:43 +0100 | [diff] [blame] | 49 | bb_progress_init(&G.pmt); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 50 | } |
| 51 | |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 52 | bb_progress_update(&G.pmt, G.curfile, G.beg_range, G.transferred, |
Denys Vlasenko | c5bbd5d | 2010-07-12 03:27:09 +0200 | [diff] [blame] | 53 | G.chunked ? 0 : G.beg_range + G.transferred + G.content_len); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 54 | |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 55 | if (flag == 0) { |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 56 | /* last call to progress_meter */ |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 57 | alarm(0); |
Denys Vlasenko | 19ced5c | 2010-06-06 21:53:09 +0200 | [diff] [blame] | 58 | bb_putchar_stderr('\n'); |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 59 | G.transferred = 0; |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 60 | } else { |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 61 | if (flag == -1) { /* first call to progress_meter */ |
| 62 | signal_SA_RESTART_empty_mask(SIGALRM, progress_meter); |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 63 | } |
| 64 | alarm(1); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 65 | } |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 66 | |
| 67 | errno = save_errno; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 68 | } |
Magnus Damm | f591499 | 2009-11-08 16:34:43 +0100 | [diff] [blame] | 69 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 70 | #else /* FEATURE_WGET_STATUSBAR */ |
| 71 | |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 72 | static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { } |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 73 | |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 74 | #endif |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 75 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 76 | |
Denys Vlasenko | 7d5ddf1 | 2009-06-30 20:36:27 +0200 | [diff] [blame] | 77 | /* IPv6 knows scoped address types i.e. link and site local addresses. Link |
| 78 | * local addresses can have a scope identifier to specify the |
| 79 | * interface/link an address is valid on (e.g. fe80::1%eth0). This scope |
| 80 | * identifier is only valid on a single node. |
| 81 | * |
| 82 | * RFC 4007 says that the scope identifier MUST NOT be sent across the wire, |
| 83 | * unless all nodes agree on the semantic. Apache e.g. regards zone identifiers |
| 84 | * in the Host header as invalid requests, see |
| 85 | * https://issues.apache.org/bugzilla/show_bug.cgi?id=35122 |
| 86 | */ |
| 87 | static void strip_ipv6_scope_id(char *host) |
| 88 | { |
| 89 | char *scope, *cp; |
| 90 | |
| 91 | /* bbox wget actually handles IPv6 addresses without [], like |
| 92 | * wget "http://::1/xxx", but this is not standard. |
| 93 | * To save code, _here_ we do not support it. */ |
| 94 | |
| 95 | if (host[0] != '[') |
| 96 | return; /* not IPv6 */ |
| 97 | |
| 98 | scope = strchr(host, '%'); |
| 99 | if (!scope) |
| 100 | return; |
| 101 | |
| 102 | /* Remove the IPv6 zone identifier from the host address */ |
| 103 | cp = strchr(host, ']'); |
| 104 | if (!cp || (cp[1] != ':' && cp[1] != '\0')) { |
| 105 | /* malformed address (not "[xx]:nn" or "[xx]") */ |
| 106 | return; |
| 107 | } |
| 108 | |
| 109 | /* cp points to "]...", scope points to "%eth0]..." */ |
| 110 | overlapping_strcpy(scope, cp); |
| 111 | } |
| 112 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 113 | /* Read NMEMB bytes into PTR from STREAM. Returns the number of bytes read, |
| 114 | * and a short count if an eof or non-interrupt error is encountered. */ |
| 115 | static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream) |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 116 | { |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 117 | size_t ret; |
| 118 | char *p = (char*)ptr; |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 119 | |
| 120 | do { |
| 121 | clearerr(stream); |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 122 | errno = 0; |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 123 | ret = fread(p, 1, nmemb, stream); |
| 124 | p += ret; |
| 125 | nmemb -= ret; |
| 126 | } while (nmemb && ferror(stream) && errno == EINTR); |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 127 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 128 | return p - (char*)ptr; |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 129 | } |
| 130 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 131 | /* Read a line or SIZE-1 bytes into S, whichever is less, from STREAM. |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 132 | * Returns S, or NULL if an eof or non-interrupt error is encountered. */ |
| 133 | static char *safe_fgets(char *s, int size, FILE *stream) |
| 134 | { |
| 135 | char *ret; |
| 136 | |
| 137 | do { |
| 138 | clearerr(stream); |
Denis Vlasenko | 00d8417 | 2008-11-24 07:34:42 +0000 | [diff] [blame] | 139 | errno = 0; |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 140 | ret = fgets(s, size, stream); |
| 141 | } while (ret == NULL && ferror(stream) && errno == EINTR); |
| 142 | |
| 143 | return ret; |
| 144 | } |
| 145 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 146 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 147 | /* Base64-encode character string. buf is assumed to be char buf[512]. */ |
| 148 | static char *base64enc_512(char buf[512], const char *str) |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 149 | { |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 150 | unsigned len = strlen(str); |
| 151 | if (len > 512/4*3 - 10) /* paranoia */ |
| 152 | len = 512/4*3 - 10; |
| 153 | bb_uuencode(buf, str, len, bb_uuenc_tbl_base64); |
Rob Landley | 76ef08c | 2006-06-13 16:44:26 +0000 | [diff] [blame] | 154 | return buf; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 155 | } |
| 156 | #endif |
| 157 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 158 | static char* sanitize_string(char *s) |
| 159 | { |
| 160 | unsigned char *p = (void *) s; |
| 161 | while (*p >= ' ') |
| 162 | p++; |
| 163 | *p = '\0'; |
| 164 | return s; |
| 165 | } |
| 166 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 167 | static FILE *open_socket(len_and_sockaddr *lsa) |
| 168 | { |
| 169 | FILE *fp; |
| 170 | |
| 171 | /* glibc 2.4 seems to try seeking on it - ??! */ |
| 172 | /* hopefully it understands what ESPIPE means... */ |
| 173 | fp = fdopen(xconnect_stream(lsa), "r+"); |
| 174 | if (fp == NULL) |
| 175 | bb_perror_msg_and_die("fdopen"); |
| 176 | |
| 177 | return fp; |
| 178 | } |
| 179 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 180 | static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf) |
| 181 | { |
| 182 | int result; |
| 183 | if (s1) { |
| 184 | if (!s2) s2 = ""; |
| 185 | fprintf(fp, "%s%s\r\n", s1, s2); |
| 186 | fflush(fp); |
| 187 | } |
| 188 | |
| 189 | do { |
| 190 | char *buf_ptr; |
| 191 | |
| 192 | if (fgets(buf, 510, fp) == NULL) { |
| 193 | bb_perror_msg_and_die("error getting response"); |
| 194 | } |
| 195 | buf_ptr = strstr(buf, "\r\n"); |
| 196 | if (buf_ptr) { |
| 197 | *buf_ptr = '\0'; |
| 198 | } |
| 199 | } while (!isdigit(buf[0]) || buf[3] != ' '); |
| 200 | |
| 201 | buf[3] = '\0'; |
| 202 | result = xatoi_u(buf); |
| 203 | buf[3] = ' '; |
| 204 | return result; |
| 205 | } |
| 206 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 207 | static void parse_url(char *src_url, struct host_info *h) |
| 208 | { |
| 209 | char *url, *p, *sp; |
| 210 | |
| 211 | /* h->allocated = */ url = xstrdup(src_url); |
| 212 | |
| 213 | if (strncmp(url, "http://", 7) == 0) { |
| 214 | h->port = bb_lookup_port("http", "tcp", 80); |
| 215 | h->host = url + 7; |
| 216 | h->is_ftp = 0; |
| 217 | } else if (strncmp(url, "ftp://", 6) == 0) { |
| 218 | h->port = bb_lookup_port("ftp", "tcp", 21); |
| 219 | h->host = url + 6; |
| 220 | h->is_ftp = 1; |
| 221 | } else |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 222 | bb_error_msg_and_die("not an http or ftp url: %s", sanitize_string(url)); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 223 | |
| 224 | // FYI: |
| 225 | // "Real" wget 'http://busybox.net?var=a/b' sends this request: |
| 226 | // 'GET /?var=a/b HTTP 1.0' |
| 227 | // and saves 'index.html?var=a%2Fb' (we save 'b') |
| 228 | // wget 'http://busybox.net?login=john@doe': |
| 229 | // request: 'GET /?login=john@doe HTTP/1.0' |
| 230 | // saves: 'index.html?login=john@doe' (we save '?login=john@doe') |
| 231 | // wget 'http://busybox.net#test/test': |
| 232 | // request: 'GET / HTTP/1.0' |
| 233 | // saves: 'index.html' (we save 'test') |
| 234 | // |
| 235 | // We also don't add unique .N suffix if file exists... |
| 236 | sp = strchr(h->host, '/'); |
| 237 | p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p; |
| 238 | p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p; |
| 239 | if (!sp) { |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 240 | h->path = ""; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 241 | } else if (*sp == '/') { |
| 242 | *sp = '\0'; |
| 243 | h->path = sp + 1; |
| 244 | } else { // '#' or '?' |
| 245 | // http://busybox.net?login=john@doe is a valid URL |
| 246 | // memmove converts to: |
| 247 | // http:/busybox.nett?login=john@doe... |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 248 | memmove(h->host - 1, h->host, sp - h->host); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 249 | h->host--; |
| 250 | sp[-1] = '\0'; |
| 251 | h->path = sp; |
| 252 | } |
| 253 | |
Vladimir Dronnikov | be168b1 | 2009-10-05 02:18:01 +0200 | [diff] [blame] | 254 | // We used to set h->user to NULL here, but this interferes |
| 255 | // with handling of code 302 ("object was moved") |
| 256 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 257 | sp = strrchr(h->host, '@'); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 258 | if (sp != NULL) { |
| 259 | h->user = h->host; |
| 260 | *sp = '\0'; |
| 261 | h->host = sp + 1; |
| 262 | } |
| 263 | |
| 264 | sp = h->host; |
| 265 | } |
| 266 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 267 | static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/) |
| 268 | { |
| 269 | char *s, *hdrval; |
| 270 | int c; |
| 271 | |
| 272 | /* *istrunc = 0; */ |
| 273 | |
| 274 | /* retrieve header line */ |
| 275 | if (fgets(buf, bufsiz, fp) == NULL) |
| 276 | return NULL; |
| 277 | |
| 278 | /* see if we are at the end of the headers */ |
| 279 | for (s = buf; *s == '\r'; ++s) |
| 280 | continue; |
| 281 | if (*s == '\n') |
| 282 | return NULL; |
| 283 | |
| 284 | /* convert the header name to lower case */ |
Denys Vlasenko | 4836331 | 2010-04-04 15:29:32 +0200 | [diff] [blame] | 285 | for (s = buf; isalnum(*s) || *s == '-' || *s == '.'; ++s) { |
| 286 | /* tolower for "A-Z", no-op for "0-9a-z-." */ |
| 287 | *s = (*s | 0x20); |
| 288 | } |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 289 | |
| 290 | /* verify we are at the end of the header name */ |
| 291 | if (*s != ':') |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 292 | bb_error_msg_and_die("bad header line: %s", sanitize_string(buf)); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 293 | |
| 294 | /* locate the start of the header value */ |
| 295 | *s++ = '\0'; |
| 296 | hdrval = skip_whitespace(s); |
| 297 | |
| 298 | /* locate the end of header */ |
| 299 | while (*s && *s != '\r' && *s != '\n') |
| 300 | ++s; |
| 301 | |
| 302 | /* end of header found */ |
| 303 | if (*s) { |
| 304 | *s = '\0'; |
| 305 | return hdrval; |
| 306 | } |
| 307 | |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 308 | /* Rats! The buffer isn't big enough to hold the entire header value */ |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 309 | while (c = getc(fp), c != EOF && c != '\n') |
| 310 | continue; |
| 311 | /* *istrunc = 1; */ |
| 312 | return hdrval; |
| 313 | } |
| 314 | |
Denis Vlasenko | 5a2ad69 | 2009-03-04 14:13:37 +0000 | [diff] [blame] | 315 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
| 316 | static char *URL_escape(const char *str) |
| 317 | { |
| 318 | /* URL encode, see RFC 2396 */ |
| 319 | char *dst; |
| 320 | char *res = dst = xmalloc(strlen(str) * 3 + 1); |
| 321 | unsigned char c; |
| 322 | |
| 323 | while (1) { |
| 324 | c = *str++; |
| 325 | if (c == '\0' |
| 326 | /* || strchr("!&'()*-.=_~", c) - more code */ |
| 327 | || c == '!' |
| 328 | || c == '&' |
| 329 | || c == '\'' |
| 330 | || c == '(' |
| 331 | || c == ')' |
| 332 | || c == '*' |
| 333 | || c == '-' |
| 334 | || c == '.' |
| 335 | || c == '=' |
| 336 | || c == '_' |
| 337 | || c == '~' |
| 338 | || (c >= '0' && c <= '9') |
| 339 | || ((c|0x20) >= 'a' && (c|0x20) <= 'z') |
| 340 | ) { |
| 341 | *dst++ = c; |
| 342 | if (c == '\0') |
| 343 | return res; |
| 344 | } else { |
| 345 | *dst++ = '%'; |
| 346 | *dst++ = bb_hexdigits_upcase[c >> 4]; |
| 347 | *dst++ = bb_hexdigits_upcase[c & 0xf]; |
| 348 | } |
| 349 | } |
| 350 | } |
| 351 | #endif |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 352 | |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 353 | static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa) |
| 354 | { |
| 355 | char buf[512]; |
| 356 | FILE *sfp; |
| 357 | char *str; |
| 358 | int port; |
| 359 | |
| 360 | if (!target->user) |
| 361 | target->user = xstrdup("anonymous:busybox@"); |
| 362 | |
| 363 | sfp = open_socket(lsa); |
| 364 | if (ftpcmd(NULL, NULL, sfp, buf) != 220) |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 365 | bb_error_msg_and_die("%s", sanitize_string(buf+4)); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 366 | |
| 367 | /* |
| 368 | * Splitting username:password pair, |
| 369 | * trying to log in |
| 370 | */ |
| 371 | str = strchr(target->user, ':'); |
| 372 | if (str) |
| 373 | *str++ = '\0'; |
| 374 | switch (ftpcmd("USER ", target->user, sfp, buf)) { |
| 375 | case 230: |
| 376 | break; |
| 377 | case 331: |
| 378 | if (ftpcmd("PASS ", str, sfp, buf) == 230) |
| 379 | break; |
| 380 | /* fall through (failed login) */ |
| 381 | default: |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 382 | bb_error_msg_and_die("ftp login: %s", sanitize_string(buf+4)); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 383 | } |
| 384 | |
| 385 | ftpcmd("TYPE I", NULL, sfp, buf); |
| 386 | |
| 387 | /* |
| 388 | * Querying file size |
| 389 | */ |
| 390 | if (ftpcmd("SIZE ", target->path, sfp, buf) == 213) { |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 391 | G.content_len = BB_STRTOOFF(buf+4, NULL, 10); |
| 392 | if (G.content_len < 0 || errno) { |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 393 | bb_error_msg_and_die("SIZE value is garbage"); |
| 394 | } |
| 395 | G.got_clen = 1; |
| 396 | } |
| 397 | |
| 398 | /* |
| 399 | * Entering passive mode |
| 400 | */ |
| 401 | if (ftpcmd("PASV", NULL, sfp, buf) != 227) { |
| 402 | pasv_error: |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 403 | bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(buf)); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 404 | } |
| 405 | // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage] |
| 406 | // Server's IP is N1.N2.N3.N4 (we ignore it) |
| 407 | // Server's port for data connection is P1*256+P2 |
| 408 | str = strrchr(buf, ')'); |
| 409 | if (str) str[0] = '\0'; |
| 410 | str = strrchr(buf, ','); |
| 411 | if (!str) goto pasv_error; |
| 412 | port = xatou_range(str+1, 0, 255); |
| 413 | *str = '\0'; |
| 414 | str = strrchr(buf, ','); |
| 415 | if (!str) goto pasv_error; |
| 416 | port += xatou_range(str+1, 0, 255) * 256; |
| 417 | set_nport(lsa, htons(port)); |
| 418 | |
| 419 | *dfpp = open_socket(lsa); |
| 420 | |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 421 | if (G.beg_range) { |
| 422 | sprintf(buf, "REST %"OFF_FMT"u", G.beg_range); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 423 | if (ftpcmd(buf, NULL, sfp, buf) == 350) |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 424 | G.content_len -= G.beg_range; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 425 | } |
| 426 | |
| 427 | if (ftpcmd("RETR ", target->path, sfp, buf) > 150) |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 428 | bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(buf)); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 429 | |
| 430 | return sfp; |
| 431 | } |
| 432 | |
| 433 | /* Must match option string! */ |
| 434 | enum { |
| 435 | WGET_OPT_CONTINUE = (1 << 0), |
| 436 | WGET_OPT_SPIDER = (1 << 1), |
| 437 | WGET_OPT_QUIET = (1 << 2), |
| 438 | WGET_OPT_OUTNAME = (1 << 3), |
| 439 | WGET_OPT_PREFIX = (1 << 4), |
| 440 | WGET_OPT_PROXY = (1 << 5), |
| 441 | WGET_OPT_USER_AGENT = (1 << 6), |
| 442 | WGET_OPT_RETRIES = (1 << 7), |
| 443 | WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8), |
| 444 | WGET_OPT_PASSIVE = (1 << 9), |
| 445 | WGET_OPT_HEADER = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS, |
| 446 | WGET_OPT_POST_DATA = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS, |
| 447 | }; |
| 448 | |
| 449 | static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd) |
| 450 | { |
| 451 | char buf[512]; |
| 452 | |
| 453 | if (!(option_mask32 & WGET_OPT_QUIET)) |
| 454 | progress_meter(-1); |
| 455 | |
| 456 | if (G.chunked) |
| 457 | goto get_clen; |
| 458 | |
| 459 | /* Loops only if chunked */ |
| 460 | while (1) { |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 461 | while (1) { |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 462 | int n; |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 463 | unsigned rdsz; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 464 | |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 465 | rdsz = sizeof(buf); |
| 466 | if (G.got_clen) { |
Denys Vlasenko | d2c879d | 2009-12-11 14:12:28 +0100 | [diff] [blame] | 467 | if (G.content_len < (off_t)sizeof(buf)) { |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 468 | if ((int)G.content_len <= 0) |
| 469 | break; |
| 470 | rdsz = (unsigned)G.content_len; |
| 471 | } |
| 472 | } |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 473 | n = safe_fread(buf, rdsz, dfp); |
| 474 | if (n <= 0) { |
| 475 | if (ferror(dfp)) { |
| 476 | /* perror will not work: ferror doesn't set errno */ |
| 477 | bb_error_msg_and_die(bb_msg_read_error); |
| 478 | } |
| 479 | break; |
| 480 | } |
| 481 | xwrite(output_fd, buf, n); |
| 482 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 483 | G.transferred += n; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 484 | #endif |
| 485 | if (G.got_clen) |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 486 | G.content_len -= n; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 487 | } |
| 488 | |
| 489 | if (!G.chunked) |
| 490 | break; |
| 491 | |
| 492 | safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */ |
| 493 | get_clen: |
| 494 | safe_fgets(buf, sizeof(buf), dfp); |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 495 | G.content_len = STRTOOFF(buf, NULL, 16); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 496 | /* FIXME: error check? */ |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 497 | if (G.content_len == 0) |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 498 | break; /* all done! */ |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 499 | G.got_clen = 1; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 500 | } |
| 501 | |
| 502 | if (!(option_mask32 & WGET_OPT_QUIET)) |
| 503 | progress_meter(0); |
| 504 | } |
| 505 | |
Denis Vlasenko | 9b49a5e | 2007-10-11 10:05:36 +0000 | [diff] [blame] | 506 | int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; |
Denis Vlasenko | a60f84e | 2008-07-05 09:18:54 +0000 | [diff] [blame] | 507 | int wget_main(int argc UNUSED_PARAM, char **argv) |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 508 | { |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 509 | char buf[512]; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 510 | struct host_info server, target; |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 511 | len_and_sockaddr *lsa; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 512 | unsigned opt; |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 513 | int redir_limit; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 514 | char *proxy = NULL; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 515 | char *dir_prefix = NULL; |
| 516 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 5a2ad69 | 2009-03-04 14:13:37 +0000 | [diff] [blame] | 517 | char *post_data; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 518 | char *extra_headers = NULL; |
Glenn L McGrath | 514aeab | 2003-12-19 12:08:56 +0000 | [diff] [blame] | 519 | llist_t *headers_llist = NULL; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 520 | #endif |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 521 | FILE *sfp; /* socket to web/ftp server */ |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 522 | FILE *dfp; /* socket to ftp server (data) */ |
| 523 | char *fname_out; /* where to direct output (-O) */ |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 524 | int output_fd = -1; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 525 | bool use_proxy; /* Use proxies if env vars are set */ |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 526 | const char *proxy_flag = "on"; /* Use proxies if env vars are set */ |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 527 | const char *user_agent = "Wget";/* "User-Agent" header field */ |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 528 | |
Denis Vlasenko | 6ca409e | 2007-08-12 20:58:27 +0000 | [diff] [blame] | 529 | static const char keywords[] ALIGN1 = |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 530 | "content-length\0""transfer-encoding\0""chunked\0""location\0"; |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 531 | enum { |
| 532 | KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location |
| 533 | }; |
Bernhard Reutner-Fischer | 289e86a | 2006-08-20 20:01:24 +0000 | [diff] [blame] | 534 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 6ca409e | 2007-08-12 20:58:27 +0000 | [diff] [blame] | 535 | static const char wget_longopts[] ALIGN1 = |
Denis Vlasenko | bdc88fd | 2007-07-23 17:14:14 +0000 | [diff] [blame] | 536 | /* name, has_arg, val */ |
| 537 | "continue\0" No_argument "c" |
| 538 | "spider\0" No_argument "s" |
| 539 | "quiet\0" No_argument "q" |
| 540 | "output-document\0" Required_argument "O" |
| 541 | "directory-prefix\0" Required_argument "P" |
| 542 | "proxy\0" Required_argument "Y" |
| 543 | "user-agent\0" Required_argument "U" |
Denis Vlasenko | 50af926 | 2009-03-02 15:08:06 +0000 | [diff] [blame] | 544 | /* Ignored: */ |
| 545 | // "tries\0" Required_argument "t" |
| 546 | // "timeout\0" Required_argument "T" |
| 547 | /* Ignored (we always use PASV): */ |
Denis Vlasenko | bdc88fd | 2007-07-23 17:14:14 +0000 | [diff] [blame] | 548 | "passive-ftp\0" No_argument "\xff" |
| 549 | "header\0" Required_argument "\xfe" |
Denis Vlasenko | 5a2ad69 | 2009-03-04 14:13:37 +0000 | [diff] [blame] | 550 | "post-data\0" Required_argument "\xfd" |
Bernhard Reutner-Fischer | 3fdba18 | 2010-02-10 19:37:29 +0100 | [diff] [blame] | 551 | /* Ignored (we don't do ssl) */ |
| 552 | "no-check-certificate\0" No_argument "\xfc" |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 553 | ; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 554 | #endif |
| 555 | |
| 556 | INIT_G(); |
| 557 | |
| 558 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | bdc88fd | 2007-07-23 17:14:14 +0000 | [diff] [blame] | 559 | applet_long_options = wget_longopts; |
Bernhard Reutner-Fischer | 8d3a6f7 | 2006-05-31 14:11:38 +0000 | [diff] [blame] | 560 | #endif |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 561 | /* server.allocated = target.allocated = NULL; */ |
Denis Vlasenko | 5e34ff2 | 2009-04-21 11:09:40 +0000 | [diff] [blame] | 562 | opt_complementary = "-1" IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::"); |
Denis Vlasenko | 540ab70 | 2008-06-29 00:32:35 +0000 | [diff] [blame] | 563 | opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:", |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 564 | &fname_out, &dir_prefix, |
Denis Vlasenko | 540ab70 | 2008-06-29 00:32:35 +0000 | [diff] [blame] | 565 | &proxy_flag, &user_agent, |
| 566 | NULL, /* -t RETRIES */ |
| 567 | NULL /* -T NETWORK_READ_TIMEOUT */ |
Denis Vlasenko | 5e34ff2 | 2009-04-21 11:09:40 +0000 | [diff] [blame] | 568 | IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) |
| 569 | IF_FEATURE_WGET_LONG_OPTIONS(, &post_data) |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 570 | ); |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 571 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 572 | if (headers_llist) { |
| 573 | int size = 1; |
| 574 | char *cp; |
Denis Vlasenko | 8d9f495 | 2007-04-08 15:08:42 +0000 | [diff] [blame] | 575 | llist_t *ll = headers_llist; |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 576 | while (ll) { |
| 577 | size += strlen(ll->data) + 2; |
| 578 | ll = ll->link; |
| 579 | } |
| 580 | extra_headers = cp = xmalloc(size); |
Glenn L McGrath | 514aeab | 2003-12-19 12:08:56 +0000 | [diff] [blame] | 581 | while (headers_llist) { |
Denis Vlasenko | d50dda8 | 2008-06-15 05:40:56 +0000 | [diff] [blame] | 582 | cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist)); |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 583 | } |
| 584 | } |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 585 | #endif |
Tim Riker | c1ef7bd | 2006-01-25 00:08:53 +0000 | [diff] [blame] | 586 | |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 587 | /* TODO: compat issue: should handle "wget URL1 URL2..." */ |
Denys Vlasenko | 7d5ddf1 | 2009-06-30 20:36:27 +0200 | [diff] [blame] | 588 | |
Vladimir Dronnikov | be168b1 | 2009-10-05 02:18:01 +0200 | [diff] [blame] | 589 | target.user = NULL; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 590 | parse_url(argv[optind], &target); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 591 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 592 | /* Use the proxy if necessary */ |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 593 | use_proxy = (strcmp(proxy_flag, "off") != 0); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 594 | if (use_proxy) { |
Robert Griebl | d776011 | 2002-05-14 23:36:45 +0000 | [diff] [blame] | 595 | proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy"); |
Denys Vlasenko | 7d5ddf1 | 2009-06-30 20:36:27 +0200 | [diff] [blame] | 596 | if (proxy && proxy[0]) { |
Denys Vlasenko | 81fe2b1 | 2010-02-11 04:23:43 +0100 | [diff] [blame] | 597 | server.user = NULL; |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 598 | parse_url(proxy, &server); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 599 | } else { |
| 600 | use_proxy = 0; |
| 601 | } |
Robert Griebl | d776011 | 2002-05-14 23:36:45 +0000 | [diff] [blame] | 602 | } |
Denys Vlasenko | 7d5ddf1 | 2009-06-30 20:36:27 +0200 | [diff] [blame] | 603 | if (!use_proxy) { |
| 604 | server.port = target.port; |
| 605 | if (ENABLE_FEATURE_IPV6) { |
| 606 | server.host = xstrdup(target.host); |
| 607 | } else { |
| 608 | server.host = target.host; |
| 609 | } |
| 610 | } |
| 611 | |
| 612 | if (ENABLE_FEATURE_IPV6) |
| 613 | strip_ipv6_scope_id(target.host); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 614 | |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 615 | /* Guess an output filename, if there was no -O FILE */ |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 616 | if (!(opt & WGET_OPT_OUTNAME)) { |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 617 | fname_out = bb_get_last_path_component_nostrip(target.path); |
| 618 | /* handle "wget http://kernel.org//" */ |
| 619 | if (fname_out[0] == '/' || !fname_out[0]) |
Denis Vlasenko | b6aae0f | 2007-01-29 22:51:25 +0000 | [diff] [blame] | 620 | fname_out = (char*)"index.html"; |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 621 | /* -P DIR is considered only if there was no -O FILE */ |
| 622 | if (dir_prefix) |
Matt Kraai | 0382eb8 | 2001-07-19 19:13:55 +0000 | [diff] [blame] | 623 | fname_out = concat_path_file(dir_prefix, fname_out); |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 624 | } else { |
| 625 | if (LONE_DASH(fname_out)) { |
| 626 | /* -O - */ |
| 627 | output_fd = 1; |
| 628 | opt &= ~WGET_OPT_CONTINUE; |
| 629 | } |
Eric Andersen | 29edd00 | 2000-12-09 16:55:35 +0000 | [diff] [blame] | 630 | } |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 631 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 632 | G.curfile = bb_get_last_path_component_nostrip(fname_out); |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 633 | #endif |
| 634 | |
Denis Vlasenko | 4e4662c | 2006-11-23 13:10:23 +0000 | [diff] [blame] | 635 | /* Impossible? |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 636 | if ((opt & WGET_OPT_CONTINUE) && !fname_out) |
Denys Vlasenko | 6331cf0 | 2009-11-13 09:08:27 +0100 | [diff] [blame] | 637 | bb_error_msg_and_die("can't specify continue (-c) without a filename (-O)"); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 638 | */ |
Eric Andersen | 29edd00 | 2000-12-09 16:55:35 +0000 | [diff] [blame] | 639 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 640 | /* Determine where to start transfer */ |
Denis Vlasenko | 4e4662c | 2006-11-23 13:10:23 +0000 | [diff] [blame] | 641 | if (opt & WGET_OPT_CONTINUE) { |
Denis Vlasenko | 7039a66 | 2006-10-08 17:54:47 +0000 | [diff] [blame] | 642 | output_fd = open(fname_out, O_WRONLY); |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 643 | if (output_fd >= 0) { |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 644 | G.beg_range = xlseek(output_fd, 0, SEEK_END); |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 645 | } |
| 646 | /* File doesn't exist. We do not create file here yet. |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 647 | * We are not sure it exists on remove side */ |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 648 | } |
| 649 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 650 | redir_limit = 5; |
| 651 | resolve_lsa: |
Denis Vlasenko | 42823d5 | 2007-02-04 02:39:08 +0000 | [diff] [blame] | 652 | lsa = xhost2sockaddr(server.host, server.port); |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 653 | if (!(opt & WGET_OPT_QUIET)) { |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 654 | char *s = xmalloc_sockaddr2dotted(&lsa->u.sa); |
| 655 | fprintf(stderr, "Connecting to %s (%s)\n", server.host, s); |
| 656 | free(s); |
Eric Andersen | e6dc439 | 2003-10-31 09:31:46 +0000 | [diff] [blame] | 657 | } |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 658 | establish_session: |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 659 | if (use_proxy || !target.is_ftp) { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 660 | /* |
| 661 | * HTTP session |
| 662 | */ |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 663 | char *str; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 664 | int status; |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 665 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 666 | /* Open socket to http server */ |
| 667 | sfp = open_socket(lsa); |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 668 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 669 | /* Send HTTP request */ |
| 670 | if (use_proxy) { |
| 671 | fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n", |
| 672 | target.is_ftp ? "f" : "ht", target.host, |
| 673 | target.path); |
| 674 | } else { |
| 675 | if (opt & WGET_OPT_POST_DATA) |
| 676 | fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path); |
| 677 | else |
| 678 | fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path); |
| 679 | } |
Glenn L McGrath | e7bdfcc | 2003-08-28 22:03:19 +0000 | [diff] [blame] | 680 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 681 | fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n", |
| 682 | target.host, user_agent); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 683 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 684 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 685 | if (target.user) { |
| 686 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6, |
| 687 | base64enc_512(buf, target.user)); |
| 688 | } |
| 689 | if (use_proxy && server.user) { |
| 690 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n", |
| 691 | base64enc_512(buf, server.user)); |
| 692 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 693 | #endif |
| 694 | |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 695 | if (G.beg_range) |
| 696 | fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range); |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 697 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 698 | if (extra_headers) |
| 699 | fputs(extra_headers, sfp); |
Denis Vlasenko | 5a2ad69 | 2009-03-04 14:13:37 +0000 | [diff] [blame] | 700 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 701 | if (opt & WGET_OPT_POST_DATA) { |
| 702 | char *estr = URL_escape(post_data); |
| 703 | fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n"); |
| 704 | fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s", |
| 705 | (int) strlen(estr), estr); |
| 706 | /*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/ |
| 707 | /*fprintf(sfp, "%s\r\n", estr);*/ |
| 708 | free(estr); |
| 709 | } else |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 710 | #endif |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 711 | { /* If "Connection:" is needed, document why */ |
| 712 | fprintf(sfp, /* "Connection: close\r\n" */ "\r\n"); |
| 713 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 714 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 715 | /* |
| 716 | * Retrieve HTTP response line and check for "200" status code. |
| 717 | */ |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 718 | read_response: |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 719 | if (fgets(buf, sizeof(buf), sfp) == NULL) |
| 720 | bb_error_msg_and_die("no response from server"); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 721 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 722 | str = buf; |
| 723 | str = skip_non_whitespace(str); |
| 724 | str = skip_whitespace(str); |
| 725 | // FIXME: no error check |
| 726 | // xatou wouldn't work: "200 OK" |
| 727 | status = atoi(str); |
| 728 | switch (status) { |
| 729 | case 0: |
| 730 | case 100: |
| 731 | while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL) |
| 732 | /* eat all remaining headers */; |
| 733 | goto read_response; |
| 734 | case 200: |
Denis Vlasenko | 50b5cac | 2008-06-22 16:28:02 +0000 | [diff] [blame] | 735 | /* |
| 736 | Response 204 doesn't say "null file", it says "metadata |
| 737 | has changed but data didn't": |
| 738 | |
| 739 | "10.2.5 204 No Content |
| 740 | The server has fulfilled the request but does not need to return |
| 741 | an entity-body, and might want to return updated metainformation. |
| 742 | The response MAY include new or updated metainformation in the form |
| 743 | of entity-headers, which if present SHOULD be associated with |
| 744 | the requested variant. |
| 745 | |
| 746 | If the client is a user agent, it SHOULD NOT change its document |
| 747 | view from that which caused the request to be sent. This response |
| 748 | is primarily intended to allow input for actions to take place |
| 749 | without causing a change to the user agent's active document view, |
| 750 | although any new or updated metainformation SHOULD be applied |
| 751 | to the document currently in the user agent's active view. |
| 752 | |
| 753 | The 204 response MUST NOT include a message-body, and thus |
| 754 | is always terminated by the first empty line after the header fields." |
| 755 | |
| 756 | However, in real world it was observed that some web servers |
| 757 | (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero. |
| 758 | */ |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 759 | case 204: |
| 760 | break; |
| 761 | case 300: /* redirection */ |
| 762 | case 301: |
| 763 | case 302: |
| 764 | case 303: |
| 765 | break; |
| 766 | case 206: |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 767 | if (G.beg_range) |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 768 | break; |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 769 | /* fall through */ |
| 770 | default: |
| 771 | bb_error_msg_and_die("server returned error: %s", sanitize_string(buf)); |
| 772 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 773 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 774 | /* |
| 775 | * Retrieve HTTP headers. |
| 776 | */ |
| 777 | while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) { |
| 778 | /* gethdr converted "FOO:" string to lowercase */ |
Matthijs van de Water | 0d58666 | 2009-08-22 20:19:48 +0200 | [diff] [blame] | 779 | smalluint key; |
| 780 | /* strip trailing whitespace */ |
| 781 | char *s = strchrnul(str, '\0') - 1; |
| 782 | while (s >= str && (*s == ' ' || *s == '\t')) { |
| 783 | *s = '\0'; |
| 784 | s--; |
| 785 | } |
| 786 | key = index_in_strings(keywords, buf) + 1; |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 787 | if (key == KEY_content_length) { |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 788 | G.content_len = BB_STRTOOFF(str, NULL, 10); |
| 789 | if (G.content_len < 0 || errno) { |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 790 | bb_error_msg_and_die("content-length %s is garbage", sanitize_string(str)); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 791 | } |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 792 | G.got_clen = 1; |
| 793 | continue; |
| 794 | } |
| 795 | if (key == KEY_transfer_encoding) { |
| 796 | if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked) |
| 797 | bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str)); |
| 798 | G.chunked = G.got_clen = 1; |
| 799 | } |
| 800 | if (key == KEY_location && status >= 300) { |
| 801 | if (--redir_limit == 0) |
| 802 | bb_error_msg_and_die("too many redirections"); |
| 803 | fclose(sfp); |
| 804 | G.got_clen = 0; |
| 805 | G.chunked = 0; |
| 806 | if (str[0] == '/') |
| 807 | /* free(target.allocated); */ |
| 808 | target.path = /* target.allocated = */ xstrdup(str+1); |
| 809 | /* lsa stays the same: it's on the same server */ |
| 810 | else { |
| 811 | parse_url(str, &target); |
| 812 | if (!use_proxy) { |
| 813 | server.host = target.host; |
Denys Vlasenko | 7d5ddf1 | 2009-06-30 20:36:27 +0200 | [diff] [blame] | 814 | /* strip_ipv6_scope_id(target.host); - no! */ |
| 815 | /* we assume remote never gives us IPv6 addr with scope id */ |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 816 | server.port = target.port; |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 817 | free(lsa); |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 818 | goto resolve_lsa; |
| 819 | } /* else: lsa stays the same: we use proxy */ |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 820 | } |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 821 | goto establish_session; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 822 | } |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 823 | } |
| 824 | // if (status >= 300) |
| 825 | // bb_error_msg_and_die("bad redirection (no Location: header from server)"); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 826 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 827 | /* For HTTP, data is pumped over the same connection */ |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 828 | dfp = sfp; |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 829 | |
| 830 | } else { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 831 | /* |
| 832 | * FTP session |
| 833 | */ |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 834 | sfp = prepare_ftp_session(&dfp, &target, lsa); |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 835 | } |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 836 | |
Bernhard Reutner-Fischer | 2e75dcc | 2007-04-05 10:31:47 +0000 | [diff] [blame] | 837 | if (opt & WGET_OPT_SPIDER) { |
| 838 | if (ENABLE_FEATURE_CLEAN_UP) |
| 839 | fclose(sfp); |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 840 | return EXIT_SUCCESS; |
Bernhard Reutner-Fischer | 2e75dcc | 2007-04-05 10:31:47 +0000 | [diff] [blame] | 841 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 842 | |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 843 | if (output_fd < 0) { |
| 844 | int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL; |
| 845 | /* compat with wget: -O FILE can overwrite */ |
| 846 | if (opt & WGET_OPT_OUTNAME) |
| 847 | o_flags = O_WRONLY | O_CREAT | O_TRUNC; |
| 848 | output_fd = xopen(fname_out, o_flags); |
| 849 | } |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 850 | |
Denys Vlasenko | 7f43280 | 2009-06-28 01:02:24 +0200 | [diff] [blame] | 851 | retrieve_file_data(dfp, output_fd); |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 852 | xclose(output_fd); |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 853 | |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 854 | if (dfp != sfp) { |
| 855 | /* It's ftp. Close it properly */ |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 856 | fclose(dfp); |
| 857 | if (ftpcmd(NULL, NULL, sfp, buf) != 226) |
Denys Vlasenko | f1fab09 | 2009-06-28 03:33:57 +0200 | [diff] [blame] | 858 | bb_error_msg_and_die("ftp error: %s", sanitize_string(buf+4)); |
Denys Vlasenko | a3aa3e3 | 2009-12-11 12:36:10 +0100 | [diff] [blame] | 859 | /* ftpcmd("QUIT", NULL, sfp, buf); - why bother? */ |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 860 | } |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 861 | |
| 862 | return EXIT_SUCCESS; |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 863 | } |