Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 1 | /* vi: set sw=4 ts=4: */ |
| 2 | /* |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 3 | * wget - retrieve a file using HTTP or FTP |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 4 | * |
Eric Andersen | 4e573f4 | 2000-11-14 23:29:24 +0000 | [diff] [blame] | 5 | * Chip Rosenthal Covad Communications <chip@laserlink.net> |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 6 | * |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 7 | */ |
| 8 | |
Denis Vlasenko | b6adbf1 | 2007-05-26 19:00:18 +0000 | [diff] [blame] | 9 | #include "libbb.h" |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 10 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 11 | struct host_info { |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 12 | // May be used if we ever will want to free() all xstrdup()s... |
| 13 | /* char *allocated; */ |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 14 | const char *path; |
| 15 | const char *user; |
| 16 | char *host; |
| 17 | int port; |
| 18 | smallint is_ftp; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 19 | }; |
| 20 | |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 21 | |
| 22 | /* Globals (can be accessed from signal handlers) */ |
| 23 | struct globals { |
| 24 | off_t content_len; /* Content-length of the file */ |
| 25 | off_t beg_range; /* Range at which continue begins */ |
| 26 | #if ENABLE_FEATURE_WGET_STATUSBAR |
| 27 | off_t lastsize; |
| 28 | off_t totalsize; |
| 29 | off_t transferred; /* Number of bytes transferred so far */ |
| 30 | const char *curfile; /* Name of current file being transferred */ |
| 31 | unsigned lastupdate_sec; |
| 32 | unsigned start_sec; |
| 33 | #endif |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 34 | smallint chunked; /* chunked transfer encoding */ |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 35 | }; |
| 36 | #define G (*(struct globals*)&bb_common_bufsiz1) |
| 37 | struct BUG_G_too_big { |
Denis Vlasenko | 6b40443 | 2008-01-07 16:13:14 +0000 | [diff] [blame] | 38 | char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1]; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 39 | }; |
| 40 | #define content_len (G.content_len ) |
| 41 | #define beg_range (G.beg_range ) |
| 42 | #define lastsize (G.lastsize ) |
| 43 | #define totalsize (G.totalsize ) |
| 44 | #define transferred (G.transferred ) |
| 45 | #define curfile (G.curfile ) |
| 46 | #define lastupdate_sec (G.lastupdate_sec ) |
| 47 | #define start_sec (G.start_sec ) |
| 48 | #define chunked (G.chunked ) |
| 49 | #define INIT_G() do { } while (0) |
| 50 | |
| 51 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 52 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 53 | enum { |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 54 | STALLTIME = 5 /* Seconds when xfer considered "stalled" */ |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 55 | }; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 56 | |
Denis Vlasenko | 5599502 | 2008-05-18 22:28:26 +0000 | [diff] [blame] | 57 | static unsigned int getttywidth(void) |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 58 | { |
Denis Vlasenko | 5599502 | 2008-05-18 22:28:26 +0000 | [diff] [blame] | 59 | unsigned width; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 60 | get_terminal_width_height(0, &width, NULL); |
| 61 | return width; |
| 62 | } |
| 63 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 64 | static void progressmeter(int flag) |
| 65 | { |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 66 | /* We can be called from signal handler */ |
| 67 | int save_errno = errno; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 68 | off_t abbrevsize; |
| 69 | unsigned since_last_update, elapsed; |
| 70 | unsigned ratio; |
| 71 | int barlength, i; |
| 72 | |
| 73 | if (flag == -1) { /* first call to progressmeter */ |
| 74 | start_sec = monotonic_sec(); |
| 75 | lastupdate_sec = start_sec; |
| 76 | lastsize = 0; |
| 77 | totalsize = content_len + beg_range; /* as content_len changes.. */ |
| 78 | } |
| 79 | |
| 80 | ratio = 100; |
| 81 | if (totalsize != 0 && !chunked) { |
| 82 | /* long long helps to have it working even if !LFS */ |
| 83 | ratio = (unsigned) (100ULL * (transferred+beg_range) / totalsize); |
| 84 | if (ratio > 100) ratio = 100; |
| 85 | } |
| 86 | |
| 87 | fprintf(stderr, "\r%-20.20s%4d%% ", curfile, ratio); |
| 88 | |
| 89 | barlength = getttywidth() - 49; |
| 90 | if (barlength > 0) { |
| 91 | /* god bless gcc for variable arrays :) */ |
| 92 | i = barlength * ratio / 100; |
| 93 | { |
| 94 | char buf[i+1]; |
| 95 | memset(buf, '*', i); |
| 96 | buf[i] = '\0'; |
| 97 | fprintf(stderr, "|%s%*s|", buf, barlength - i, ""); |
| 98 | } |
| 99 | } |
| 100 | i = 0; |
| 101 | abbrevsize = transferred + beg_range; |
| 102 | while (abbrevsize >= 100000) { |
| 103 | i++; |
| 104 | abbrevsize >>= 10; |
| 105 | } |
| 106 | /* see http://en.wikipedia.org/wiki/Tera */ |
| 107 | fprintf(stderr, "%6d%c ", (int)abbrevsize, " kMGTPEZY"[i]); |
| 108 | |
| 109 | // Nuts! Ain't it easier to update progress meter ONLY when we transferred++? |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 110 | |
| 111 | elapsed = monotonic_sec(); |
| 112 | since_last_update = elapsed - lastupdate_sec; |
| 113 | if (transferred > lastsize) { |
| 114 | lastupdate_sec = elapsed; |
| 115 | lastsize = transferred; |
| 116 | if (since_last_update >= STALLTIME) { |
| 117 | /* We "cut off" these seconds from elapsed time |
| 118 | * by adjusting start time */ |
| 119 | start_sec += since_last_update; |
| 120 | } |
| 121 | since_last_update = 0; /* we are un-stalled now */ |
| 122 | } |
| 123 | elapsed -= start_sec; /* now it's "elapsed since start" */ |
| 124 | |
| 125 | if (since_last_update >= STALLTIME) { |
| 126 | fprintf(stderr, " - stalled -"); |
| 127 | } else { |
| 128 | off_t to_download = totalsize - beg_range; |
| 129 | if (transferred <= 0 || (int)elapsed <= 0 || transferred > to_download || chunked) { |
| 130 | fprintf(stderr, "--:--:-- ETA"); |
| 131 | } else { |
| 132 | /* to_download / (transferred/elapsed) - elapsed: */ |
| 133 | int eta = (int) ((unsigned long long)to_download*elapsed/transferred - elapsed); |
| 134 | /* (long long helps to have working ETA even if !LFS) */ |
| 135 | i = eta % 3600; |
| 136 | fprintf(stderr, "%02d:%02d:%02d ETA", eta / 3600, i / 60, i % 60); |
| 137 | } |
| 138 | } |
| 139 | |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 140 | if (flag == 0) { |
| 141 | /* last call to progressmeter */ |
| 142 | alarm(0); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 143 | transferred = 0; |
Denis Vlasenko | 4daad90 | 2007-09-27 10:20:47 +0000 | [diff] [blame] | 144 | fputc('\n', stderr); |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 145 | } else { |
Denis Vlasenko | 8e2cfec | 2008-03-12 23:19:35 +0000 | [diff] [blame] | 146 | if (flag == -1) { /* first call to progressmeter */ |
| 147 | signal_SA_RESTART_empty_mask(SIGALRM, progressmeter); |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 148 | } |
| 149 | alarm(1); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 150 | } |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 151 | |
| 152 | errno = save_errno; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 153 | } |
| 154 | /* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff, |
| 155 | * much of which was blatantly stolen from openssh. */ |
| 156 | /*- |
| 157 | * Copyright (c) 1992, 1993 |
| 158 | * The Regents of the University of California. All rights reserved. |
| 159 | * |
| 160 | * Redistribution and use in source and binary forms, with or without |
| 161 | * modification, are permitted provided that the following conditions |
| 162 | * are met: |
| 163 | * 1. Redistributions of source code must retain the above copyright |
| 164 | * notice, this list of conditions and the following disclaimer. |
| 165 | * 2. Redistributions in binary form must reproduce the above copyright |
| 166 | * notice, this list of conditions and the following disclaimer in the |
| 167 | * documentation and/or other materials provided with the distribution. |
| 168 | * |
| 169 | * 3. <BSD Advertising Clause omitted per the July 22, 1999 licensing change |
| 170 | * ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change> |
| 171 | * |
| 172 | * 4. Neither the name of the University nor the names of its contributors |
| 173 | * may be used to endorse or promote products derived from this software |
| 174 | * without specific prior written permission. |
| 175 | * |
| 176 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
| 177 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 178 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 179 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
| 180 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 181 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 182 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 183 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 184 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 185 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 186 | * SUCH DAMAGE. |
| 187 | * |
| 188 | */ |
| 189 | #else /* FEATURE_WGET_STATUSBAR */ |
| 190 | |
Denis Vlasenko | 85c2471 | 2008-03-17 09:04:04 +0000 | [diff] [blame] | 191 | static ALWAYS_INLINE void progressmeter(int flag ATTRIBUTE_UNUSED) { } |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 192 | |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 193 | #endif |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 194 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 195 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 196 | /* Read NMEMB bytes into PTR from STREAM. Returns the number of bytes read, |
| 197 | * and a short count if an eof or non-interrupt error is encountered. */ |
| 198 | static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream) |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 199 | { |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 200 | size_t ret; |
| 201 | char *p = (char*)ptr; |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 202 | |
| 203 | do { |
| 204 | clearerr(stream); |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 205 | ret = fread(p, 1, nmemb, stream); |
| 206 | p += ret; |
| 207 | nmemb -= ret; |
| 208 | } while (nmemb && ferror(stream) && errno == EINTR); |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 209 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 210 | return p - (char*)ptr; |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 211 | } |
| 212 | |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 213 | /* Read a line or SIZE-1 bytes into S, whichever is less, from STREAM. |
Matt Kraai | 854125f | 2001-05-09 19:15:46 +0000 | [diff] [blame] | 214 | * Returns S, or NULL if an eof or non-interrupt error is encountered. */ |
| 215 | static char *safe_fgets(char *s, int size, FILE *stream) |
| 216 | { |
| 217 | char *ret; |
| 218 | |
| 219 | do { |
| 220 | clearerr(stream); |
| 221 | ret = fgets(s, size, stream); |
| 222 | } while (ret == NULL && ferror(stream) && errno == EINTR); |
| 223 | |
| 224 | return ret; |
| 225 | } |
| 226 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 227 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 228 | /* Base64-encode character string. buf is assumed to be char buf[512]. */ |
| 229 | static char *base64enc_512(char buf[512], const char *str) |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 230 | { |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 231 | unsigned len = strlen(str); |
| 232 | if (len > 512/4*3 - 10) /* paranoia */ |
| 233 | len = 512/4*3 - 10; |
| 234 | bb_uuencode(buf, str, len, bb_uuenc_tbl_base64); |
Rob Landley | 76ef08c | 2006-06-13 16:44:26 +0000 | [diff] [blame] | 235 | return buf; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 236 | } |
| 237 | #endif |
| 238 | |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 239 | |
| 240 | static FILE *open_socket(len_and_sockaddr *lsa) |
| 241 | { |
| 242 | FILE *fp; |
| 243 | |
| 244 | /* glibc 2.4 seems to try seeking on it - ??! */ |
| 245 | /* hopefully it understands what ESPIPE means... */ |
| 246 | fp = fdopen(xconnect_stream(lsa), "r+"); |
| 247 | if (fp == NULL) |
| 248 | bb_perror_msg_and_die("fdopen"); |
| 249 | |
| 250 | return fp; |
| 251 | } |
| 252 | |
| 253 | |
| 254 | static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf) |
| 255 | { |
| 256 | int result; |
| 257 | if (s1) { |
| 258 | if (!s2) s2 = ""; |
| 259 | fprintf(fp, "%s%s\r\n", s1, s2); |
| 260 | fflush(fp); |
| 261 | } |
| 262 | |
| 263 | do { |
| 264 | char *buf_ptr; |
| 265 | |
| 266 | if (fgets(buf, 510, fp) == NULL) { |
| 267 | bb_perror_msg_and_die("error getting response"); |
| 268 | } |
| 269 | buf_ptr = strstr(buf, "\r\n"); |
| 270 | if (buf_ptr) { |
| 271 | *buf_ptr = '\0'; |
| 272 | } |
| 273 | } while (!isdigit(buf[0]) || buf[3] != ' '); |
| 274 | |
| 275 | buf[3] = '\0'; |
| 276 | result = xatoi_u(buf); |
| 277 | buf[3] = ' '; |
| 278 | return result; |
| 279 | } |
| 280 | |
| 281 | |
| 282 | static void parse_url(char *src_url, struct host_info *h) |
| 283 | { |
| 284 | char *url, *p, *sp; |
| 285 | |
| 286 | /* h->allocated = */ url = xstrdup(src_url); |
| 287 | |
| 288 | if (strncmp(url, "http://", 7) == 0) { |
| 289 | h->port = bb_lookup_port("http", "tcp", 80); |
| 290 | h->host = url + 7; |
| 291 | h->is_ftp = 0; |
| 292 | } else if (strncmp(url, "ftp://", 6) == 0) { |
| 293 | h->port = bb_lookup_port("ftp", "tcp", 21); |
| 294 | h->host = url + 6; |
| 295 | h->is_ftp = 1; |
| 296 | } else |
| 297 | bb_error_msg_and_die("not an http or ftp url: %s", url); |
| 298 | |
| 299 | // FYI: |
| 300 | // "Real" wget 'http://busybox.net?var=a/b' sends this request: |
| 301 | // 'GET /?var=a/b HTTP 1.0' |
| 302 | // and saves 'index.html?var=a%2Fb' (we save 'b') |
| 303 | // wget 'http://busybox.net?login=john@doe': |
| 304 | // request: 'GET /?login=john@doe HTTP/1.0' |
| 305 | // saves: 'index.html?login=john@doe' (we save '?login=john@doe') |
| 306 | // wget 'http://busybox.net#test/test': |
| 307 | // request: 'GET / HTTP/1.0' |
| 308 | // saves: 'index.html' (we save 'test') |
| 309 | // |
| 310 | // We also don't add unique .N suffix if file exists... |
| 311 | sp = strchr(h->host, '/'); |
| 312 | p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p; |
| 313 | p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p; |
| 314 | if (!sp) { |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 315 | h->path = ""; |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 316 | } else if (*sp == '/') { |
| 317 | *sp = '\0'; |
| 318 | h->path = sp + 1; |
| 319 | } else { // '#' or '?' |
| 320 | // http://busybox.net?login=john@doe is a valid URL |
| 321 | // memmove converts to: |
| 322 | // http:/busybox.nett?login=john@doe... |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 323 | memmove(h->host - 1, h->host, sp - h->host); |
Denis Vlasenko | 47ddd01 | 2007-09-24 18:24:17 +0000 | [diff] [blame] | 324 | h->host--; |
| 325 | sp[-1] = '\0'; |
| 326 | h->path = sp; |
| 327 | } |
| 328 | |
| 329 | sp = strrchr(h->host, '@'); |
| 330 | h->user = NULL; |
| 331 | if (sp != NULL) { |
| 332 | h->user = h->host; |
| 333 | *sp = '\0'; |
| 334 | h->host = sp + 1; |
| 335 | } |
| 336 | |
| 337 | sp = h->host; |
| 338 | } |
| 339 | |
| 340 | |
| 341 | static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/) |
| 342 | { |
| 343 | char *s, *hdrval; |
| 344 | int c; |
| 345 | |
| 346 | /* *istrunc = 0; */ |
| 347 | |
| 348 | /* retrieve header line */ |
| 349 | if (fgets(buf, bufsiz, fp) == NULL) |
| 350 | return NULL; |
| 351 | |
| 352 | /* see if we are at the end of the headers */ |
| 353 | for (s = buf; *s == '\r'; ++s) |
| 354 | continue; |
| 355 | if (*s == '\n') |
| 356 | return NULL; |
| 357 | |
| 358 | /* convert the header name to lower case */ |
| 359 | for (s = buf; isalnum(*s) || *s == '-' || *s == '.'; ++s) |
| 360 | *s = tolower(*s); |
| 361 | |
| 362 | /* verify we are at the end of the header name */ |
| 363 | if (*s != ':') |
| 364 | bb_error_msg_and_die("bad header line: %s", buf); |
| 365 | |
| 366 | /* locate the start of the header value */ |
| 367 | *s++ = '\0'; |
| 368 | hdrval = skip_whitespace(s); |
| 369 | |
| 370 | /* locate the end of header */ |
| 371 | while (*s && *s != '\r' && *s != '\n') |
| 372 | ++s; |
| 373 | |
| 374 | /* end of header found */ |
| 375 | if (*s) { |
| 376 | *s = '\0'; |
| 377 | return hdrval; |
| 378 | } |
| 379 | |
| 380 | /* Rats! The buffer isn't big enough to hold the entire header value. */ |
| 381 | while (c = getc(fp), c != EOF && c != '\n') |
| 382 | continue; |
| 383 | /* *istrunc = 1; */ |
| 384 | return hdrval; |
| 385 | } |
| 386 | |
| 387 | |
Denis Vlasenko | 9b49a5e | 2007-10-11 10:05:36 +0000 | [diff] [blame] | 388 | int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; |
Denis Vlasenko | 68404f1 | 2008-03-17 09:00:54 +0000 | [diff] [blame] | 389 | int wget_main(int argc ATTRIBUTE_UNUSED, char **argv) |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 390 | { |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 391 | char buf[512]; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 392 | struct host_info server, target; |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 393 | len_and_sockaddr *lsa; |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 394 | int status; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 395 | int port; |
| 396 | int try = 5; |
| 397 | unsigned opt; |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 398 | char *str; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 399 | char *proxy = 0; |
| 400 | char *dir_prefix = NULL; |
| 401 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
| 402 | char *extra_headers = NULL; |
Glenn L McGrath | 514aeab | 2003-12-19 12:08:56 +0000 | [diff] [blame] | 403 | llist_t *headers_llist = NULL; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 404 | #endif |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 405 | FILE *sfp = NULL; /* socket to web/ftp server */ |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 406 | FILE *dfp; /* socket to ftp server (data) */ |
| 407 | char *fname_out; /* where to direct output (-O) */ |
Denis Vlasenko | 4661117 | 2007-08-06 15:43:17 +0000 | [diff] [blame] | 408 | bool got_clen = 0; /* got content-length: from server */ |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 409 | int output_fd = -1; |
Denis Vlasenko | 4661117 | 2007-08-06 15:43:17 +0000 | [diff] [blame] | 410 | bool use_proxy = 1; /* Use proxies if env vars are set */ |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 411 | const char *proxy_flag = "on"; /* Use proxies if env vars are set */ |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 412 | const char *user_agent = "Wget";/* "User-Agent" header field */ |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 413 | |
Denis Vlasenko | 6ca409e | 2007-08-12 20:58:27 +0000 | [diff] [blame] | 414 | static const char keywords[] ALIGN1 = |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 415 | "content-length\0""transfer-encoding\0""chunked\0""location\0"; |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 416 | enum { |
| 417 | KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location |
| 418 | }; |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 419 | enum { |
| 420 | WGET_OPT_CONTINUE = 0x1, |
Bernhard Reutner-Fischer | 2e75dcc | 2007-04-05 10:31:47 +0000 | [diff] [blame] | 421 | WGET_OPT_SPIDER = 0x2, |
| 422 | WGET_OPT_QUIET = 0x4, |
| 423 | WGET_OPT_OUTNAME = 0x8, |
| 424 | WGET_OPT_PREFIX = 0x10, |
| 425 | WGET_OPT_PROXY = 0x20, |
| 426 | WGET_OPT_USER_AGENT = 0x40, |
| 427 | WGET_OPT_PASSIVE = 0x80, |
| 428 | WGET_OPT_HEADER = 0x100, |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 429 | }; |
Bernhard Reutner-Fischer | 289e86a | 2006-08-20 20:01:24 +0000 | [diff] [blame] | 430 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 6ca409e | 2007-08-12 20:58:27 +0000 | [diff] [blame] | 431 | static const char wget_longopts[] ALIGN1 = |
Denis Vlasenko | bdc88fd | 2007-07-23 17:14:14 +0000 | [diff] [blame] | 432 | /* name, has_arg, val */ |
| 433 | "continue\0" No_argument "c" |
| 434 | "spider\0" No_argument "s" |
| 435 | "quiet\0" No_argument "q" |
| 436 | "output-document\0" Required_argument "O" |
| 437 | "directory-prefix\0" Required_argument "P" |
| 438 | "proxy\0" Required_argument "Y" |
| 439 | "user-agent\0" Required_argument "U" |
| 440 | "passive-ftp\0" No_argument "\xff" |
| 441 | "header\0" Required_argument "\xfe" |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 442 | ; |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 443 | #endif |
| 444 | |
| 445 | INIT_G(); |
| 446 | |
| 447 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | bdc88fd | 2007-07-23 17:14:14 +0000 | [diff] [blame] | 448 | applet_long_options = wget_longopts; |
Bernhard Reutner-Fischer | 8d3a6f7 | 2006-05-31 14:11:38 +0000 | [diff] [blame] | 449 | #endif |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 450 | /* server.allocated = target.allocated = NULL; */ |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 451 | opt_complementary = "-1" USE_FEATURE_WGET_LONG_OPTIONS(":\xfe::"); |
Denis Vlasenko | 540ab70 | 2008-06-29 00:32:35 +0000 | [diff] [blame^] | 452 | opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:", |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 453 | &fname_out, &dir_prefix, |
Denis Vlasenko | 540ab70 | 2008-06-29 00:32:35 +0000 | [diff] [blame^] | 454 | &proxy_flag, &user_agent, |
| 455 | NULL, /* -t RETRIES */ |
| 456 | NULL /* -T NETWORK_READ_TIMEOUT */ |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 457 | USE_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) |
| 458 | ); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 459 | if (strcmp(proxy_flag, "off") == 0) { |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 460 | /* Use the proxy if necessary */ |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 461 | use_proxy = 0; |
Glenn L McGrath | 514aeab | 2003-12-19 12:08:56 +0000 | [diff] [blame] | 462 | } |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 463 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 464 | if (headers_llist) { |
| 465 | int size = 1; |
| 466 | char *cp; |
Denis Vlasenko | 8d9f495 | 2007-04-08 15:08:42 +0000 | [diff] [blame] | 467 | llist_t *ll = headers_llist; |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 468 | while (ll) { |
| 469 | size += strlen(ll->data) + 2; |
| 470 | ll = ll->link; |
| 471 | } |
| 472 | extra_headers = cp = xmalloc(size); |
Glenn L McGrath | 514aeab | 2003-12-19 12:08:56 +0000 | [diff] [blame] | 473 | while (headers_llist) { |
Denis Vlasenko | d50dda8 | 2008-06-15 05:40:56 +0000 | [diff] [blame] | 474 | cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist)); |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 475 | } |
| 476 | } |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 477 | #endif |
Tim Riker | c1ef7bd | 2006-01-25 00:08:53 +0000 | [diff] [blame] | 478 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 479 | parse_url(argv[optind], &target); |
| 480 | server.host = target.host; |
| 481 | server.port = target.port; |
| 482 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 483 | /* Use the proxy if necessary */ |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 484 | if (use_proxy) { |
Robert Griebl | d776011 | 2002-05-14 23:36:45 +0000 | [diff] [blame] | 485 | proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy"); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 486 | if (proxy && *proxy) { |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 487 | parse_url(proxy, &server); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 488 | } else { |
| 489 | use_proxy = 0; |
| 490 | } |
Robert Griebl | d776011 | 2002-05-14 23:36:45 +0000 | [diff] [blame] | 491 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 492 | |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 493 | /* Guess an output filename, if there was no -O FILE */ |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 494 | if (!(opt & WGET_OPT_OUTNAME)) { |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 495 | fname_out = bb_get_last_path_component_nostrip(target.path); |
| 496 | /* handle "wget http://kernel.org//" */ |
| 497 | if (fname_out[0] == '/' || !fname_out[0]) |
Denis Vlasenko | b6aae0f | 2007-01-29 22:51:25 +0000 | [diff] [blame] | 498 | fname_out = (char*)"index.html"; |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 499 | /* -P DIR is considered only if there was no -O FILE */ |
| 500 | if (dir_prefix) |
Matt Kraai | 0382eb8 | 2001-07-19 19:13:55 +0000 | [diff] [blame] | 501 | fname_out = concat_path_file(dir_prefix, fname_out); |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 502 | } else { |
| 503 | if (LONE_DASH(fname_out)) { |
| 504 | /* -O - */ |
| 505 | output_fd = 1; |
| 506 | opt &= ~WGET_OPT_CONTINUE; |
| 507 | } |
Eric Andersen | 29edd00 | 2000-12-09 16:55:35 +0000 | [diff] [blame] | 508 | } |
Denis Vlasenko | 818322b | 2007-09-24 18:27:04 +0000 | [diff] [blame] | 509 | #if ENABLE_FEATURE_WGET_STATUSBAR |
| 510 | curfile = bb_get_last_path_component_nostrip(fname_out); |
| 511 | #endif |
| 512 | |
Denis Vlasenko | 4e4662c | 2006-11-23 13:10:23 +0000 | [diff] [blame] | 513 | /* Impossible? |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 514 | if ((opt & WGET_OPT_CONTINUE) && !fname_out) |
Denis Vlasenko | 4e4662c | 2006-11-23 13:10:23 +0000 | [diff] [blame] | 515 | bb_error_msg_and_die("cannot specify continue (-c) without a filename (-O)"); */ |
Eric Andersen | 29edd00 | 2000-12-09 16:55:35 +0000 | [diff] [blame] | 516 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 517 | /* Determine where to start transfer */ |
Denis Vlasenko | 4e4662c | 2006-11-23 13:10:23 +0000 | [diff] [blame] | 518 | if (opt & WGET_OPT_CONTINUE) { |
Denis Vlasenko | 7039a66 | 2006-10-08 17:54:47 +0000 | [diff] [blame] | 519 | output_fd = open(fname_out, O_WRONLY); |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 520 | if (output_fd >= 0) { |
Denis Vlasenko | ea62077 | 2006-10-14 02:23:43 +0000 | [diff] [blame] | 521 | beg_range = xlseek(output_fd, 0, SEEK_END); |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 522 | } |
| 523 | /* File doesn't exist. We do not create file here yet. |
| 524 | We are not sure it exists on remove side */ |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 525 | } |
| 526 | |
Eric Andersen | e6dc439 | 2003-10-31 09:31:46 +0000 | [diff] [blame] | 527 | /* We want to do exactly _one_ DNS lookup, since some |
| 528 | * sites (i.e. ftp.us.debian.org) use round-robin DNS |
| 529 | * and we want to connect to only one IP... */ |
Denis Vlasenko | 42823d5 | 2007-02-04 02:39:08 +0000 | [diff] [blame] | 530 | lsa = xhost2sockaddr(server.host, server.port); |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 531 | if (!(opt & WGET_OPT_QUIET)) { |
Denis Vlasenko | 85629f0 | 2007-01-22 09:36:41 +0000 | [diff] [blame] | 532 | fprintf(stderr, "Connecting to %s (%s)\n", server.host, |
Bernhard Reutner-Fischer | 8c69afd | 2008-01-29 10:33:34 +0000 | [diff] [blame] | 533 | xmalloc_sockaddr2dotted(&lsa->u.sa)); |
Denis Vlasenko | 85629f0 | 2007-01-22 09:36:41 +0000 | [diff] [blame] | 534 | /* We leak result of xmalloc_sockaddr2dotted */ |
Eric Andersen | e6dc439 | 2003-10-31 09:31:46 +0000 | [diff] [blame] | 535 | } |
| 536 | |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 537 | if (use_proxy || !target.is_ftp) { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 538 | /* |
| 539 | * HTTP session |
| 540 | */ |
| 541 | do { |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 542 | got_clen = 0; |
| 543 | chunked = 0; |
Glenn L McGrath | e7bdfcc | 2003-08-28 22:03:19 +0000 | [diff] [blame] | 544 | |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 545 | if (!--try) |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 546 | bb_error_msg_and_die("too many redirections"); |
Eric Andersen | 7d69701 | 2001-01-24 20:28:35 +0000 | [diff] [blame] | 547 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 548 | /* Open socket to http server */ |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 549 | if (sfp) fclose(sfp); |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 550 | sfp = open_socket(lsa); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 551 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 552 | /* Send HTTP request. */ |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 553 | if (use_proxy) { |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 554 | fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n", |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 555 | target.is_ftp ? "f" : "ht", target.host, |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 556 | target.path); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 557 | } else { |
Eric Andersen | 6d7fa43 | 2001-04-10 18:17:05 +0000 | [diff] [blame] | 558 | fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 559 | } |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 560 | |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 561 | fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n", |
| 562 | target.host, user_agent); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 563 | |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 564 | #if ENABLE_FEATURE_WGET_AUTHENTICATION |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 565 | if (target.user) { |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 566 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6, |
| 567 | base64enc_512(buf, target.user)); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 568 | } |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 569 | if (use_proxy && server.user) { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 570 | fprintf(sfp, "Proxy-Authorization: Basic %s\r\n", |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 571 | base64enc_512(buf, server.user)); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 572 | } |
| 573 | #endif |
| 574 | |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 575 | if (beg_range) |
Denis Vlasenko | cf30cc8 | 2006-11-24 14:53:18 +0000 | [diff] [blame] | 576 | fprintf(sfp, "Range: bytes=%"OFF_FMT"d-\r\n", beg_range); |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 577 | #if ENABLE_FEATURE_WGET_LONG_OPTIONS |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 578 | if (extra_headers) |
| 579 | fputs(extra_headers, sfp); |
Denis Vlasenko | c8400a2 | 2006-10-25 00:33:44 +0000 | [diff] [blame] | 580 | #endif |
Denis Vlasenko | 7534e08 | 2006-10-23 23:21:58 +0000 | [diff] [blame] | 581 | fprintf(sfp, "Connection: close\r\n\r\n"); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 582 | |
| 583 | /* |
Tim Riker | c1ef7bd | 2006-01-25 00:08:53 +0000 | [diff] [blame] | 584 | * Retrieve HTTP response line and check for "200" status code. |
| 585 | */ |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 586 | read_response: |
Glenn L McGrath | e7bdfcc | 2003-08-28 22:03:19 +0000 | [diff] [blame] | 587 | if (fgets(buf, sizeof(buf), sfp) == NULL) |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 588 | bb_error_msg_and_die("no response from server"); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 589 | |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 590 | str = buf; |
| 591 | str = skip_non_whitespace(str); |
| 592 | str = skip_whitespace(str); |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 593 | // FIXME: no error check |
| 594 | // xatou wouldn't work: "200 OK" |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 595 | status = atoi(str); |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 596 | switch (status) { |
| 597 | case 0: |
| 598 | case 100: |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 599 | while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL) |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 600 | /* eat all remaining headers */; |
| 601 | goto read_response; |
| 602 | case 200: |
Denis Vlasenko | 50b5cac | 2008-06-22 16:28:02 +0000 | [diff] [blame] | 603 | /* |
| 604 | Response 204 doesn't say "null file", it says "metadata |
| 605 | has changed but data didn't": |
| 606 | |
| 607 | "10.2.5 204 No Content |
| 608 | The server has fulfilled the request but does not need to return |
| 609 | an entity-body, and might want to return updated metainformation. |
| 610 | The response MAY include new or updated metainformation in the form |
| 611 | of entity-headers, which if present SHOULD be associated with |
| 612 | the requested variant. |
| 613 | |
| 614 | If the client is a user agent, it SHOULD NOT change its document |
| 615 | view from that which caused the request to be sent. This response |
| 616 | is primarily intended to allow input for actions to take place |
| 617 | without causing a change to the user agent's active document view, |
| 618 | although any new or updated metainformation SHOULD be applied |
| 619 | to the document currently in the user agent's active view. |
| 620 | |
| 621 | The 204 response MUST NOT include a message-body, and thus |
| 622 | is always terminated by the first empty line after the header fields." |
| 623 | |
| 624 | However, in real world it was observed that some web servers |
| 625 | (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero. |
| 626 | */ |
| 627 | case 204: |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 628 | break; |
| 629 | case 300: /* redirection */ |
| 630 | case 301: |
| 631 | case 302: |
| 632 | case 303: |
| 633 | break; |
| 634 | case 206: |
| 635 | if (beg_range) |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 636 | break; |
Bernhard Reutner-Fischer | a985d30 | 2008-02-11 11:44:38 +0000 | [diff] [blame] | 637 | /* fall through */ |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 638 | default: |
Denis Vlasenko | 067e3f0 | 2006-11-10 23:25:53 +0000 | [diff] [blame] | 639 | /* Show first line only and kill any ESC tricks */ |
| 640 | buf[strcspn(buf, "\n\r\x1b")] = '\0'; |
| 641 | bb_error_msg_and_die("server returned error: %s", buf); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 642 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 643 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 644 | /* |
| 645 | * Retrieve HTTP headers. |
| 646 | */ |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 647 | while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) { |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 648 | /* gethdr did already convert the "FOO:" string to lowercase */ |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 649 | smalluint key = index_in_strings(keywords, *&buf) + 1; |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 650 | if (key == KEY_content_length) { |
| 651 | content_len = BB_STRTOOFF(str, NULL, 10); |
Denis Vlasenko | d686a04 | 2006-11-27 14:43:21 +0000 | [diff] [blame] | 652 | if (errno || content_len < 0) { |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 653 | bb_error_msg_and_die("content-length %s is garbage", str); |
Eric Andersen | 2479445 | 2004-03-06 22:11:45 +0000 | [diff] [blame] | 654 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 655 | got_clen = 1; |
| 656 | continue; |
| 657 | } |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 658 | if (key == KEY_transfer_encoding) { |
Denis Vlasenko | 990d0f6 | 2007-07-24 15:54:42 +0000 | [diff] [blame] | 659 | if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked) |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 660 | bb_error_msg_and_die("transfer encoding '%s' is not supported", str); |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 661 | chunked = got_clen = 1; |
Eric Andersen | 6d7fa43 | 2001-04-10 18:17:05 +0000 | [diff] [blame] | 662 | } |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 663 | if (key == KEY_location) { |
| 664 | if (str[0] == '/') |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 665 | /* free(target.allocated); */ |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 666 | target.path = /* target.allocated = */ xstrdup(str+1); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 667 | else { |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 668 | parse_url(str, &target); |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 669 | if (use_proxy == 0) { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 670 | server.host = target.host; |
| 671 | server.port = target.port; |
| 672 | } |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 673 | free(lsa); |
Denis Vlasenko | 42823d5 | 2007-02-04 02:39:08 +0000 | [diff] [blame] | 674 | lsa = xhost2sockaddr(server.host, server.port); |
Glenn L McGrath | 58a2e0e | 2004-01-17 23:07:14 +0000 | [diff] [blame] | 675 | break; |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 676 | } |
| 677 | } |
| 678 | } |
Denis Vlasenko | 3469c18 | 2006-12-16 22:19:47 +0000 | [diff] [blame] | 679 | } while (status >= 300); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 680 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 681 | dfp = sfp; |
Denis Vlasenko | 96e9d3c | 2006-10-07 14:28:55 +0000 | [diff] [blame] | 682 | |
| 683 | } else { |
| 684 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 685 | /* |
| 686 | * FTP session |
| 687 | */ |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 688 | if (!target.user) |
Rob Landley | d921b2e | 2006-08-03 15:41:12 +0000 | [diff] [blame] | 689 | target.user = xstrdup("anonymous:busybox@"); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 690 | |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 691 | sfp = open_socket(lsa); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 692 | if (ftpcmd(NULL, NULL, sfp, buf) != 220) |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 693 | bb_error_msg_and_die("%s", buf+4); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 694 | |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 695 | /* |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 696 | * Splitting username:password pair, |
| 697 | * trying to log in |
| 698 | */ |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 699 | str = strchr(target.user, ':'); |
| 700 | if (str) |
| 701 | *(str++) = '\0'; |
Denis Vlasenko | c16bd21 | 2006-09-27 19:51:06 +0000 | [diff] [blame] | 702 | switch (ftpcmd("USER ", target.user, sfp, buf)) { |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 703 | case 230: |
| 704 | break; |
| 705 | case 331: |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 706 | if (ftpcmd("PASS ", str, sfp, buf) == 230) |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 707 | break; |
Bernhard Reutner-Fischer | a985d30 | 2008-02-11 11:44:38 +0000 | [diff] [blame] | 708 | /* fall through (failed login) */ |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 709 | default: |
| 710 | bb_error_msg_and_die("ftp login: %s", buf+4); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 711 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 712 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 713 | ftpcmd("TYPE I", NULL, sfp, buf); |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 714 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 715 | /* |
| 716 | * Querying file size |
| 717 | */ |
Rob Landley | af12cb3 | 2006-06-27 18:41:03 +0000 | [diff] [blame] | 718 | if (ftpcmd("SIZE ", target.path, sfp, buf) == 213) { |
Denis Vlasenko | d686a04 | 2006-11-27 14:43:21 +0000 | [diff] [blame] | 719 | content_len = BB_STRTOOFF(buf+4, NULL, 10); |
| 720 | if (errno || content_len < 0) { |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 721 | bb_error_msg_and_die("SIZE value is garbage"); |
Eric Andersen | 2479445 | 2004-03-06 22:11:45 +0000 | [diff] [blame] | 722 | } |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 723 | got_clen = 1; |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 724 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 725 | |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 726 | /* |
| 727 | * Entering passive mode |
| 728 | */ |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 729 | if (ftpcmd("PASV", NULL, sfp, buf) != 227) { |
| 730 | pasv_error: |
| 731 | bb_error_msg_and_die("bad response to %s: %s", "PASV", buf); |
| 732 | } |
Denis Vlasenko | f8c8bb1 | 2006-11-21 19:10:26 +0000 | [diff] [blame] | 733 | // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage] |
Denis Vlasenko | 023b57d | 2006-10-15 17:05:55 +0000 | [diff] [blame] | 734 | // Server's IP is N1.N2.N3.N4 (we ignore it) |
| 735 | // Server's port for data connection is P1*256+P2 |
Bernhard Reutner-Fischer | 7e8a53a | 2007-04-10 09:37:29 +0000 | [diff] [blame] | 736 | str = strrchr(buf, ')'); |
| 737 | if (str) str[0] = '\0'; |
| 738 | str = strrchr(buf, ','); |
| 739 | if (!str) goto pasv_error; |
| 740 | port = xatou_range(str+1, 0, 255); |
| 741 | *str = '\0'; |
| 742 | str = strrchr(buf, ','); |
| 743 | if (!str) goto pasv_error; |
| 744 | port += xatou_range(str+1, 0, 255) * 256; |
Denis Vlasenko | 5d68724 | 2007-01-12 20:59:31 +0000 | [diff] [blame] | 745 | set_nport(lsa, htons(port)); |
Denis Vlasenko | 6536a9b | 2007-01-12 10:35:23 +0000 | [diff] [blame] | 746 | dfp = open_socket(lsa); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 747 | |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 748 | if (beg_range) { |
Denis Vlasenko | cf30cc8 | 2006-11-24 14:53:18 +0000 | [diff] [blame] | 749 | sprintf(buf, "REST %"OFF_FMT"d", beg_range); |
Denis Vlasenko | a94554d | 2006-09-23 17:49:09 +0000 | [diff] [blame] | 750 | if (ftpcmd(buf, NULL, sfp, buf) == 350) |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 751 | content_len -= beg_range; |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 752 | } |
Eric Andersen | c7bda1c | 2004-03-15 08:29:22 +0000 | [diff] [blame] | 753 | |
Rob Landley | af12cb3 | 2006-06-27 18:41:03 +0000 | [diff] [blame] | 754 | if (ftpcmd("RETR ", target.path, sfp, buf) > 150) |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 755 | bb_error_msg_and_die("bad response to %s: %s", "RETR", buf); |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 756 | } |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 757 | |
Bernhard Reutner-Fischer | 2e75dcc | 2007-04-05 10:31:47 +0000 | [diff] [blame] | 758 | if (opt & WGET_OPT_SPIDER) { |
| 759 | if (ENABLE_FEATURE_CLEAN_UP) |
| 760 | fclose(sfp); |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 761 | return EXIT_SUCCESS; |
Bernhard Reutner-Fischer | 2e75dcc | 2007-04-05 10:31:47 +0000 | [diff] [blame] | 762 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 763 | |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 764 | /* |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 765 | * Retrieve file |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 766 | */ |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 767 | |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 768 | /* Do it before progressmeter (want to have nice error message) */ |
Denis Vlasenko | a36535b | 2007-09-27 15:07:23 +0000 | [diff] [blame] | 769 | if (output_fd < 0) { |
| 770 | int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL; |
| 771 | /* compat with wget: -O FILE can overwrite */ |
| 772 | if (opt & WGET_OPT_OUTNAME) |
| 773 | o_flags = O_WRONLY | O_CREAT | O_TRUNC; |
| 774 | output_fd = xopen(fname_out, o_flags); |
| 775 | } |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 776 | |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 777 | if (!(opt & WGET_OPT_QUIET)) |
Glenn L McGrath | 1bca5ed | 2000-12-09 08:12:06 +0000 | [diff] [blame] | 778 | progressmeter(-1); |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 779 | |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 780 | if (chunked) |
| 781 | goto get_clen; |
| 782 | |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 783 | /* Loops only if chunked */ |
| 784 | while (1) { |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 785 | while (content_len > 0 || !got_clen) { |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 786 | int n; |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 787 | unsigned rdsz = sizeof(buf); |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 788 | |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 789 | if (content_len < sizeof(buf) && (chunked || got_clen)) |
| 790 | rdsz = (unsigned)content_len; |
Denis Vlasenko | 12d2129 | 2007-06-27 21:40:07 +0000 | [diff] [blame] | 791 | n = safe_fread(buf, rdsz, dfp); |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 792 | if (n <= 0) { |
| 793 | if (ferror(dfp)) { |
| 794 | /* perror will not work: ferror doesn't set errno */ |
| 795 | bb_error_msg_and_die(bb_msg_read_error); |
| 796 | } |
Denis Vlasenko | 3526a13 | 2006-09-09 12:20:57 +0000 | [diff] [blame] | 797 | break; |
Glenn L McGrath | 83e4a5b | 2003-08-28 21:55:22 +0000 | [diff] [blame] | 798 | } |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 799 | xwrite(output_fd, buf, n); |
Denis Vlasenko | 9cade08 | 2006-11-21 10:43:02 +0000 | [diff] [blame] | 800 | #if ENABLE_FEATURE_WGET_STATUSBAR |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 801 | transferred += n; |
Eric Andersen | b520e08 | 2000-10-03 00:21:45 +0000 | [diff] [blame] | 802 | #endif |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 803 | if (got_clen) |
Denis Vlasenko | f8aa109 | 2006-10-01 10:58:54 +0000 | [diff] [blame] | 804 | content_len -= n; |
Glenn L McGrath | 83e4a5b | 2003-08-28 21:55:22 +0000 | [diff] [blame] | 805 | } |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 806 | |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 807 | if (!chunked) |
| 808 | break; |
Eric Andersen | 6d7fa43 | 2001-04-10 18:17:05 +0000 | [diff] [blame] | 809 | |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 810 | safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */ |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 811 | get_clen: |
Denis Vlasenko | 06783a5 | 2007-09-24 13:51:54 +0000 | [diff] [blame] | 812 | safe_fgets(buf, sizeof(buf), dfp); |
| 813 | content_len = STRTOOFF(buf, NULL, 16); |
| 814 | /* FIXME: error check? */ |
| 815 | if (content_len == 0) |
| 816 | break; /* all done! */ |
| 817 | } |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 818 | |
Denis Vlasenko | a552eeb | 2006-09-26 09:22:12 +0000 | [diff] [blame] | 819 | if (!(opt & WGET_OPT_QUIET)) |
Denis Vlasenko | a7ce207 | 2007-09-24 18:25:08 +0000 | [diff] [blame] | 820 | progressmeter(0); |
Rob Landley | 19a3940 | 2006-06-13 17:10:26 +0000 | [diff] [blame] | 821 | |
Glenn L McGrath | f1c4b11 | 2004-02-22 00:27:34 +0000 | [diff] [blame] | 822 | if ((use_proxy == 0) && target.is_ftp) { |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 823 | fclose(dfp); |
| 824 | if (ftpcmd(NULL, NULL, sfp, buf) != 226) |
Manuel Novoa III | cad5364 | 2003-03-19 09:13:01 +0000 | [diff] [blame] | 825 | bb_error_msg_and_die("ftp error: %s", buf+4); |
Eric Andersen | 79757c9 | 2001-04-05 21:45:54 +0000 | [diff] [blame] | 826 | ftpcmd("QUIT", NULL, sfp, buf); |
| 827 | } |
Denis Vlasenko | 7710563 | 2007-09-24 15:04:00 +0000 | [diff] [blame] | 828 | |
| 829 | return EXIT_SUCCESS; |
Eric Andersen | 9670083 | 2000-09-04 15:15:55 +0000 | [diff] [blame] | 830 | } |