=== modified file 'src/exits.c' --- src/exits.c 2010-09-29 11:34:09 +0000 +++ src/exits.c 2010-12-21 15:35:00 +0000 @@ -77,7 +77,7 @@ case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN: case PROXERR: return WGET_EXIT_SERVER_ERROR; - case URLERROR: case QUOTEXC: case SSLINITFAILED: + case URLERROR: case QUOTEXC: case SSLINITFAILED: case HARDQUOTAREACHED: default: return WGET_EXIT_UNKNOWN; } === modified file 'src/ftp.c' --- src/ftp.c 2010-12-01 12:15:13 +0000 +++ src/ftp.c 2010-12-21 15:36:38 +0000 @@ -978,6 +978,7 @@ con->csock = -1; fd_close (dtsock); fd_close (local_sock); + if (res == HARDQUOTAREACHED) return HARDQUOTAREACHED; return RETRFINISHED; } @@ -1262,8 +1263,17 @@ if (!output_stream || con->cmd & DO_LIST) fclose (fp); + if (res == -3) + { + logprintf (LOG_NOTQUIET, _("Hard quota reached: %s, closing control connection.\n"), + con->target); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + return HARDQUOTAREACHED; + } /* If fd_read_body couldn't write to fp, bail out. */ - if (res == -2) + else if (res == -2) { logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), con->target, strerror (errno)); @@ -1498,6 +1508,7 @@ case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED: case UNLINKERR: + case HARDQUOTAREACHED: /* Fatal errors, give up. */ return err; case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: @@ -1574,6 +1585,7 @@ number of bytes and files downloaded. */ { total_downloaded_bytes += qtyread; + total_downloaded_bytes_exact += qtyread; numurls++; } @@ -1589,6 +1601,7 @@ for instance, may want to know how many bytes and files they've downloaded through it. */ total_downloaded_bytes += qtyread; + total_downloaded_bytes_exact += qtyread; numurls++; if (opt.delete_after && !input_file_url (opt.input_filename)) @@ -1727,6 +1740,11 @@ --depth; return QUOTEXC; } + if (opt.hard_quota && total_downloaded_bytes_exact >= opt.hard_quota) + { + --depth; + return HARDQUOTAREACHED; + } old_target = con->target; ofile = xstrdup (u->file); @@ -1903,7 +1921,7 @@ xfree (ofile); /* Break on fatals. */ - if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR) + if (err == HARDQUOTAREACHED || err == QUOTEXC || err == HOSTERR || err == FWRITEERR) break; con->cmd &= ~ (DO_CWD | DO_LOGIN); f = f->next; @@ -1937,6 +1955,8 @@ if (opt.quota && total_downloaded_bytes > opt.quota) break; + if (opt.hard_quota && total_downloaded_bytes_exact >= opt.hard_quota) + break; if (f->type != FT_DIRECTORY) continue; @@ -1975,17 +1995,21 @@ odir = xstrdup (u->dir); /* because url_set_dir will free u->dir. */ url_set_dir (u, newdir); - ftp_retrieve_glob (u, con, GLOB_GETALL); + uerr_t res=ftp_retrieve_glob (u, con, GLOB_GETALL); url_set_dir (u, odir); xfree (odir); /* Set the time-stamp? */ + + //fatal, bail out. + if(res == HARDQUOTAREACHED)return HARDQUOTAREACHED; } if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; - else - return RETROK; + if (opt.hard_quota && total_downloaded_bytes_exact >= opt.hard_quota) + return HARDQUOTAREACHED; + return RETROK; } /* Return true if S has a leading '/' or contains '../' */ @@ -2114,7 +2138,7 @@ if (start) { /* Just get everything. */ - ftp_retrieve_list (u, start, con); + res = ftp_retrieve_list (u, start, con); } else { @@ -2140,11 +2164,12 @@ */ } freefileinfo (start); + if (res == HARDQUOTAREACHED) + return HARDQUOTAREACHED; if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; - else - /* #### Should we return `res' here? */ - return RETROK; + /* #### Should we return `res' here? */ + return RETROK; } /* The wrapper that calls an appropriate routine according to contents === modified file 'src/http.c' --- src/http.c 2010-12-01 12:15:13 +0000 +++ src/http.c 2010-12-21 15:37:45 +0000 @@ -2570,7 +2570,12 @@ else { if (hs->res < 0) - hs->rderrmsg = xstrdup (fd_errstr (sock)); + { + if (hs->res == -3) + hs->rderrmsg = xstrdup (_("Hard quota reached")); + else + hs->rderrmsg = xstrdup (fd_errstr (sock)); + } CLOSE_INVALIDATE (sock); } @@ -2578,6 +2583,11 @@ fclose (fp); if (hs->res == -2) return FWRITEERR; + if (hs->res == -3) + { + logprintf (LOG_NOTQUIET, "%s\n", hs->rderrmsg); + return HARDQUOTAREACHED; + } return RETRFINISHED; } @@ -2779,7 +2789,7 @@ logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"), quote (hstat.local_file), strerror (errno)); case HOSTERR: case CONIMPOSSIBLE: case PROXERR: case AUTHFAILED: - case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR: + case SSLINITFAILED: case CONTNOTSUPPORTED: case VERIFCERTERR: case HARDQUOTAREACHED: /* Fatal errors just return from the function. */ ret = err; goto exit; @@ -3036,6 +3046,7 @@ } ++numurls; total_downloaded_bytes += hstat.rd_size; + total_downloaded_bytes_exact += hstat.rd_size; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -3069,6 +3080,7 @@ } ++numurls; total_downloaded_bytes += hstat.rd_size; + total_downloaded_bytes_exact += hstat.rd_size; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -3101,8 +3113,34 @@ goto exit; } } - else /* from now on hstat.res can only be -1 */ + else /* from now on hstat.res can only be -1 or -3 */ { + //start of -3 handling + if (hstat.res == -3) + { + if (hstat.contlen == -1) + { + logprintf (LOG_VERBOSE, + _("%s (%s) - Interrupting at byte %s (%s).\n"), + tms, tmrate, number_to_static_string (hstat.len), + hstat.rderrmsg); + ret = HARDQUOTAREACHED; + break; + } + else /* hstat.res == -3 and contlen is given */ + { + logprintf (LOG_VERBOSE, + _("%s (%s) - Interrupting at byte %s/%s (%s).\n"), + tms, tmrate, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), + hstat.rderrmsg); + ret = HARDQUOTAREACHED; + break; + } + } + //end of -3 handling + if (hstat.contlen == -1) { logprintf (LOG_VERBOSE, === modified file 'src/init.c' --- src/init.c 2010-12-01 12:15:13 +0000 +++ src/init.c 2010-12-10 01:52:56 +0000 @@ -172,6 +172,7 @@ #endif /* def __VMS */ { "ftpuser", &opt.ftp_user, cmd_string }, { "glob", &opt.ftp_glob, cmd_boolean }, + { "hardquota", &opt.hard_quota, cmd_bytes_sum }, { "header", NULL, cmd_spec_header }, { "htmlextension", &opt.adjust_extension, cmd_boolean }, /* deprecated */ { "htmlify", NULL, cmd_spec_htmlify }, === modified file 'src/main.c' --- src/main.c 2010-12-09 12:27:17 +0000 +++ src/main.c 2010-12-21 15:40:01 +0000 @@ -244,6 +244,7 @@ { "proxy-user", 0, OPT_VALUE, "proxyuser", -1 }, { "quiet", 'q', OPT_BOOLEAN, "quiet", -1 }, { "quota", 'Q', OPT_VALUE, "quota", -1 }, + { "hard-quota", 0, OPT_VALUE, "hardquota", -1 }, { "random-file", 0, OPT_VALUE, "randomfile", -1 }, { "random-wait", 0, OPT_BOOLEAN, "randomwait", -1 }, { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, @@ -486,7 +487,9 @@ N_("\ --no-proxy explicitly turn off proxy.\n"), N_("\ - -Q, --quota=NUMBER set retrieval quota to NUMBER.\n"), + -Q, --quota=NUMBER set retrieval soft quota to NUMBER.\n"), + N_("\ + --hard-quota=NUMBER set retrieval hard quota to NUMBER.\n"), N_("\ --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host.\n"), N_("\ @@ -877,6 +880,8 @@ int nurl; bool append_to_log = false; + total_downloaded_bytes = 0; total_downloaded_bytes_exact = 0; + program_name = argv[0]; struct ptimer *timer = ptimer_new (); @@ -1320,6 +1325,8 @@ signal (SIGWINCH, progress_handle_sigwinch); #endif + uerr_t status=RETROK; + /* Retrieve the URLs from argument list. */ for (t = url; *t; t++) { @@ -1351,13 +1358,13 @@ if (url_scheme (*t) == SCHEME_FTP) opt.follow_ftp = 1; - retrieve_tree (url_parsed, NULL); + status = retrieve_tree (url_parsed, NULL); opt.follow_ftp = old_follow_ftp; } else { - retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL, + status = retrieve_url (url_parsed, *t, &filename, &redirected_URL, NULL, &dt, opt.recursive, iri, true); } @@ -1373,10 +1380,11 @@ url_free (url_parsed); } iri_free (iri); + if(status == HARDQUOTAREACHED)break; } /* And then from the input file, if any. */ - if (opt.input_filename) + if (status != HARDQUOTAREACHED && opt.input_filename) { int count; retrieve_from_file (opt.input_filename, opt.force_html, &count); @@ -1420,6 +1428,11 @@ human_readable (opt.quota)); } + if (opt.hard_quota && total_downloaded_bytes_exact > opt.hard_quota) + logprintf (LOG_NOTQUIET, + _("Download hard quota of %s exceeded!\n"), + human_readable (opt.hard_quota)); + if (opt.cookies_output) save_cookies (); === modified file 'src/options.h' --- src/options.h 2010-10-28 22:20:31 +0000 +++ src/options.h 2010-12-10 01:52:56 +0000 @@ -125,6 +125,7 @@ many bps. */ SUM_SIZE_INT quota; /* Maximum file size to download and store. */ + SUM_SIZE_INT hard_quota; /* Maximum traffic (bytes) to download. */ bool server_response; /* Do we print server response? */ bool save_headers; /* Do we save headers together with === modified file 'src/recur.c' --- src/recur.c 2010-12-01 12:15:13 +0000 +++ src/recur.c 2010-12-21 15:40:57 +0000 @@ -232,7 +232,9 @@ if (opt.quota && total_downloaded_bytes > opt.quota) break; - if (status == FWRITEERR) + if (opt.hard_quota && total_downloaded_bytes_exact >= opt.hard_quota) + break; + if (status == FWRITEERR || status == HARDQUOTAREACHED) break; /* Get the next URL from the queue... */ @@ -281,6 +283,7 @@ status = retrieve_url (url_parsed, url, &file, &redirected, referer, &dt, false, i, true); + if(status == HARDQUOTAREACHED)break; if (html_allowed && file && status == RETROK && (dt & RETROKF) && (dt & TEXTHTML)) @@ -468,10 +471,13 @@ if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; - else if (status == FWRITEERR) + if (status == HARDQUOTAREACHED) + return HARDQUOTAREACHED; + if (opt.hard_quota && total_downloaded_bytes_exact > opt.hard_quota) + return HARDQUOTAREACHED; + if (status == FWRITEERR) return FWRITEERR; - else - return RETROK; + return RETROK; } /* Based on the context provided by retrieve_tree, decide whether a === modified file 'src/retr.c' --- src/retr.c 2010-12-01 12:15:13 +0000 +++ src/retr.c 2010-12-21 15:41:52 +0000 @@ -54,7 +54,7 @@ #include "iri.h" /* Total size of downloaded files. Used to enforce quota. */ -SUM_SIZE_INT total_downloaded_bytes; +SUM_SIZE_INT total_downloaded_bytes, total_downloaded_bytes_exact; /* Total download time in seconds. */ double total_download_time; @@ -183,9 +183,9 @@ } /* Read the contents of file descriptor FD until it the connection - terminates or a read error occurs. The data is read in portions of - up to 16K and written to OUT as it arrives. If opt.verbose is set, - the progress is shown. + terminates, a read error occurs, or traffic hard quota is exceeded. + The data is read in portions of up to 16K and written to OUT as it + arrives. If opt.verbose is set, the progress is shown. TOREAD is the amount of data expected to arrive, normally only used by the progress gauge. @@ -199,7 +199,8 @@ The function exits and returns the amount of data read. In case of error while reading data, -1 is returned. In case of error while - writing data, -2 is returned. */ + writing data, -2 is returned. In case of traffic hard quota reached, + -3 is returned. */ int fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, @@ -238,6 +239,13 @@ if (opt.verbose) { + if (opt.hard_quota) + { + logprintf(LOG_VERBOSE, _("Hard quota: %s (%s).\n"), + number_to_static_string(opt.hard_quota), + human_readable (opt.hard_quota)); + + } /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL argument to progress_create because the indicator doesn't (yet) know about "skipping" data. */ @@ -301,6 +309,14 @@ else rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize; + if (opt.hard_quota) + { + rdsize = MIN (rdsize, opt.hard_quota - total_downloaded_bytes_exact); + if (rdsize < 0) rdsize = 0; + if (rdsize == 0) { ret = -3; goto out; } + } + + if (progress_interactive) { /* For interactive progress gauges, always specify a ~1s @@ -341,6 +357,7 @@ if (ret > 0) { sum_read += ret; + total_downloaded_bytes_exact += ret; if (!write_data (out, dlbuf, ret, &skip, &sum_written)) { ret = -2; === modified file 'src/retr.h' --- src/retr.h 2010-05-08 19:56:15 +0000 +++ src/retr.h 2010-12-10 02:33:10 +0000 @@ -36,6 +36,7 @@ /* These global vars should be made static to retr.c and exported via functions! */ extern SUM_SIZE_INT total_downloaded_bytes; +extern SUM_SIZE_INT total_downloaded_bytes_exact; extern double total_download_time; extern FILE *output_stream; extern bool output_stream_regular; === modified file 'src/wget.h' --- src/wget.h 2010-09-29 11:34:09 +0000 +++ src/wget.h 2010-12-21 15:42:40 +0000 @@ -352,7 +352,7 @@ PROXERR, /* 50 */ AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, - UNLINKERR + UNLINKERR, HARDQUOTAREACHED } uerr_t; /* 2005-02-19 SMS.