Don't attempt to read more than content-length amount of data

This patch fixes the bug described at http://tinyurl.com/8tv59/, in
pretty much the manner described there.  Since it makes the
"kill_longer" wgetrc command unnecessary, it is removed as well.

I will apply the corresponding fix on the 1.10 branch as well.


2005-08-09  Hrvoje Niksic  <[email protected]>

	* http.c (gethttp): Don't read more than the amount of data
	specified by the content-length header.

Index: src/options.h
===================================================================
--- src/options.h	(revision 2008)
+++ src/options.h	(working copy)
@@ -35,9 +35,6 @@
   int ntry;			/* Number of tries per URL */
   bool retry_connrefused;	/* Treat CONNREFUSED as non-fatal. */
   bool background;		/* Whether we should work in background. */
-  bool kill_longer;		/* Do we reject messages with *more*
-				   data than specified in
-				   content-length? */
   bool ignore_length;		/* Do we heed content-length at all?  */
   bool recursive;		/* Are we recursive? */
   bool spanhost;			/* Do we span across hosts in
Index: src/init.c
===================================================================
--- src/init.c	(revision 2009)
+++ src/init.c	(working copy)
@@ -173,7 +173,6 @@
 #endif
   { "input",		&opt.input_filename,	cmd_file },
   { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
-  { "killlonger",	&opt.kill_longer,	cmd_boolean },
   { "limitrate",	&opt.limit_rate,	cmd_bytes },
   { "loadcookies",	&opt.cookies_input,	cmd_file },
   { "logfile",		&opt.lfilename,		cmd_file },
Index: src/http.c
===================================================================
--- src/http.c	(revision 2008)
+++ src/http.c	(working copy)
@@ -1950,7 +1950,10 @@
 
   /* Download the request body.  */
   flags = 0;
-  if (keep_alive)
+  if (contlen != -1)
+    /* If content-length is present, read that much; otherwise, read
+       until EOF.  The HTTP spec doesn't require the server to
+       actually close the connection when it's done sending data. */
     flags |= rb_read_exactly;
   if (hs->restval > 0 && contrange == 0)
     /* If the server ignored our range request, instruct fd_read_body
@@ -2352,9 +2355,7 @@
       if ((tmr != (time_t) (-1))
 	  && !opt.spider
 	  && ((hstat.len == hstat.contlen) ||
-	      ((hstat.res == 0) &&
-	       ((hstat.contlen == -1) ||
-		(hstat.len >= hstat.contlen && !opt.kill_longer)))))
+	      ((hstat.res == 0) && (hstat.contlen == -1))))
 	{
 	  /* #### This code repeats in http.c and ftp.c.  Move it to a
              function!  */
@@ -2450,43 +2451,10 @@
 	      free_hstat (&hstat);
 	      continue;
 	    }
-	  else if (!opt.kill_longer) /* meaning we got more than expected */
-	    {
-	      logprintf (LOG_VERBOSE,
-			 _("%s (%s) - `%s' saved [%s/%s]\n\n"),
-			 tms, tmrate, locf,
-			 number_to_static_string (hstat.len),
-			 number_to_static_string (hstat.contlen));
-	      logprintf (LOG_NONVERBOSE,
-			 "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
-			 tms, u->url,
-			 number_to_static_string (hstat.len),
-			 number_to_static_string (hstat.contlen),
-			 locf, count);
-	      ++opt.numurls;
-	      total_downloaded_bytes += hstat.len;
-
-	      /* Remember that we downloaded the file for later ".orig" code. */
-	      if (*dt & ADDED_HTML_EXTENSION)
-		downloaded_file(FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, locf);
-	      else
-		downloaded_file(FILE_DOWNLOADED_NORMALLY, locf);
-	      
-	      free_hstat (&hstat);
-	      xfree_null (dummy);
-	      return RETROK;
-	    }
-	  else			/* the same, but not accepted */
-	    {
-	      logprintf (LOG_VERBOSE,
-			 _("%s (%s) - Connection closed at byte %s/%s. "),
-			 tms, tmrate,
-			 number_to_static_string (hstat.len),
-			 number_to_static_string (hstat.contlen));
-	      printwhat (count, opt.ntry);
-	      free_hstat (&hstat);
-	      continue;
-	    }
+	  else
+	    /* Getting here would mean reading more data than
+	       requested with content-length, which we never do.  */
+	    abort ();
 	}
       else			/* now hstat.res can only be -1 */
 	{
Index: doc/wget.texi
===================================================================
--- doc/wget.texi	(revision 2008)
+++ doc/wget.texi	(working copy)
@@ -2664,12 +2664,6 @@
 @item input = @var{file}
 Read the @sc{url}s from @var{string}, like @samp{-i @var{file}}.
 
-@item kill_longer = on/off
-Consider data longer than specified in content-length header as invalid
-(and retry getting it).  The default behavior is to save as much data
-as there is, provided there is more than or equal to the value in
-@code{Content-Length}.
-
 @item limit_rate = @var{rate}
 Limit the download speed to no more than @var{rate} bytes per second.
 The same as @samp{--limit-rate=@var{rate}}.