Re: New zlib patch (supports gzip encoding)

Index: configure.in
===================================================================
RCS file: /pack/anoncvs/wget/configure.in,v
retrieving revision 1.81
diff -u -r1.81 configure.in
--- configure.in	2005/04/26 21:41:33	1.81
+++ configure.in	2005/04/29 17:39:16
@@ -245,6 +245,29 @@
 dnl Checks for libraries.
 dnl
 
+dnl $with_zlib
+AC_ARG_WITH(zlib,
+[  --with-zlib[[=DIR]]       use libz in DIR],[
+  if test "$withval" != "no" -a "$withval" != "yes"; then
+    Z_DIR=$withval
+    CPPFLAGS="${CPPFLAGS} -I$withval/include"
+    LDFLAGS="${LDFLAGS} -L$withval/lib"
+  fi
+])
+if test "$with_zlib" = "no"; then
+    echo "Disabling compression support"
+else
+    AC_CHECK_HEADERS(zlib.h,
+	AC_CHECK_LIB(z, gzopen,[
+	    AC_DEFINE([HAVE_ZLIB], [], [Have compression library])
+	    if test "x${Z_DIR}" != "x"; then
+		CPPFLAGS="${CPPFLAGS} -I${Z_DIR}/include"
+		LIBS="${LIBS} -L${Z_DIR}/lib -lz"
+	    else
+		LIBS="-lz"
+	    fi]))
+fi
+
 dnl $with_ssl can be one of:
 dnl  - empty string or "auto", meaning autodetect SSL and use it if found.
 dnl  - "yes", meaning link with SSL or bail out.
Index: src/http.c
===================================================================
RCS file: /pack/anoncvs/wget/src/http.c,v
retrieving revision 1.173
diff -u -r1.173 http.c
--- src/http.c	2005/04/28 13:56:31	1.173
+++ src/http.c	2005/04/29 17:39:17
@@ -74,6 +74,9 @@
 # include "gen-md5.h"
 #endif
 #include "convert.h"
+#ifdef HAVE_ZLIB
+# include "zlib.h"
+#endif
 
 extern char *version_string;
 extern LARGE_INT total_downloaded_bytes;
@@ -1107,6 +1110,9 @@
 static int known_authentication_scheme_p PARAMS ((const char *, const char *));
 
 time_t http_atotm PARAMS ((const char *));
+#ifdef HAVE_ZLIB
+static int zlib_decomp PARAMS ((const char *)); 
+#endif
 
 #define BEGINS_WITH(line, string_constant)				\
   (!strncasecmp (line, string_constant, sizeof (string_constant) - 1)	\
@@ -1177,6 +1183,9 @@
   int inhibit_keep_alive =
     !opt.http_keep_alive || opt.ignore_length || proxy != NULL;
 
+  /* Content encoding */
+  char *encoding;
+
   /* Headers sent when using POST. */
   wgint post_data_size = 0;
 
@@ -1239,6 +1248,11 @@
   SET_USER_AGENT (req);
   request_set_header (req, "Accept", "*/*", rel_none);
 
+#ifdef HAVE_ZLIB
+  if (opt.compress)
+    request_set_header (req, "Accept-Encoding", "gzip", rel_none);
+#endif
+
   /* Find the username and password for authentication. */
   user = u->user;
   passwd = u->passwd;
@@ -1374,6 +1388,7 @@
      for the Digest authorization scheme.)  */
 
   keep_alive = 0;
+  encoding = 0;
 
   /* Establish the connection.  */
 
@@ -1704,6 +1719,9 @@
 	  *tmp = '\0';
 	}
     }
+
+  encoding = resp_header_strdup (resp, "Content-Encoding");
+
   hs->newloc = resp_header_strdup (resp, "Location");
   hs->remote_time = resp_header_strdup (resp, "Last-Modified");
 
@@ -1960,6 +1978,19 @@
   }
   if (hs->res == -2)
     return FWRITEERR;
+
+  /* now that we have the file, if we are an compressed response, go
+     ahead and decompress */
+#ifdef HAVE_ZLIB
+  if (encoding && ((strncasecmp(encoding, "gzip", 4)) == 0))
+  {
+    /* we don't check the return here, since failure to decompress
+       will be logged and recursion will stop since it leaves no
+       text file to parse for URLs */
+    zlib_decomp(*hs->local_file); 
+  }
+#endif
+
   return RETRFINISHED;
 }
 
@@ -2953,6 +2984,77 @@
       abort ();
     }
 }
+
+#ifdef HAVE_ZLIB
+/* real simple zlib decompression of a file, a buffered version would
+   probably speed things up */
+static int 
+zlib_decomp (const char * fname) 
+{
+
+  int rc = -1; /* pre set the error condition */
+  gzFile * fp = NULL;
+  FILE * ofp = NULL;
+  char * fname_gz = NULL;
+  char c = '\0';
+  size_t dest_size = strlen(fname) + strlen (".gz");
+
+  /* first create  a '.gz' version of the filename and move
+     the original there */
+  fname_gz = (char *) xmalloc (dest_size + 1);
+  memset (fname_gz,0,dest_size + 1);
+  strncpy (fname_gz, fname, dest_size);
+  if (fname_gz) 
+  {
+
+    strncat (fname_gz, ".gz", 3);
+    if ((rename (fname, fname_gz)) < 0)
+    {
+      logprintf (LOG_NOTQUIET, "%s->%s: %s\n", fname, fname_gz, 
+                 strerror (errno));
+      goto failed_zlib_decomp;
+
+    }
+        
+    /* open for reading (binary) */
+    fp = gzopen (fname_gz, "rb");
+    if (!fp)
+    {
+      logprintf (LOG_NOTQUIET, "%s: %s\n", fname_gz, strerror (errno));
+      goto failed_zlib_decomp;
+    }
+
+    /* open for writing (binary) */
+    ofp = fopen (fname, "wb");
+    if (!ofp)
+    {
+      logprintf (LOG_NOTQUIET, "%s: %s\n", fname, strerror (errno));
+      goto failed_zlib_decomp;
+    }
+  
+    /* write the uncompressed version */
+    while ((c = gzgetc (fp)) != -1)
+      fputc (c,ofp);
+
+  } 
+
+  rc = 0;
+  /* fall thru */
+
+failed_zlib_decomp:
+
+  if (fp) { gzclose (fp); }
+  if (ofp) { fclose (ofp); }
+ 
+  if (fname_gz) {
+    unlink (fname_gz);
+    xfree (fname_gz);
+  }
+
+  return rc;
+}
+#endif 
+
 
 void
 save_cookies (void)
Index: src/init.c
===================================================================
RCS file: /pack/anoncvs/wget/src/init.c,v
retrieving revision 1.110
diff -u -r1.110 init.c
--- src/init.c	2005/04/28 09:32:13	1.110
+++ src/init.c	2005/04/29 17:39:17
@@ -131,6 +131,9 @@
   { "certificatetype",	&opt.cert_type,		cmd_cert_type },
   { "checkcertificate", &opt.check_cert,	cmd_boolean },
 #endif
+#ifdef HAVE_ZLIB
+  { "compress",     &opt.compress,      cmd_boolean },
+#endif
   { "connecttimeout",	&opt.connect_timeout,	cmd_time },
   { "continue",		&opt.always_rest,	cmd_boolean },
   { "convertlinks",	&opt.convert_links,	cmd_boolean },
Index: src/main.c
===================================================================
RCS file: /pack/anoncvs/wget/src/main.c,v
retrieving revision 1.134
diff -u -r1.134 main.c
--- src/main.c	2005/04/27 22:08:28	1.134
+++ src/main.c	2005/04/29 17:39:17
@@ -165,6 +165,9 @@
     { IF_SSL ("check-certificate"), 0, OPT_BOOLEAN, "checkcertificate", -1 },
     { "clobber", 0, OPT__CLOBBER, NULL, optional_argument },
     { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 },
+#ifdef HAVE_ZLIB
+    { "compress", 'z', OPT_BOOLEAN, "compress", -1 },
+#endif
     { "continue", 'c', OPT_BOOLEAN, "continue", -1 },
     { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 },
     { "cookies", 0, OPT_BOOLEAN, "cookies", -1 },
@@ -539,6 +542,10 @@
        --post-data=STRING      use the POST method; send STRING as the data.\n"),
     N_("\
        --post-file=FILE        use the POST method; send contents of FILE.\n"),
+#ifdef HAVE_ZLIB
+    N_("\
+  -z   --compress              request gzip http compression\n"),
+#endif
     "\n",
 
 #ifdef HAVE_SSL
Index: src/options.h
===================================================================
RCS file: /pack/anoncvs/wget/src/options.h,v
retrieving revision 1.52
diff -u -r1.52 options.h
--- src/options.h	2005/04/27 21:30:22	1.52
+++ src/options.h	2005/04/29 17:39:17
@@ -213,6 +213,11 @@
   int ipv4_only;		/* IPv4 connections have been requested. */
   int ipv6_only;		/* IPv4 connections have been requested. */
 #endif
+
+#ifdef HAVE_ZLIB
+  int compress;          /* enable gzip compression requests */
+#endif
+
   enum {
     prefer_ipv4,
     prefer_ipv6,