--output_filter - filtering fetched pages

Hello.


As I wrote to wget maillist, I needed to modify urls in html pages after
fetching it (that was about converting javascript links to normal urls). I
created this simple patch to add the functionality. Command-line parameter
--output_filter=CMD allows to specify filtering program (shell command,
actually), which is run after saving each fetched file. The file name is
single-quoted and added to the end of command.

Obviously, this patch is not the best way to achieve filtering capabilities.
I think, better way to do this is to spawn filter process and send data to
its stdin, reading from stdout - this will allow filtering fetched data
without saving it to files.


I am interested to know, what do wget developers think about those filtering
features? My opinion is that ability to specify filters for pages and/or
urls will improve wget functionality without making it heavier (as regexp
support would make). I browsed maillist archives and found that there were
feature requests for url filters. I think I can work on it when time
permits.


The primitive patch goes below.


-- 
WBR, Sergey Martynoff



diff -urbBw wget-1.10.1/src/http.c wget-1.10.1.dev/src/http.c
--- wget-1.10.1/src/http.c	Tue Aug  9 02:54:16 2005
+++ wget-1.10.1.dev/src/http.c	Thu Sep  1 14:49:08 2005
@@ -1127,6 +1127,26 @@
 #define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping
\
 		       || opt.dirstruct || opt.output_document)
 
+		       
+/* Run external filtering program */
+static void
+apply_filter(const char * fname)
+{
+  int maxlen = strlen (opt.output_filter) + 2 + strlen (fname) + 2;
+  char *cmd = (char *)alloca (maxlen);
+  int rc;
+
+  sprintf (cmd, "%s '%s'", opt.output_filter, fname);
+  logprintf (LOG_NOTQUIET,
+	     _("Running external filter (%s)\n"), cmd);
+  rc = system (cmd);
+
+  if (rc < 0)
+    logprintf (LOG_NONVERBOSE,
+	       _("Failed to execute external filter (rc=%d)\n"), rc );
+}
+
+
 /* Retrieve a document through HTTP protocol.  It recognizes status
    code, and correctly handles redirections.  It closes the network
    socket.  If it receives an error from the functions below it, it
@@ -1979,6 +1999,8 @@
   }
   if (hs->res == -2)
     return FWRITEERR;
+  if (!output_stream && opt.output_filter)
+    apply_filter( *hs->local_file );  /* apply document filter */
   return RETRFINISHED;
 }
 
diff -urbBw wget-1.10.1/src/init.c wget-1.10.1.dev/src/init.c
--- wget-1.10.1/src/init.c	Tue Aug  9 02:54:16 2005
+++ wget-1.10.1.dev/src/init.c	Wed Aug 31 14:04:37 2005
@@ -191,6 +191,7 @@
   { "noproxy",		&opt.no_proxy,		cmd_vector },
   { "numtries",		&opt.ntry,		cmd_number_inf },/*
deprecated*/
   { "outputdocument",	&opt.output_document,	cmd_file },
+  { "outputfilter",	&opt.output_filter,	cmd_string },
   { "pagerequisites",	&opt.page_requisites,	cmd_boolean },
   { "passiveftp",	&opt.ftp_pasv,		cmd_lockable_boolean },
   { "passwd",	        &opt.ftp_passwd,	cmd_string },/* deprecated*/
@@ -1504,6 +1505,7 @@
   xfree_null (opt.dir_prefix);
   xfree_null (opt.input_filename);
   xfree_null (opt.output_document);
+  xfree_null (opt.output_filter);
   free_vec (opt.accepts);
   free_vec (opt.rejects);
   free_vec (opt.excludes);
diff -urbBw wget-1.10.1/src/main.c wget-1.10.1.dev/src/main.c
--- wget-1.10.1/src/main.c	Fri Jul  1 05:20:30 2005
+++ wget-1.10.1.dev/src/main.c	Wed Aug 31 14:08:05 2005
@@ -218,6 +218,7 @@
     { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 },
     { "output-document", 'O', OPT_VALUE, "outputdocument", -1 },
     { "output-file", 'o', OPT_VALUE, "logfile", -1 },
+    { "output-filter", 0, OPT_VALUE, "outputfilter", -1 },
     { "page-requisites", 'p', OPT_BOOLEAN, "pagerequisites", -1 },
     { "parent", 0, OPT__PARENT, NULL, optional_argument },
     { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 },
@@ -430,6 +431,8 @@
        --retry-connrefused       retry even if connection is refused.\n"),
     N_("\
   -O,  --output-document=FILE    write documents to FILE.\n"),
+    N_("\
+       --output-filter=CMD       pass documents through CMD filter.\n"),
     N_("\
   -nc, --no-clobber              skip downloads that would download to\n\
                                  existing files.\n"),
diff -urbBw wget-1.10.1/src/options.h wget-1.10.1.dev/src/options.h
--- wget-1.10.1/src/options.h	Tue Aug  9 02:54:16 2005
+++ wget-1.10.1.dev/src/options.h	Wed Aug 31 14:01:13 2005
@@ -76,6 +76,8 @@
 				   FTP. */
   char *output_document;	/* The output file to which the
 				   documents will be printed.  */
+  char *output_filter;		/* The filter program to which the documents
+				   will be passed just after retrieving. */
 
   char *user;			/* Generic username */
   char *passwd;			/* Generic password */