New --set-content-location functionality...

I got tired of not knowing where all my files came from,
I thought that my shell experience could be a little richer,
and so I wrote this patch to do some information capture for me.

When you use --set-content-location, wget will add an extended attribute
onto all of your downloaded files telling you where the file came from.
The extended attribute is called user.Content-Location, and it will
naturally only be created on file-systems that support extended attributes.

This patch can be broken down into 4 parts:
 - `./configure --enable-xattr' option:  configure.in INSTALL NEWS
 - `wget --set-content-location' option: options.h init.c main.c
 - core functionality:                   http.c ftp.c ftp-ls.c convert.c
 - documentation:                        wget.texi sample.wgetrc

I think that the ChangeLog entries should look something like this:

2005-08-13  Anjan Samanta  <[email protected]>
	New --set-content-location functionality.
	* NEWS: New --set-content-location flag.
	* INSTALL: New --enable-xattr option to ./configure.
	* configure.in: New --enable-xattr option checks for extended file
	attribute functionality.

2005-08-13  Anjan Samanta  <[email protected]>
	New --set-content-location functionality.
	* options.h (struct options): New flag opt.content_location_xattr.
	* init.c (commands): Added setcontentlocation.
	* main.c (option_data): Added set-content-location/setcontentlocation.
	(print_help): Added --set-content-location flag.
	(main) Now unlink()-s opt.output_document if we were going to truncate
	it anyway, in order to have a nice clean file to set xattrs on.

	New --set-content-location functionality.
	* http.c (gethttp): Compute and set user.Content-Location xattr.
	* ftp.c (getftp, ftp_retrieve_list): Set user.Content-Location xattr.
	* ftp-ls (ftp_index): Set user.Content-Location xattr on symlink.
	* convert.c (convert_links): Transfer user.Content-Location xattr
	from the unlink()-d file to the file that replaces it.

2005-08-13  Anjan Samanta  <[email protected]>
	New --set-content-location functionality.
	* wget.texi (Download Options): Added --set-content-location flag.
	* sample.wgetrc: Added set-content-location flag.

Here's the patch (the output of: svn diff):

Index: configure.in
===================================================================
--- configure.in	(revision 2052)
+++ configure.in	(working copy)
@@ -447,6 +447,30 @@
 dnl internationalization macros
 WGET_WITH_NLS
 
+
+AC_ARG_ENABLE(xattr,
+[  --disable-xattr         disable support for extended attributes on files],
+ENABLE_XATTR=$enableval, ENABLE_XATTR=yes)
+if test x"${ENABLE_XATTR}" = xyes; then
+  AC_CHECK_HEADER([sys/xattr.h], [], [
+    AC_MSG_NOTICE([Disabling xattr support; your system does not provide sys/xattr.h])
+    ENABLE_XATTR=no
+  ])
+else
+  AC_MSG_NOTICE([Skipping support for xattr])
+fi
+if test x"${ENABLE_XATTR}" = xyes; then
+  AC_CHECK_FUNCS(fsetxattr lsetxattr getxattr, [], [
+    AC_MSG_NOTICE([Disabling xattr support; cannot find getxattr/setxattr(2) implementation])
+    ENABLE_XATTR=no
+  ])
+fi
+if test x"${ENABLE_XATTR}" = xyes; then
+  AC_DEFINE([ENABLE_XATTR], 1, [Define if you want to enable xattr functionality.])
+  AC_MSG_NOTICE([Enabling support for xattr])
+fi
+
+
 dnl
 dnl Find makeinfo.  We used to provide support for Emacs processing
 dnl Texinfo using `emacs -batch -eval ...' where makeinfo is
Index: src/options.h
===================================================================
--- src/options.h	(revision 2052)
+++ src/options.h	(working copy)
@@ -220,6 +220,12 @@
     prefer_none
   } prefer_family;		/* preferred address family when more
 				   than one type is available */
+
+#ifdef ENABLE_XATTR
+  bool content_location_xattr;  /* whether to try to set an extended file
+				   attribute, user.Content-Type, on the files
+				   downloaded.  */
+#endif
 };
 
 extern struct options opt;
Index: src/init.c
===================================================================
--- src/init.c	(revision 2052)
+++ src/init.c	(working copy)
@@ -224,6 +224,9 @@
   { "secureprotocol",	&opt.secure_protocol,	cmd_spec_secure_protocol },
 #endif
   { "serverresponse",	&opt.server_response,	cmd_boolean },
+#ifdef ENABLE_XATTR
+  { "setcontentlocation", &opt.content_location_xattr, cmd_boolean },
+#endif
   { "spanhosts",	&opt.spanhost,		cmd_boolean },
   { "spider",		&opt.spider,		cmd_boolean },
   { "strictcomments",	&opt.strict_comments,	cmd_boolean },
Index: src/http.c
===================================================================
--- src/http.c	(revision 2052)
+++ src/http.c	(working copy)
@@ -39,6 +39,9 @@
 #include <errno.h>
 #include <time.h>
 #include <locale.h>
+#ifdef ENABLE_XATTR
+# include <sys/xattr.h>
+#endif
 
 #include "wget.h"
 #include "http.h"
@@ -1141,6 +1144,7 @@
   struct request *req;
 
   char *type;
+  char *content_location = 0;
   char *user, *passwd;
   char *proxyauth;
   int statcode;
@@ -1717,6 +1721,15 @@
 	  *tmp = '\0';
 	}
     }
+  content_location = resp_header_strdup (resp, "Content-Location");
+  if (content_location)
+    {
+      char *tmp = uri_merge (u->url, content_location);
+      xfree_null (content_location);
+      content_location = tmp;
+    }
+  else
+    content_location = xstrdup (u->url);
   hs->newloc = resp_header_strdup (resp, "Location");
   hs->remote_time = resp_header_strdup (resp, "Last-Modified");
 
@@ -1772,6 +1785,7 @@
 	  else
 	    CLOSE_INVALIDATE (sock);
 	  xfree_null (type);
+	  xfree_null (content_location);
 	  return NEWLOCATION;
 	}
     }
@@ -1830,6 +1844,7 @@
       /* Mark as successfully retrieved. */
       *dt |= RETROKF;
       xfree_null (type);
+      xfree_null (content_location);
       CLOSE_INVALIDATE (sock);	/* would be CLOSE_FINISH, but there
 				   might be more bytes in the body. */
       return RETRUNNEEDED;
@@ -1840,6 +1855,7 @@
       /* The Range request was somehow misunderstood by the server.
 	 Bail out.  */
       xfree_null (type);
+      xfree_null (content_location);
       CLOSE_INVALIDATE (sock);
       return RANGEERR;
     }
@@ -1889,6 +1905,7 @@
       hs->len = 0;
       hs->res = 0;
       xfree_null (type);
+      xfree_null (content_location);
       if (head_only)
 	/* Pre-1.10 Wget used CLOSE_INVALIDATE here.  Now we trust the
 	   servers not to send body in response to a HEAD request, and
@@ -1911,12 +1928,12 @@
       if (opt.backups)
 	rotate_backups (*hs->local_file);
       if (hs->restval)
-	fp = fopen (*hs->local_file, "ab");
+	fp = fopen (*hs->local_file, "ab"); /* xattr set below */
       else if (ALLOW_CLOBBER)
-	fp = fopen (*hs->local_file, "wb");
+	fp = fopen (*hs->local_file, "wb"); /* xattr set below */
       else
 	{
-	  fp = fopen_excl (*hs->local_file, true);
+	  fp = fopen_excl (*hs->local_file, true); /* xattr set below */
 	  if (!fp && errno == EEXIST)
 	    {
 	      /* We cannot just invent a new name and use it (which is
@@ -1940,6 +1957,18 @@
   else
     fp = output_stream;
 
+#ifdef ENABLE_XATTR
+  if (opt.content_location_xattr)
+    {
+      if (fsetxattr (fileno (fp), "user.Content-Location", content_location,
+                     strlen (content_location), XATTR_CREATE) != 0)
+	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
+    }
+#endif
+
+  xfree_null (content_location);
+  content_location = NULL;	/* We don't need it any more.  */
+
   /* #### This confuses the timestamping code that checks for file
      size.  Maybe we should save some additional information?  */
   if (opt.save_headers)
Index: src/ftp.c
===================================================================
--- src/ftp.c	(revision 2052)
+++ src/ftp.c	(working copy)
@@ -37,6 +37,9 @@
 #endif
 #include <assert.h>
 #include <errno.h>
+#ifdef ENABLE_XATTR
+# include <sys/xattr.h>
+#endif
 
 #include "wget.h"
 #include "utils.h"
@@ -914,13 +917,13 @@
 	rotate_backups (con->target);
 
       if (restval)
-	fp = fopen (con->target, "ab");
+	fp = fopen (con->target, "ab"); /* xattr set below */
       else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
 	       || opt.output_document)
-	fp = fopen (con->target, "wb");
+	fp = fopen (con->target, "wb"); /* xattr set below */
       else
 	{
-	  fp = fopen_excl (con->target, true);
+	  fp = fopen_excl (con->target, true); /* xattr set below */
 	  if (!fp && errno == EEXIST)
 	    {
 	      /* We cannot just invent a new name and use it (which is
@@ -949,6 +952,15 @@
   else
     fp = output_stream;
 
+#ifdef ENABLE_XATTR
+  if (opt.content_location_xattr)
+    {
+      if (fsetxattr (fileno (fp), "user.Content-Location",
+                     u->url, strlen (u->url), XATTR_CREATE) != 0)
+	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
+    }
+#endif
+
   if (*len)
     {
       print_length (*len, restval, true);
@@ -1492,6 +1504,11 @@
 		  unlink (con->target);
 		  if (symlink (f->linkto, con->target) == -1)
 		    logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
+#ifdef ENABLE_XATTR
+		  if (opt.content_location_xattr)
+		    lsetxattr (con->target, "user.Content-Location",
+                               u->url, strlen (u->url), 0);
+#endif /* ENABLE_XATTR */
 		  logputs (LOG_VERBOSE, "\n");
 		} /* have f->linkto */
 #else  /* not HAVE_SYMLINK */
Index: src/ftp-ls.c
===================================================================
--- src/ftp-ls.c	(revision 2052)
+++ src/ftp-ls.c	(working copy)
@@ -37,6 +37,9 @@
 #endif
 #include <errno.h>
 #include <time.h>
+#ifdef ENABLE_XATTR
+# include <sys/xattr.h>
+#endif
 
 #include "wget.h"
 #include "utils.h"
@@ -857,7 +860,7 @@
 
   if (!output_stream)
     {
-      fp = fopen (file, "wb");
+      fp = fopen (file, "wb"); /* xattr set below */
       if (!fp)
 	{
 	  logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
@@ -866,6 +869,16 @@
     }
   else
     fp = output_stream;
+
+#ifdef ENABLE_XATTR
+  if (opt.content_location_xattr)
+    {
+      if (fsetxattr (fileno (fp), "user.Content-Location",
+                     u->url, strlen (u->url), XATTR_CREATE) != 0)
+	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
+    }
+#endif
+
   if (u->user)
     {
       char *tmpu, *tmpp;        /* temporary, clean user and passwd */
Index: src/convert.c
===================================================================
--- src/convert.c	(revision 2052)
+++ src/convert.c	(working copy)
@@ -37,6 +37,9 @@
 #endif /* HAVE_UNISTD_H */
 #include <errno.h>
 #include <assert.h>
+#ifdef ENABLE_XATTR
+# include <sys/xattr.h>
+#endif
 
 #include "wget.h"
 #include "convert.h"
@@ -189,6 +192,7 @@
   FILE *fp;
   const char *p;
   downloaded_file_t downloaded_file_return;
+  char *content_location = 0;
 
   struct urlpos *link;
   int to_url_count = 0, to_file_count = 0;
@@ -223,6 +227,25 @@
   if (opt.backup_converted && downloaded_file_return)
     write_backup_file (file, downloaded_file_return);
 
+#ifdef ENABLE_XATTR
+  if (opt.content_location_xattr)
+    {
+      ssize_t tmp = getxattr (file, "user.Content-Location", 0, 0);
+      if (tmp >= 0)
+	{
+	  content_location = xmalloc (tmp + 1);
+	  tmp = getxattr (file, "user.Content-Location", content_location, tmp);
+	  if (tmp >= 0)
+	    content_location[tmp] = '\0';
+	  else
+	    {
+	      xfree_null (content_location);
+	      content_location = 0;
+	    }
+	}
+    }
+#endif
+
   /* Before opening the file for writing, unlink the file.  This is
      important if the data in FM is mmaped.  In such case, nulling the
      file, which is what fopen() below does, would make us read all
@@ -232,6 +255,7 @@
       logprintf (LOG_NOTQUIET, _("Unable to delete `%s': %s\n"),
 		 file, strerror (errno));
       read_file_free (fm);
+      xfree_null (content_location);
       return;
     }
   /* Now open the file for writing.  */
@@ -241,9 +265,21 @@
       logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
 		 file, strerror (errno));
       read_file_free (fm);
+      xfree_null (content_location);
       return;
     }
+#ifdef ENABLE_XATTR
+  else
+    {
+      if (opt.content_location_xattr && content_location)
+	fsetxattr (fileno (fp), "user.Content-Location",
+                   content_location, strlen (content_location), 0);
+    }
+#endif
 
+  xfree_null (content_location);
+  content_location = 0;
+
   /* Here we loop through all the URLs in file, replacing those of
      them that are downloaded with relative references.  */
   p = fm->content;
Index: src/main.c
===================================================================
--- src/main.c	(revision 2052)
+++ src/main.c	(working copy)
@@ -227,6 +227,9 @@
     { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
     { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 },
     { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
+#ifdef ENABLE_XATTR
+    { "set-content-location", 0, OPT_BOOLEAN, "setcontentlocation", -1 },
+#endif
     { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
     { "spider", 0, OPT_BOOLEAN, "spider", -1 },
     { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
@@ -462,6 +465,11 @@
        --user=USER               set both ftp and http user to USER.\n"),
     N_("\
        --password=PASS           set both ftp and http password to PASS.\n"),
+#ifdef ENABLE_XATTR
+    N_("\
+       --set-content-location    set an extended file attribute,\n\
+                                 user.Content-Location, on downloaded files.\n"),
+#endif
     "\n",
 
     N_("\
@@ -897,8 +905,11 @@
       else
 	{
 	  struct_fstat st;
+	  if (opt.always_rest == 0)
+	    unlink (opt.output_document); /* nuke problematic state, if any */
 	  output_stream = fopen (opt.output_document,
 				 opt.always_rest ? "ab" : "wb");
+	  /* xattr set just before stream usage */
 	  if (output_stream == NULL)
 	    {
 	      perror (opt.output_document);
Index: doc/sample.wgetrc
===================================================================
--- doc/sample.wgetrc	(revision 2052)
+++ doc/sample.wgetrc	(working copy)
@@ -58,6 +58,12 @@
 ## are doing before doing so.
 ##
 
+# Set this to on to put an extended attribute called
+# user.Content-Location on all your downloaded files containing the url
+# of where the file was downloaded from. (This option has no effect if
+# your filesystem does not support extended attributes.)
+#set-content-location = off
+
 # Set this to on to use timestamping by default:
 #timestamping = off
 
Index: doc/wget.texi
===================================================================
--- doc/wget.texi	(revision 2052)
+++ doc/wget.texi	(working copy)
@@ -969,6 +969,13 @@
 using the @samp{--ftp-user} and @samp{--ftp-password} options for 
 @sc{ftp} connections and the @samp{--http-user} and @samp{--http-password} 
 options for @sc{http} connections.
+
+@cindex Content-Location, capture
+@cindex extended file attributes
+@item --set-content-location
+On filesystems that support it, add onto every downloaded file, an
+extended attribute called user.Content-Location, containing the url of
+where the file was downloaded from.
 @end table
 
 @node Directory Options
Index: INSTALL
===================================================================
--- INSTALL	(revision 2052)
+++ INSTALL	(working copy)
@@ -98,6 +98,7 @@
   --disable-largefile     omit support for large files
   --disable-ipv6          disable IPv6 support
   --disable-rpath         do not hardcode runtime library paths
+  --disable-xattr         disable support for extended attributes on files
 
 You can inspect decisions made by configure by editing the generated
 Makefiles and the `src/config.h' include file.  The defaults should
Index: NEWS
===================================================================
--- NEWS	(revision 2052)
+++ NEWS	(working copy)
@@ -7,6 +7,12 @@
 
 * Changes in Wget 1.11.
 
+** The new function `--set-content-location' makes Wget put an
+extended attribute called `user.Content-Location' on all downloaded
+files containing the url of where the file was downloaded file. (This
+option has no effect if your filesystem does not support extended
+attributes.)
+
 ** The new function `--ignore-case' makes Wget ignore case when
 matching files, directories, and wildcards.  This affects the -X, -I,
 -A, and -R options, as well as globbing in FTP URLs.