Re: New --set-content-location functionality...

Not much response on this patch so far; so I'm wondering whether it fell into
a hole... Is this kind of feature in principle a thumbs-up, a thumbs-down, or
a needs-thinking-about? I can repost to the general mailing list, if that
would be better...
--anjansamanta

On Saturday 13 August 2005 16:31, Anjan Samanta wrote:
# I got tired of not knowing where all my files came from,
# I thought that my shell experience could be a little richer,
# and so I wrote this patch to do some information capture for me.
# 
# When you use --set-content-location, wget will add an extended attribute
# onto all of your downloaded files telling you where the file came from.
# The extended attribute is called user.Content-Location, and it will
# naturally only be created on file-systems that support extended attributes.
# 
# This patch can be broken down into 4 parts:
#  - `./configure --enable-xattr' option:  configure.in INSTALL NEWS
#  - `wget --set-content-location' option: options.h init.c main.c
#  - core functionality:                   http.c ftp.c ftp-ls.c convert.c
#  - documentation:                        wget.texi sample.wgetrc
# 
# I think that the ChangeLog entries should look something like this:
# 
# 2005-08-13  Anjan Samanta  <[email protected]>
# 	New --set-content-location functionality.
# 	* NEWS: New --set-content-location flag.
# 	* INSTALL: New --enable-xattr option to ./configure.
# 	* configure.in: New --enable-xattr option checks for extended file
# 	attribute functionality.
# 
# 2005-08-13  Anjan Samanta  <[email protected]>
# 	New --set-content-location functionality.
# 	* options.h (struct options): New flag opt.content_location_xattr.
# 	* init.c (commands): Added setcontentlocation.
# 	* main.c (option_data): Added set-content-location/setcontentlocation.
# 	(print_help): Added --set-content-location flag.
# 	(main) Now unlink()-s opt.output_document if we were going to truncate
# 	it anyway, in order to have a nice clean file to set xattrs on.
# 
# 	New --set-content-location functionality.
# 	* http.c (gethttp): Compute and set user.Content-Location xattr.
# 	* ftp.c (getftp, ftp_retrieve_list): Set user.Content-Location xattr.
# 	* ftp-ls (ftp_index): Set user.Content-Location xattr on symlink.
# 	* convert.c (convert_links): Transfer user.Content-Location xattr
# 	from the unlink()-d file to the file that replaces it.
# 
# 2005-08-13  Anjan Samanta  <[email protected]>
# 	New --set-content-location functionality.
# 	* wget.texi (Download Options): Added --set-content-location flag.
# 	* sample.wgetrc: Added set-content-location flag.
# 
# Here's the patch (the output of: svn diff):
# 
# Index: configure.in
# ===================================================================
# --- configure.in	(revision 2052)
# +++ configure.in	(working copy)
# @@ -447,6 +447,30 @@
#  dnl internationalization macros
#  WGET_WITH_NLS
#  
# +
# +AC_ARG_ENABLE(xattr,
# +[  --disable-xattr         disable support for extended attributes on files],
# +ENABLE_XATTR=$enableval, ENABLE_XATTR=yes)
# +if test x"${ENABLE_XATTR}" = xyes; then
# +  AC_CHECK_HEADER([sys/xattr.h], [], [
# +    AC_MSG_NOTICE([Disabling xattr support; your system does not provide sys/xattr.h])
# +    ENABLE_XATTR=no
# +  ])
# +else
# +  AC_MSG_NOTICE([Skipping support for xattr])
# +fi
# +if test x"${ENABLE_XATTR}" = xyes; then
# +  AC_CHECK_FUNCS(fsetxattr lsetxattr getxattr, [], [
# +    AC_MSG_NOTICE([Disabling xattr support; cannot find getxattr/setxattr(2) implementation])
# +    ENABLE_XATTR=no
# +  ])
# +fi
# +if test x"${ENABLE_XATTR}" = xyes; then
# +  AC_DEFINE([ENABLE_XATTR], 1, [Define if you want to enable xattr functionality.])
# +  AC_MSG_NOTICE([Enabling support for xattr])
# +fi
# +
# +
#  dnl
#  dnl Find makeinfo.  We used to provide support for Emacs processing
#  dnl Texinfo using `emacs -batch -eval ...' where makeinfo is
# Index: src/options.h
# ===================================================================
# --- src/options.h	(revision 2052)
# +++ src/options.h	(working copy)
# @@ -220,6 +220,12 @@
#      prefer_none
#    } prefer_family;		/* preferred address family when more
#  				   than one type is available */
# +
# +#ifdef ENABLE_XATTR
# +  bool content_location_xattr;  /* whether to try to set an extended file
# +				   attribute, user.Content-Type, on the files
# +				   downloaded.  */
# +#endif
#  };
#  
#  extern struct options opt;
# Index: src/init.c
# ===================================================================
# --- src/init.c	(revision 2052)
# +++ src/init.c	(working copy)
# @@ -224,6 +224,9 @@
#    { "secureprotocol",	&opt.secure_protocol,	cmd_spec_secure_protocol },
#  #endif
#    { "serverresponse",	&opt.server_response,	cmd_boolean },
# +#ifdef ENABLE_XATTR
# +  { "setcontentlocation", &opt.content_location_xattr, cmd_boolean },
# +#endif
#    { "spanhosts",	&opt.spanhost,		cmd_boolean },
#    { "spider",		&opt.spider,		cmd_boolean },
#    { "strictcomments",	&opt.strict_comments,	cmd_boolean },
# Index: src/http.c
# ===================================================================
# --- src/http.c	(revision 2052)
# +++ src/http.c	(working copy)
# @@ -39,6 +39,9 @@
#  #include <errno.h>
#  #include <time.h>
#  #include <locale.h>
# +#ifdef ENABLE_XATTR
# +# include <sys/xattr.h>
# +#endif
#  
#  #include "wget.h"
#  #include "http.h"
# @@ -1141,6 +1144,7 @@
#    struct request *req;
#  
#    char *type;
# +  char *content_location = 0;
#    char *user, *passwd;
#    char *proxyauth;
#    int statcode;
# @@ -1717,6 +1721,15 @@
#  	  *tmp = '\0';
#  	}
#      }
# +  content_location = resp_header_strdup (resp, "Content-Location");
# +  if (content_location)
# +    {
# +      char *tmp = uri_merge (u->url, content_location);
# +      xfree_null (content_location);
# +      content_location = tmp;
# +    }
# +  else
# +    content_location = xstrdup (u->url);
#    hs->newloc = resp_header_strdup (resp, "Location");
#    hs->remote_time = resp_header_strdup (resp, "Last-Modified");
#  
# @@ -1772,6 +1785,7 @@
#  	  else
#  	    CLOSE_INVALIDATE (sock);
#  	  xfree_null (type);
# +	  xfree_null (content_location);
#  	  return NEWLOCATION;
#  	}
#      }
# @@ -1830,6 +1844,7 @@
#        /* Mark as successfully retrieved. */
#        *dt |= RETROKF;
#        xfree_null (type);
# +      xfree_null (content_location);
#        CLOSE_INVALIDATE (sock);	/* would be CLOSE_FINISH, but there
#  				   might be more bytes in the body. */
#        return RETRUNNEEDED;
# @@ -1840,6 +1855,7 @@
#        /* The Range request was somehow misunderstood by the server.
#  	 Bail out.  */
#        xfree_null (type);
# +      xfree_null (content_location);
#        CLOSE_INVALIDATE (sock);
#        return RANGEERR;
#      }
# @@ -1889,6 +1905,7 @@
#        hs->len = 0;
#        hs->res = 0;
#        xfree_null (type);
# +      xfree_null (content_location);
#        if (head_only)
#  	/* Pre-1.10 Wget used CLOSE_INVALIDATE here.  Now we trust the
#  	   servers not to send body in response to a HEAD request, and
# @@ -1911,12 +1928,12 @@
#        if (opt.backups)
#  	rotate_backups (*hs->local_file);
#        if (hs->restval)
# -	fp = fopen (*hs->local_file, "ab");
# +	fp = fopen (*hs->local_file, "ab"); /* xattr set below */
#        else if (ALLOW_CLOBBER)
# -	fp = fopen (*hs->local_file, "wb");
# +	fp = fopen (*hs->local_file, "wb"); /* xattr set below */
#        else
#  	{
# -	  fp = fopen_excl (*hs->local_file, true);
# +	  fp = fopen_excl (*hs->local_file, true); /* xattr set below */
#  	  if (!fp && errno == EEXIST)
#  	    {
#  	      /* We cannot just invent a new name and use it (which is
# @@ -1940,6 +1957,18 @@
#    else
#      fp = output_stream;
#  
# +#ifdef ENABLE_XATTR
# +  if (opt.content_location_xattr)
# +    {
# +      if (fsetxattr (fileno (fp), "user.Content-Location", content_location,
# +                     strlen (content_location), XATTR_CREATE) != 0)
# +	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
# +    }
# +#endif
# +
# +  xfree_null (content_location);
# +  content_location = NULL;	/* We don't need it any more.  */
# +
#    /* #### This confuses the timestamping code that checks for file
#       size.  Maybe we should save some additional information?  */
#    if (opt.save_headers)
# Index: src/ftp.c
# ===================================================================
# --- src/ftp.c	(revision 2052)
# +++ src/ftp.c	(working copy)
# @@ -37,6 +37,9 @@
#  #endif
#  #include <assert.h>
#  #include <errno.h>
# +#ifdef ENABLE_XATTR
# +# include <sys/xattr.h>
# +#endif
#  
#  #include "wget.h"
#  #include "utils.h"
# @@ -914,13 +917,13 @@
#  	rotate_backups (con->target);
#  
#        if (restval)
# -	fp = fopen (con->target, "ab");
# +	fp = fopen (con->target, "ab"); /* xattr set below */
#        else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
#  	       || opt.output_document)
# -	fp = fopen (con->target, "wb");
# +	fp = fopen (con->target, "wb"); /* xattr set below */
#        else
#  	{
# -	  fp = fopen_excl (con->target, true);
# +	  fp = fopen_excl (con->target, true); /* xattr set below */
#  	  if (!fp && errno == EEXIST)
#  	    {
#  	      /* We cannot just invent a new name and use it (which is
# @@ -949,6 +952,15 @@
#    else
#      fp = output_stream;
#  
# +#ifdef ENABLE_XATTR
# +  if (opt.content_location_xattr)
# +    {
# +      if (fsetxattr (fileno (fp), "user.Content-Location",
# +                     u->url, strlen (u->url), XATTR_CREATE) != 0)
# +	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
# +    }
# +#endif
# +
#    if (*len)
#      {
#        print_length (*len, restval, true);
# @@ -1492,6 +1504,11 @@
#  		  unlink (con->target);
#  		  if (symlink (f->linkto, con->target) == -1)
#  		    logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
# +#ifdef ENABLE_XATTR
# +		  if (opt.content_location_xattr)
# +		    lsetxattr (con->target, "user.Content-Location",
# +                               u->url, strlen (u->url), 0);
# +#endif /* ENABLE_XATTR */
#  		  logputs (LOG_VERBOSE, "\n");
#  		} /* have f->linkto */
#  #else  /* not HAVE_SYMLINK */
# Index: src/ftp-ls.c
# ===================================================================
# --- src/ftp-ls.c	(revision 2052)
# +++ src/ftp-ls.c	(working copy)
# @@ -37,6 +37,9 @@
#  #endif
#  #include <errno.h>
#  #include <time.h>
# +#ifdef ENABLE_XATTR
# +# include <sys/xattr.h>
# +#endif
#  
#  #include "wget.h"
#  #include "utils.h"
# @@ -857,7 +860,7 @@
#  
#    if (!output_stream)
#      {
# -      fp = fopen (file, "wb");
# +      fp = fopen (file, "wb"); /* xattr set below */
#        if (!fp)
#  	{
#  	  logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno));
# @@ -866,6 +869,16 @@
#      }
#    else
#      fp = output_stream;
# +
# +#ifdef ENABLE_XATTR
# +  if (opt.content_location_xattr)
# +    {
# +      if (fsetxattr (fileno (fp), "user.Content-Location",
# +                     u->url, strlen (u->url), XATTR_CREATE) != 0)
# +	fsetxattr (fileno (fp), "user.Content-Location", "", 0, 0);
# +    }
# +#endif
# +
#    if (u->user)
#      {
#        char *tmpu, *tmpp;        /* temporary, clean user and passwd */
# Index: src/convert.c
# ===================================================================
# --- src/convert.c	(revision 2052)
# +++ src/convert.c	(working copy)
# @@ -37,6 +37,9 @@
#  #endif /* HAVE_UNISTD_H */
#  #include <errno.h>
#  #include <assert.h>
# +#ifdef ENABLE_XATTR
# +# include <sys/xattr.h>
# +#endif
#  
#  #include "wget.h"
#  #include "convert.h"
# @@ -189,6 +192,7 @@
#    FILE *fp;
#    const char *p;
#    downloaded_file_t downloaded_file_return;
# +  char *content_location = 0;
#  
#    struct urlpos *link;
#    int to_url_count = 0, to_file_count = 0;
# @@ -223,6 +227,25 @@
#    if (opt.backup_converted && downloaded_file_return)
#      write_backup_file (file, downloaded_file_return);
#  
# +#ifdef ENABLE_XATTR
# +  if (opt.content_location_xattr)
# +    {
# +      ssize_t tmp = getxattr (file, "user.Content-Location", 0, 0);
# +      if (tmp >= 0)
# +	{
# +	  content_location = xmalloc (tmp + 1);
# +	  tmp = getxattr (file, "user.Content-Location", content_location, tmp);
# +	  if (tmp >= 0)
# +	    content_location[tmp] = '\0';
# +	  else
# +	    {
# +	      xfree_null (content_location);
# +	      content_location = 0;
# +	    }
# +	}
# +    }
# +#endif
# +
#    /* Before opening the file for writing, unlink the file.  This is
#       important if the data in FM is mmaped.  In such case, nulling the
#       file, which is what fopen() below does, would make us read all
# @@ -232,6 +255,7 @@
#        logprintf (LOG_NOTQUIET, _("Unable to delete `%s': %s\n"),
#  		 file, strerror (errno));
#        read_file_free (fm);
# +      xfree_null (content_location);
#        return;
#      }
#    /* Now open the file for writing.  */
# @@ -241,9 +265,21 @@
#        logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"),
#  		 file, strerror (errno));
#        read_file_free (fm);
# +      xfree_null (content_location);
#        return;
#      }
# +#ifdef ENABLE_XATTR
# +  else
# +    {
# +      if (opt.content_location_xattr && content_location)
# +	fsetxattr (fileno (fp), "user.Content-Location",
# +                   content_location, strlen (content_location), 0);
# +    }
# +#endif
#  
# +  xfree_null (content_location);
# +  content_location = 0;
# +
#    /* Here we loop through all the URLs in file, replacing those of
#       them that are downloaded with relative references.  */
#    p = fm->content;
# Index: src/main.c
# ===================================================================
# --- src/main.c	(revision 2052)
# +++ src/main.c	(working copy)
# @@ -227,6 +227,9 @@
#      { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
#      { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 },
#      { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 },
# +#ifdef ENABLE_XATTR
# +    { "set-content-location", 0, OPT_BOOLEAN, "setcontentlocation", -1 },
# +#endif
#      { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 },
#      { "spider", 0, OPT_BOOLEAN, "spider", -1 },
#      { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 },
# @@ -462,6 +465,11 @@
#         --user=USER               set both ftp and http user to USER.\n"),
#      N_("\
#         --password=PASS           set both ftp and http password to PASS.\n"),
# +#ifdef ENABLE_XATTR
# +    N_("\
# +       --set-content-location    set an extended file attribute,\n\
# +                                 user.Content-Location, on downloaded files.\n"),
# +#endif
#      "\n",
#  
#      N_("\
# @@ -897,8 +905,11 @@
#        else
#  	{
#  	  struct_fstat st;
# +	  if (opt.always_rest == 0)
# +	    unlink (opt.output_document); /* nuke problematic state, if any */
#  	  output_stream = fopen (opt.output_document,
#  				 opt.always_rest ? "ab" : "wb");
# +	  /* xattr set just before stream usage */
#  	  if (output_stream == NULL)
#  	    {
#  	      perror (opt.output_document);
# Index: doc/sample.wgetrc
# ===================================================================
# --- doc/sample.wgetrc	(revision 2052)
# +++ doc/sample.wgetrc	(working copy)
# @@ -58,6 +58,12 @@
#  ## are doing before doing so.
#  ##
#  
# +# Set this to on to put an extended attribute called
# +# user.Content-Location on all your downloaded files containing the url
# +# of where the file was downloaded from. (This option has no effect if
# +# your filesystem does not support extended attributes.)
# +#set-content-location = off
# +
#  # Set this to on to use timestamping by default:
#  #timestamping = off
#  
# Index: doc/wget.texi
# ===================================================================
# --- doc/wget.texi	(revision 2052)
# +++ doc/wget.texi	(working copy)
# @@ -969,6 +969,13 @@
#  using the @samp{--ftp-user} and @samp{--ftp-password} options for 
#  @sc{ftp} connections and the @samp{--http-user} and @samp{--http-password} 
#  options for @sc{http} connections.
# +
# +@cindex Content-Location, capture
# +@cindex extended file attributes
# +@item --set-content-location
# +On filesystems that support it, add onto every downloaded file, an
# +extended attribute called user.Content-Location, containing the url of
# +where the file was downloaded from.
#  @end table
#  
#  @node Directory Options
# Index: INSTALL
# ===================================================================
# --- INSTALL	(revision 2052)
# +++ INSTALL	(working copy)
# @@ -98,6 +98,7 @@
#    --disable-largefile     omit support for large files
#    --disable-ipv6          disable IPv6 support
#    --disable-rpath         do not hardcode runtime library paths
# +  --disable-xattr         disable support for extended attributes on files
#  
#  You can inspect decisions made by configure by editing the generated
#  Makefiles and the `src/config.h' include file.  The defaults should
# Index: NEWS
# ===================================================================
# --- NEWS	(revision 2052)
# +++ NEWS	(working copy)
# @@ -7,6 +7,12 @@
#  
#  * Changes in Wget 1.11.
#  
# +** The new function `--set-content-location' makes Wget put an
# +extended attribute called `user.Content-Location' on all downloaded
# +files containing the url of where the file was downloaded file. (This
# +option has no effect if your filesystem does not support extended
# +attributes.)
# +
#  ** The new function `--ignore-case' makes Wget ignore case when
#  matching files, directories, and wildcards.  This affects the -X, -I,
#  -A, and -R options, as well as globbing in FTP URLs.
#