New command line option: unfollowed-links

The following patch adds a command line option that allows one to specify a
file to capture all links that wget ignores because of other command line
options. For example:
 
wget http://www.somesite.com --mirror --unfollowed-links=somesite.txt
 
will result in the file somesite.txt containing all external links from
somesite.com. I originally wrote this feature for wget version 1.8.1, but
never submitted it because I wasn't sure what to do with the ChangeLog and
other documentation. I'm submitting it anyway in the hopes that Hrvoje can
fill in the gaps. :-)
 
Hrvoje, I was prompted to send this now by the message from Dan Jacobson
with the subject "add --print-uris or --dry-run". My submission does not
provide the exact functionality that is requested, but I think the concept
may prove useful as you consider that feature.
 
2005-06-28  Tony Lewis <[email protected]>
 
        * main.c: Added --unfollowed-links option.
        * init.c: Added initialization for opt.unfollowed.
        * recur.c: Added code to record links that are not followed and
restored
        recursive_cleanup.
        * gen-md5.c: Added include of <stdio.h> to eliminate compile error.
        * options.h: Defined ufp and unfollowed.
        * recur.h: Defined unfollowed.
 
Index: main.c
===================================================================
--- main.c (version 1.10)
+++ main.c (working copy)
@@ -258,6 +258,7 @@
     { "timeout", 'T', OPT_VALUE, "timeout", -1 },
     { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 },
     { "tries", 't', OPT_VALUE, "tries", -1 },
+    { "unfollowed-links", 0, OPT_VALUE, "unfollowedlinks", -1 },
     { "user", 0, OPT_VALUE, "user", -1 },
     { "user-agent", 'U', OPT_VALUE, "useragent", -1 },
     { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 },
@@ -633,6 +634,8 @@
     N_("\
   -X,  --exclude-directories=LIST  list of excluded directories.\n"),
     N_("\
+       --unfollowed-links=FILE     log unfollowed links to FILE.\n"),
+    N_("\
   -np, --no-parent                 don't ascend to the parent
directory.\n"),
     "\n",

@@ -821,6 +824,13 @@
        opt.dirstruct = 1;      /* normally handled by cmd_spec_recursive()
*/
     }

+  if (opt.unfollowed && !opt.recursive)
+    {
+      printf (_("%s: Must be recursive to have unfollowed links\n"),
exec_name);
+      print_usage ();
+      exit (1);
+    }
+
   if (opt.verbose == -1)
     opt.verbose = !opt.quiet;

@@ -906,6 +916,17 @@
            }
          if (fstat (fileno (output_stream), &st) == 0 && S_ISREG
(st.st_mode))
            output_stream_regular = 1;
+       }
+    }
+
+  /* Open the unfollowed links filename if necessary.  */
+  if (opt.unfollowed)
+    {
+      opt.ufp = fopen (opt.unfollowed, "wb");
+      if (opt.ufp == NULL)
+       {
+         perror (opt.unfollowed);
+         exit (1);
        }
     }

Index: init.c
===================================================================
--- init.c (version 1.10)
+++ init.c (working copy)
@@ -238,6 +238,7 @@
   { "timeout",         NULL,                   cmd_spec_timeout },
   { "timestamping",    &opt.timestamping,      cmd_boolean },
   { "tries",           &opt.ntry,              cmd_number_inf },
+  { "unfollowedlinks",  &opt.unfollowed,        cmd_file },
   { "useproxy",                &opt.use_proxy,         cmd_boolean },
   { "user",            &opt.user,              cmd_string },
   { "useragent",       NULL,                   cmd_spec_useragent },
@@ -572,7 +573,7 @@
   p = line;

   cmdstart = p;
-  while (p < end && (ISALPHA (*p) || *p == '_' || *p == '-'))
+  while (p < end && (ISALNUM (*p) || *p == '_' || *p == '-'))
     ++p;
   cmdend = p;

@@ -1471,6 +1472,9 @@
        checks for errors) after any data arrives.  */
   }

+  if (opt.ufp)
+    fclose (opt.ufp);
+
   /* We're exiting anyway so there's no real need to call free()
      hundreds of times.  Skipping the frees will make Wget exit
      faster.
@@ -1480,6 +1484,7 @@
      memory which grows with the size of the program.  */

 #ifdef DEBUG_MALLOC
+  recursive_cleanup ();
   convert_cleanup ();
   res_cleanup ();
   http_cleanup ();
@@ -1525,6 +1530,8 @@
   xfree_null (opt.bind_address);
   xfree_null (opt.cookies_input);
   xfree_null (opt.cookies_output);
+  xfree_null (opt.unfollowed);
+  xfree_null (opt.ufp);
   xfree_null (opt.user);
   xfree_null (opt.passwd);
 #endif /* DEBUG_MALLOC */

Index: recur.c
===================================================================
--- recur.c (version 1.10)
+++ recur.c (working copy)
@@ -63,6 +63,8 @@

 extern struct hash_table *dl_url_file_map;
 extern struct hash_table *downloaded_html_set;
+struct hash_table *unfollowed_logged;
+


 /* Functions for maintaining the URL queue.  */

@@ -118,7 +120,7 @@
   if (queue->count > queue->maxcount)
     queue->maxcount = queue->count;

-  DEBUGP (("Enqueuing %s at depth %d\n", url, depth));
+  DEBUGP (("Enqueuing %s at depth %d (from %s)\n", url, depth, referer));
   DEBUGP (("Queue count %d, maxcount %d.\n", queue->count,
queue->maxcount));

   if (queue->tail)
@@ -328,6 +330,14 @@

          if (opt.use_robots && meta_disallow_follow)
            {
+              if (opt.ufp)
+                {
+                 struct urlpos *child = children;
+
+                 for (; child; child = child->next)
+                     unfollowed(child->url->url);
+                }
+
              free_urlpos (children);
              children = NULL;
            }
@@ -341,9 +351,15 @@
              for (; child; child = child->next)
                {
                  if (child->ignore_when_downloading)
-                   continue;
+                    {
+                     unfollowed(child->url->url);
+                     continue;
+                    }
                  if (dash_p_leaf_HTML && !child->link_inline_p)
-                   continue;
+                    {
+                     unfollowed(child->url->url);
+                     continue;
+                    }
                  if (download_child_p (child, url_parsed, depth,
start_url_parsed,
                                        blacklist))
                    {
@@ -433,7 +449,7 @@
   if (string_set_contains (blacklist, url))
     {
       DEBUGP (("Already on the black list.\n"));
-      goto out;
+      goto out_blacklist;
     }

   /* Several things to check for:
@@ -588,6 +604,8 @@
   return 1;

  out:
+  unfollowed(u->url);
+ out_blacklist:
   DEBUGP (("Decided NOT to load it.\n"));

   return 0;
@@ -626,4 +644,31 @@
     DEBUGP (("Redirection \"%s\" failed the test.\n", redirected));

   return success;
+}
+
+/* Cleanup the data structures associated with recursive retrieving
+   (the variables above).  */
+void
+recursive_cleanup (void)
+{
+  if (unfollowed_logged)
+    string_set_free (unfollowed_logged);
+  unfollowed_logged = NULL;
+}
+
+/* Record unfollowed links */
+void
+unfollowed (const char *url)
+{
+  if (!opt.ufp)
+    return;
+
+  if (unfollowed_logged == NULL)
+    unfollowed_logged = make_string_hash_table (0);
+
+  if (string_set_contains (unfollowed_logged, url))
+    return;
+
+  string_set_add (unfollowed_logged, url);
+  fprintf(opt.ufp, "%s\n", url);
 }

Index: gen-md5.c
===================================================================
--- gen-md5.c (version 1.10)
+++ get-md5.c (working copy)
@@ -28,6 +28,7 @@
 so, delete this exception statement from your version.  */

 #include <config.h>
+#include <stdio.h>
 #include "wget.h"

 #include "gen-md5.h"

Index: options.h
===================================================================
--- options.h (version 1.10)
+++ options.h (working copy)
@@ -219,6 +219,11 @@
     prefer_none
   } prefer_family;             /* preferred address family when more
                                   than one type is available */
+
+  FILE *ufp;                   /* The file pointer to the unfollowed links
+                                  file. */
+  char *unfollowed;             /* The file to which unfollowed links will
+                                   be recorded. */
 };

 extern struct options opt;

Index: recur.h
===================================================================
--- recur.h (version 1.10)
+++ recur.h (working copy)
 struct urlpos *get_urls_html PARAMS ((const char *, const char *, int *));
 void free_urlpos PARAMS ((struct urlpos *));

+void unfollowed PARAMS ((const char *));
+
 #endif /* RECUR_H */