Re: regex in wget, it is dificult to implement?
Tobias Tiederle <[email protected]> Mon, 30 May 2005 15:06:44 +0200
| Newsgroups | gmane.comp.web.wget.patches |
|---|---|
| Message-ID | <[email protected]> |
Hi, here is the promised regex patch against current CVS. The same things said before apply: --snip-- I used pcre library from http://www.pcre.org which was pretty easy to use, given the fact that I never ever touched a single line of C (or C++) code before. Unfortunately I don't know jack about autoconf, makefiles etc. The patch in its current form is only useful with MSVC as I didn't alter any other makefiles. I hope someone can do that for me and include the pcre license from http://www.pcre.org/license.txt As you can see pcre.h and pcre.lib need to be somewhere the compiler can find them and HAVE_REGEX needs to be defined. Files and directories are ignored if the regex given on the command line match. For Syntax see wget --help. --snip-- I hope you can use this or come up with something more elegant. Tobias
pcre-regex.diff
(text/plain, 9.7 KB)
Only in wget-regex: pcre-r
diff -ruwb -x CVS wget-clean\src\ftp.c wget-regex\src\ftp.c
--- wget-clean\src\ftp.c Mon May 23 01:57:15 2005
+++ wget-regex\src\ftp.c Mon May 30 14:05:27 2005
@@ -1705,7 +1705,11 @@
return res;
/* First: weed out that do not conform the global rules given in
opt.accepts and opt.rejects. */
+#ifdef HAVE_REGEX
+ if (opt.accepts || opt.rejects || opt.exclregfile)
+#else
if (opt.accepts || opt.rejects)
+#endif /* HAVE_REGEX */
{
f = start;
while (f)
diff -ruwb -x CVS wget-clean\src\init.c wget-regex\src\init.c
--- wget-clean\src\init.c Mon May 23 01:57:15 2005
+++ wget-regex\src\init.c Mon May 30 14:12:02 2005
@@ -154,6 +154,10 @@
#endif
{ "excludedirectories", &opt.excludes, cmd_directory_vector },
{ "excludedomains", &opt.exclude_domains, cmd_vector },
+#ifdef HAVE_REGEX
+ { "excluderegexdir", &opt.exclregdir, cmd_string },
+ { "excluderegexfile", &opt.exclregfile, cmd_string },
+#endif /* HAVE_REGEX */
{ "followftp", &opt.follow_ftp, cmd_boolean },
{ "followtags", &opt.follow_tags, cmd_vector },
{ "forcehtml", &opt.force_html, cmd_boolean },
@@ -1505,6 +1509,12 @@
xfree_null (opt.http_user);
xfree_null (opt.http_passwd);
free_vec (opt.user_headers);
+#ifdef HAVE_REGEX
+ xfree_null (opt.exclregdir_c)
+ xfree_null (opt.exclregfile_c)
+ xfree_null (opt.exclregdir);
+ xfree_null (opt.exclregfile);
+#endif /* HAVE_REGEX */
# ifdef HAVE_SSL
xfree_null (opt.cert_file);
xfree_null (opt.private_key);
@@ -1513,6 +1523,7 @@
xfree_null (opt.random_file);
xfree_null (opt.egd_file);
# endif
+
xfree_null (opt.bind_address);
xfree_null (opt.cookies_input);
xfree_null (opt.cookies_output);
diff -ruwb -x CVS wget-clean\src\main.c wget-regex\src\main.c
--- wget-clean\src\main.c Mon May 23 01:57:15 2005
+++ wget-regex\src\main.c Mon May 30 14:12:01 2005
@@ -68,6 +68,10 @@
/* On GNU system this will include system-wide getopt.h. */
#include "getopt.h"
+#ifdef HAVE_REGEX
+#include <pcre.h>
+#endif /* HAVE_REGEX */
+
#ifndef PATH_SEPARATOR
# define PATH_SEPARATOR '/'
#endif
@@ -183,6 +187,10 @@
{ "egd-file", 0, OPT_VALUE, "egdfile", -1 },
{ "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 },
{ "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 },
+#ifdef HAVE_REGEX
+ { "exclude-regex-dirs", 0, OPT_VALUE, "excluderegexdir", -1 },
+ { "exclude-regex-files", 0, OPT_VALUE, "excluderegexfile", -1 },
+#endif
{ "execute", 'e', OPT__EXECUTE, NULL, required_argument },
{ "follow-ftp", 0, OPT_BOOLEAN, "followftp", -1 },
{ "follow-tags", 0, OPT_VALUE, "followtags", -1 },
@@ -618,6 +626,12 @@
-D, --domains=LIST comma-separated list of accepted domains.\n"),
N_("\
--exclude-domains=LIST comma-separated list of rejected domains.\n"),
+#ifdef HAVE_REGEX
+ N_("\
+ --exclude-regex-dirs=PATTERN pattern of directories to reject.\n"),
+ N_("\
+ --exclude-regex-files=PATTERN pattern of files to reject.\n"),
+#endif /* HAVE_REGEX */
N_("\
--follow-ftp follow FTP links from HTML documents.\n"),
N_("\
@@ -674,6 +688,7 @@
int i, ret, longindex;
int nurl, status;
int append_to_log = 0;
+ const char *error;
i18n_initialize ();
@@ -846,6 +861,40 @@
exit (1);
}
#endif
+
+#ifdef HAVE_REGEX
+ if (opt.exclregdir)
+ {
+ opt.exclregdir_c = pcre_compile(
+ opt.exclregdir, /* the pattern */
+ 0, /* default options */
+ &error, /* for error message */
+ &i, /* for error offset */
+ NULL); /* use default character tables */
+
+ if (opt.exclregdir_c == NULL)
+ {
+ printf (_("Directory RegEx compilation failed at offset %d: %s\n"), i, error);
+ exit (1);
+ }
+ }
+
+ if (opt.exclregfile)
+ {
+ opt.exclregfile_c = pcre_compile(
+ opt.exclregfile, /* the pattern */
+ 0, /* default options */
+ &error, /* for error message */
+ &i, /* for error offset */
+ NULL); /* use default character tables */
+
+ if (opt.exclregfile_c == NULL)
+ {
+ printf (_("File RegEx compilation failed at offset %d: %s\n"), i, error);
+ exit (1);
+ }
+ }
+#endif /* HAVE_REGEX */
nurl = argc - optind;
if (!nurl && !opt.input_filename)
diff -ruwb -x CVS wget-clean\src\options.h wget-regex\src\options.h
--- wget-clean\src\options.h Mon May 23 01:57:15 2005
+++ wget-regex\src\options.h Mon May 30 14:05:28 2005
@@ -27,6 +27,10 @@
file, but you are not obligated to do so. If you do not wish to do
so, delete this exception statement from your version. */
+#ifdef HAVE_REGEX
+#include <pcre.h>
+#endif /* HAVE_REGEX */
+
struct options
{
int verbose; /* Are we verbose? */
@@ -65,6 +69,12 @@
char **excludes; /* List of excluded FTP directories. */
char **includes; /* List of FTP directories to
follow. */
+#ifdef HAVE_REGEX
+ char *exclregdir; /* Pattern for regex exclusion */
+ char *exclregfile; /* Pattern for regex exclusion */
+ pcre *exclregdir_c;
+ pcre *exclregfile_c;
+#endif /* HAVE_REGEX */
char **domains; /* See host.c */
char **exclude_domains;
diff -ruwb -x CVS wget-clean\src\recur.c wget-regex\src\recur.c
--- wget-clean\src\recur.c Thu Apr 14 11:33:47 2005
+++ wget-regex\src\recur.c Mon May 30 14:05:28 2005
@@ -506,7 +506,11 @@
/* 5. If the file does not match the acceptance list, or is on the
rejection list, chuck it out. The same goes for the directory
exclusion and inclusion lists. */
+#ifdef HAVE_REGEX
+ if (opt.includes || opt.excludes || opt.exclregdir)
+#else
if (opt.includes || opt.excludes)
+#endif /* HAVE_REGEX */
{
if (!accdir (u->dir, ALLABS))
{
diff -ruwb -x CVS wget-clean\src\utils.c wget-regex\src\utils.c
--- wget-clean\src\utils.c Mon May 23 01:57:16 2005
+++ wget-regex\src\utils.c Mon May 30 14:56:08 2005
@@ -104,6 +104,10 @@
#include "utils.h"
#include "hash.h"
+#ifdef HAVE_REGEX
+#include <pcre.h>
+#endif /* HAVE_REGEX */
+
#ifndef errno
extern int errno;
#endif
@@ -652,6 +656,15 @@
--l;
if (s[l] == '/')
s += (l + 1);
+
+#ifdef HAVE_REGEX
+ if (opt.exclregfile) {
+ if (!accregex((const pcre *const *)opt.exclregfile_c, s)) {
+ return 0;
+ }
+ }
+#endif /* HAVE_REGEX */
+
if (opt.accepts)
{
if (opt.rejects)
@@ -665,6 +678,31 @@
return 1;
}
+#ifdef HAVE_REGEX
+
+int
+accregex(const pcre *re, const char *s)
+{
+ int rc;
+ int ovector[3];
+
+ rc = pcre_exec(
+ re, /* the compiled pattern */
+ NULL, /* no extra data - we didn't study the pattern */
+ s, /* the subject string */
+ (int)strlen(s), /* the length of the subject */
+ 0, /* start at offset 0 in the subject */
+ 0, /* default options */
+ ovector, /* output vector for substring information */
+ 3); /* number of elements in the output vector */
+
+ // if rc < 0, the pattern did match -> accept
+ if (rc < 0) return 1;
+ else return 0;
+}
+
+#endif /* HAVE_REGEX */
+
/* Compare S1 and S2 frontally; S2 must begin with S1. E.g. if S1 is
`/something', frontcmp() will return 1 only if S2 begins with
`/something'. Otherwise, 0 is returned. */
@@ -710,6 +748,16 @@
/* Remove starting '/'. */
if (flags & ALLABS && *directory == '/')
++directory;
+
+#ifdef HAVE_REGEX
+ if (opt.exclregdir) {
+ if (!accregex((const pcre *const *)opt.exclregdir_c, directory)) {
+ printf("Excluding directory %s (RegEx)\n",directory);
+ return 0;
+ }
+ }
+#endif /* HAVE_REGEX */
+
if (opt.includes)
{
if (!proclist (opt.includes, directory, flags))
Only in wget-regex\src: utils.c.bak
diff -ruwb -x CVS wget-clean\windows\Makefile.src wget-regex\windows\Makefile.src
--- wget-clean\windows\Makefile.src Mon May 23 01:57:16 2005
+++ wget-regex\windows\Makefile.src Mon May 30 14:53:45 2005
@@ -36,14 +36,22 @@
SSLOBJ = openssl$o http-ntlm$o
!endif
+# RegEx support requires the PCRE library (see http://www.pcre.org/).
+# If you do not have PCRE installed or wish to build Wget without RegEx
+# support, either comment-out the following lines or define NO_REGEX.
+!ifndef NO_REGEX
+REGDEFS = /DHAVE_REGEX
+REGLIBS = pcre.lib
+!endif
+
o = .obj
CC = cl
LD = link
RM = -del
-DEFS = /DWINDOWS /D_CONSOLE /DHAVE_CONFIG_H $(SSLDEFS)
-LIBS = kernel32.lib advapi32.lib wsock32.lib user32.lib gdi32.lib $(SSLLIBS)
+DEFS = /DWINDOWS /D_CONSOLE /DHAVE_CONFIG_H $(SSLDEFS) $(REGDEFS)
+LIBS = kernel32.lib advapi32.lib wsock32.lib user32.lib gdi32.lib $(SSLLIBS) $(REGLIBS)
!ifdef DEBUG
CFLAGS = /nologo /MTd /Od /Zi /I. $(DEFS)
diff -ruwb -x CVS wget-clean\windows\README wget-regex\windows\README
--- wget-clean\windows\README Mon May 23 01:57:16 2005
+++ wget-regex\windows\README Mon May 30 14:05:29 2005
@@ -39,6 +39,12 @@
lines in windows\Makefile.src; then follow the normal instructions
(configure.bat and so on).
+By default (for MSVC), wget is built with RegEx support, using the PCRE Library.
+You need PCRE from http://www.pcre.org. Get the source and compile it.
+Place pcre.h and pcre.lib somewhere your compiler can find them.
+pcre.h to "C:\Program Files\Microsoft Visual Studio\VC98\lib"
+pcre.lib to "C:\Program Files\Microsoft Visual Studio\VC98\include"
+
If you want to build the help file you will need a copy of makeinfo to
convert wget.texi to rtf and html. I've made a copy available at
<URL:ftp://sunsite.dk/projects/wget/makeinfo.zip>. This copy of