[PATCH] Indexer patches
"Vlad Romanenko" <[email protected]>
| Newsgroups | gmane.comp.web.perlfect-search |
|---|---|
| Message-ID | <[email protected]> |
> From: Daniel Naber [mailto:[email protected]] > I've now set up a page with descriptions on how to get the > source: http://www.danielnaber.de/perlfectsearch/cvs.php Daniel, thank you very much! I'd like to contribute some more patches: 1) search_form.html: chmod a-x search_form.html 2) tools.pl: get_url(): Check for request error first and for content-type second. Attached tools.pl.patch 3) indexer_filesystem.pl: crawl_filesystem(): Use to_be_ignored() to check if directory is in @no_index list - in order to execute cut_document_root() on directory name. Otherwise directories aren't properly ignored. Attached indexer_filesystem.pl.patch Regards, Vlad.
tools.pl.patch
(application/octet-stream, 1.3 KB)
--- tools.pl Mon Sep 30 12:22:18 2002
+++ /home/vlad/my/work/itcs-com-au/cgi-bin/search/tools.pl Mon Sep 30 12:20:39 2002
@@ -36,17 +36,21 @@
my $request = HTTP::Request->new(GET => $url);
my $response = $http_user_agent->request($request);
- my $buffer = $response->content;
- my ($content_type) = ($response->headers_as_string =~ m/^Content-Type:\s*(.+)$/im);
- $content_type =~ s/^(.*?);.*$/$1/; # ignore possible charset value
- if( ! grep(/^$content_type$/i, @HTTP_CONTENT_TYPES) ) {
- print STDERR "Ignoring '$url': content-type '$content_type'\n" if( $HTTP_DEBUG );
- return;
- }
if( $response->is_error ) {
print STDERR "Error: Couldn't get '$url': response code " .$response->code. "\n";
return;
}
+
+ if( $response->headers_as_string =~ m/^Content-Type:\s*(.+)$/im ) {
+ my $content_type = $1;
+ $content_type =~ s/^(.*?);.*$/$1/; # ignore possible charset value
+ if( ! grep(/^$content_type$/i, @HTTP_CONTENT_TYPES) ) {
+ print STDERR "Ignoring '$url': content-type '$content_type'\n" if( $HTTP_DEBUG );
+ return;
+ }
+ }
+
+ my $buffer = $response->content;
my $size = length($buffer);
print STDERR "Fetched '$url', $size bytes\n" if( $HTTP_DEBUG );
# Maybe we are we redirected, so use the new URL.
indexer_filesystem.pl.patch
(application/octet-stream, 895 B)
--- indexer_filesystem.pl Mon Sep 30 12:22:18 2002
+++ /home/vlad/my/work/itcs-com-au/cgi-bin/search/indexer_filesystem.pl Mon Sep 30 12:47:45 2002
@@ -28,7 +28,7 @@
my $dir = $_[0];
my $doc_id;
my $file;
-
+
print $dir,"\n";
chdir $dir or (warn "Cannot chdir $dir: $!" and return);
@@ -39,7 +39,7 @@
# to ignore symbolic links, add "and not -l" to both greps:
my @dirs = grep {-d and not /^\.{1,2}$/} @contents;
my @files = grep {-f and /^.+\.(.+)$/ and grep {/^\Q$1\E$/} @EXT} @contents;
-
+
FILE: foreach my $f (@files) {
$file = $dir."/".$f;
$file =~ s/\/\//\//og;
@@ -79,10 +79,9 @@
DIR: foreach my $d (@dirs) {
$file = $dir."/".$d;
$file =~ s/\/\//\//og;
-
- foreach my $regexp (@no_index) {
- next DIR if $file =~ /^$regexp$/;
- }
+
+ next DIR if( to_be_ignored($file) );
+
crawl_filesystem($file);
}
}