RE: problem w/ robotrules record parse

--- RobotRules.pm	2005-09-21 10:14:03.000000000 -0700
+++ RobotRules.smallchanges.pm	2005-09-21 10:53:47.000000000 -0700
@@ -34,6 +34,7 @@
     my $ua;
     my $is_me = 0;		# 1 iff this record is for me
     my $is_anon = 0;		# 1 iff this record is for *
+    my $seen_disallow = 0;      # watch for missing record separators
     my @me_disallowed = ();	# rules disallowed for me
     my @anon_disallowed = ();	# rules disallowed for *
 
@@ -53,10 +54,19 @@
 	if (/^\s*$/) {	    # blank line
 	    last if $is_me; # That was our record. No need to read the rest.
 	    $is_anon = 0;
+	    $seen_disallow = 0;
 	}
         elsif (/^\s*User-Agent\s*:\s*(.*)/i) {
 	    $ua = $1;
 	    $ua =~ s/\s+$//;
+
+	    if ($seen_disallow) {
+		# malformed robots.txt - should be a newline here
+		warn "RobotRules <$robot_txt_uri>: User-agent after Disallow in same record\n" if $^W;
+		# treat as start of a new record
+		$seen_disallow = 0;
+		last if $is_me; # That was our record. No need to read the rest.
+		$is_anon = 0;
+	    }
+
 	    if ($is_me) {
 		# This record already had a User-agent that
 		# we matched, so just continue.
@@ -75,6 +85,7 @@
 	    }
 	    my $disallow = $1;
 	    $disallow =~ s/\s+$//;
+	    $seen_disallow = 1;
 	    if (length $disallow) {
 		my $ignore;
 		eval {
@@ -372,6 +383,14 @@
 value is '*', the record describes the default access policy for any
 robot that has not not matched any of the other records.
 
+The I<User-Agent> fields must occur before the I<Disallow> fields.  If a
+record contains a I<User-Agent> field after a I<Disallow> field, that
+constitutes a malformed record.  This parser will assume that a blank
+line should have been placed before that I<User-Agent> field, and will
+break the record into two.  All the fields before the I<User-Agent> field
+will constitute a record, and the I<User-Agent> field will be the first
+field in a new record.
+
 =item Disallow
 
 The value of this field specifies a partial URL that is not to be
@@ -406,6 +425,22 @@
   User-agent: *
   Disallow: /
 
+This is an example of a malformed robots.txt file.
+
+  # robots.txt for ancientcastle.example.com
+  # I've locked myself away.
+  User-agent: *
+  Disallow: /
+  # The castle is your home now, so you can go anywhere you like.
+  User-agent: Belle
+  Disallow: /west-wing/ # except the west wing!
+  # It's good to be the Prince...
+  User-agent: Beast
+  Disallow: 
+
+This file is missing the required blank lines between records.
+However, the intention is clear.
+
 =head1 SEE ALSO
 
 L<LWP::RobotUA>, L<WWW::RobotRules::AnyDBM_File>

--- rules.t	2005-09-21 11:23:19.000000000 -0700
+++ rules-new.t	2005-09-21 11:23:26.000000000 -0700
@@ -15,7 +15,7 @@
 use Carp;
 use strict;
 
-print "1..38\n"; # for Test::Harness
+print "1..50\n"; # for Test::Harness
 
 # We test a number of different /robots.txt files,
 #
@@ -64,6 +64,31 @@
 Disallow: http://bar/
 EOM
 
+my $content5 = <<EOM;
+# I've locked myself away
+User-agent: *
+Disallow: /
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# It's good to be the Prince...
+User-agent: Beast
+Disallow: 
+EOM
+
+# same thing backwards
+my $content6 = <<EOM;
+# It's good to be the Prince...
+User-agent: Beast
+Disallow: 
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# I've locked myself away
+User-agent: *
+Disallow: /
+EOM
+
 # and a number of different robots:
 
 my @tests1 = (
@@ -147,6 +172,36 @@
 	    38 => "http://bar/" => 1,
 	   ],
 
+	   [$content5, 'Villager/1.0' =>
+	    39 => 'http://foo/west-wing/' => 0,
+	    40 => 'http://foo/' => 0,
+	   ],
+
+	   [$content5, 'Belle/2.0' =>
+	    41 => 'http://foo/west-wing/' => 0,
+	    42 => 'http://foo/' => 1,
+	   ],
+
+	   [$content5, 'Beast/3.0' =>
+	    43 => 'http://foo/west-wing/' => 1,
+	    44 => 'http://foo/' => 1,
+	   ],
+
+	   [$content6, 'Villager/1.0' =>
+	    45 => 'http://foo/west-wing/' => 0,
+	    46 => 'http://foo/' => 0,
+	   ],
+
+	   [$content6, 'Belle/2.0' =>
+	    47 => 'http://foo/west-wing/' => 0,
+	    48 => 'http://foo/' => 1,
+	   ],
+
+	   [$content6, 'Beast/3.0' =>
+	    49 => 'http://foo/west-wing/' => 1,
+	    50 => 'http://foo/' => 1,
+	   ],
+
 	   # when adding tests, remember to increase
 	   # the maximum at the top