xmltv/grab/uk_tvguide tv_grab_uk_tvguide,1.20,1.21

Update of /cvsroot/xmltv/xmltv/grab/uk_tvguide
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv13720

Modified Files:
	tv_grab_uk_tvguide 
Log Message:
Retry web page fetch 5 times before ignoring page (bugs #514 #518)

Index: tv_grab_uk_tvguide
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/uk_tvguide/tv_grab_uk_tvguide,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** tv_grab_uk_tvguide	1 Jan 2017 13:01:03 -0000	1.20
--- tv_grab_uk_tvguide	22 Jan 2017 22:01:26 -0000	1.21
***************
*** 23,27 ****
  use XMLTV::Configure::Writer;
  use XMLTV::Supplement 0.005065 qw/SetSupplementRoot GetSupplementDir GetSupplementLines/;
! use XMLTV::Get_nice qw/get_nice_tree/;
  
  use File::Path;
--- 23,27 ----
  use XMLTV::Configure::Writer;
  use XMLTV::Supplement 0.005065 qw/SetSupplementRoot GetSupplementDir GetSupplementLines/;
! use XMLTV::Get_nice 0.005065 qw/get_nice_tree/;
  
  use File::Path;
***************
*** 32,35 ****
--- 32,36 ----
  use Encode;
  use URI::Escape;
+ use HTML::TreeBuilder;
  
  use HTTP::Cache::Transparent;
***************
*** 200,204 ****
  				
  				# Fetch the page
! 				my $tree = XMLTV::Get_nice::get_nice_tree($url);
  				# $tree->dump; exit;
  
--- 201,206 ----
  				
  				# Fetch the page
! 				#   my $tree = XMLTV::Get_nice::get_nice_tree($url);
! 				my $tree = fetch_url($url);
  				# $tree->dump; exit;
  
***************
*** 271,275 ****
  
  							# Fetch the page
! 							my $treedetail = XMLTV::Get_nice::get_nice_tree($url);
  							
  							my $showdetail = $treedetail;						
--- 273,278 ----
  
  							# Fetch the page
! 							#   my $treedetail = XMLTV::Get_nice::get_nice_tree($url);
! 							my $treedetail = fetch_url($url);
  							
  							my $showdetail = $treedetail;						
***************
*** 434,439 ****
  							#                         <span class="season">6:00 am </span>
  							# but this just doesn't sound right to me (i.e. I think it might change again), so let's try both ways
! 							# @ 2016-08-05 looks like this is permanent
! 							#   my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
  							my $showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
  							if (!$prog{'start'}) {
--- 437,443 ----
  							#                         <span class="season">6:00 am </span>
  							# but this just doesn't sound right to me (i.e. I think it might change again), so let's try both ways
! 							#
! 							# @2016-08-05 looks like this is permanent
! 							# my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
  							my $showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
  							if (!$prog{'start'}) {
***************
*** 838,841 ****
--- 842,866 ----
  }
  
+ sub fetch_url {
+ 	# fetch a url with up to 5 retries
+ 		my ($url) = @_;
+ 		$XMLTV::Get_nice::FailOnError = 0;
+ 		my $content;
+ 		my $maxretry = 5;
+ 		my $retry = 0;
+ 		while ( (not defined($content = XMLTV::Get_nice::get_nice($url))) || (length($content) == 0) ) {
+ 			my $r = $XMLTV::Get_nice::Response;
+ 			print STDERR "HTTP error: ".$r->status_line."\n";
+ 			$retry++;
+ 			return undef if $retry > $maxretry;
+ 			print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
+ 		}
+ 		$content = decode('UTF-8', $content);
+ 		my $t = HTML::TreeBuilder->new();
+ 		$t->parse($content) or die "cannot parse content of $url\n";
+ 		$t->eof;
+ 		return $t;
+ }
+ 
  sub get_default_dir {
      my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH} 


------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, SlashDot.org! http://sdm.link/slashdot