xmltv/grab/fi test.sh, 2.06, 2.07 tv_grab_fi.pl, 2.04, 2.05

Update of /cvsroot/xmltv/xmltv/grab/fi
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv23369/grab/fi

Modified Files:
	test.sh tv_grab_fi.pl 
Log Message:
- main: implement randomized access pattern
- test: add "norandomize" option
- documentation: fix email address


Index: tv_grab_fi.pl
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi/tv_grab_fi.pl,v
retrieving revision 2.04
retrieving revision 2.05
diff -C2 -d -r2.04 -r2.05
*** tv_grab_fi.pl	1 Nov 2013 22:55:13 -0000	2.04
--- tv_grab_fi.pl	21 Jun 2014 16:36:15 -0000	2.05
***************
*** 18,21 ****
--- 18,22 ----
  # Perl core modules
  use Getopt::Long;
+ use List::Util qw(shuffle);
  use Pod::Usage;
  
***************
*** 89,92 ****
--- 90,94 ----
  	       "help|h|?",
  	       "list-channels",
+ 	       "no-randomize",
  	       "offset=i",
  	       "output=s",
***************
*** 347,378 ****
    my $writer = _createXMLTVWriter();
  
!   # For each channel and each day
!   my %seen;
!   my @programmes;
!   _createProgressBar("getting listings", keys(%channels) * (@{ $dates } - 2));
    foreach my $id (sort keys %channels) {
-     debug(1, "XMLTV channel ID: $id");
      for (my $i = 1; $i < $#{ $dates }; $i++) {
!       debug(1, "Fetching day $dates->[$i]");
!       foreach my $source (@sources) {
! 	if (my $programmes = $source->grab($id,
! 					   @{ $dates }[$i - 1..$i + 1],
! 					   $Option{offset} + $i - 1)) {
  
! 	  if (@{ $programmes }) {
! 	    # Add channel ID & name (once)
! 	    _addChannel($writer, $id, $channels{$id},
! 			$programmes->[0]->language())
! 	      unless $seen{$id}++;
  
! 	    # Add programmes to list
! 	    push(@programmes, @{ $programmes });
! 	  } elsif ($Option{'test-mode'}) {
! 	    die "test failure: source '" . $source->description . "' didn't retrieve any programmes for '$id'!\n";
! 	  }
  	}
        }
-       _updateProgressBar();
      }
    }
    _destroyProgressBar();
--- 349,395 ----
    my $writer = _createXMLTVWriter();
  
!   # Generate task list with one task per channel and day
!   my @tasklist;
    foreach my $id (sort keys %channels) {
      for (my $i = 1; $i < $#{ $dates }; $i++) {
!       push(@tasklist, [$id,
! 		       @{ $dates }[$i - 1..$i + 1],
! 		       $Option{offset} + $i - 1]);
!     }
!   }
  
!   # Randomize the task list in order to create a random access pattern
!   # NOTE: if you use only one source, then this is basically a no-op
!   if (not $Option{'no-randomize'}) {
!     debug(1, "Randomizing task list");
!     @tasklist = shuffle(@tasklist);
!   }
  
!   # For each entry in the task list
!   my %seen;
!   my @programmes;
!   _createProgressBar("getting listings", @tasklist);
!   foreach my $task (@tasklist) {
!     my($id, $yesterday, $today, $tomorrow, $offset) = @{$task};
!     debug(1, "XMLTV channel ID '$id' fetching day $today");
!     foreach my $source (@sources) {
!       if (my $programmes = $source->grab($id,
! 					 $yesterday, $today, $tomorrow,
! 					 $offset)) {
! 
! 	if (@{ $programmes }) {
! 	  # Add channel ID & name (once)
! 	  _addChannel($writer, $id, $channels{$id},
! 		      $programmes->[0]->language())
! 	    unless $seen{$id}++;
! 
! 	  # Add programmes to list
! 	  push(@programmes, @{ $programmes });
! 	} elsif ($Option{'test-mode'}) {
! 	  die "test failure: source '" . $source->description . "' didn't retrieve any programmes for '$id'!\n";
  	}
        }
      }
+     _updateProgressBar();
    }
    _destroyProgressBar();
***************
*** 404,407 ****
--- 421,425 ----
             [--days E<lt>NE<gt>]
             [--gui [E<lt>OPTIONE<gt>]]
+            [--no-randomize]
             [--offset E<lt>NE<gt>]
             [--output E<lt>FILEE<gt>]
***************
*** 537,540 ****
--- 555,566 ----
  Default is 14 days.
  
+ =item B<--no-randomize>
+ 
+ Grab TV data in deterministic order, i.e. first fetch channel 1, days 1 to N,
+ then channel 2, and so on.
+ 
+ Default is to use a random access pattern. If you only grab TV data from one
+ source then the randomizing is a no-op.
+ 
  =item B<--offset C<N>>
  
***************
*** 603,607 ****
  =over
  
! =item Stefan Becker C<stefan dot becker at nokia dot com>
  
  =item Ville Ahonen C<ville dot ahonen at iki dot fi>
--- 629,633 ----
  =over
  
! =item Stefan Becker C<chemobejk at gmail dot com>
  
  =item Ville Ahonen C<ville dot ahonen at iki dot fi>

Index: test.sh
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi/test.sh,v
retrieving revision 2.06
retrieving revision 2.07
diff -C2 -d -r2.06 -r2.07
*** test.sh	14 Jun 2014 18:18:36 -0000	2.06
--- test.sh	21 Jun 2014 16:36:15 -0000	2.07
***************
*** 43,46 ****
--- 43,49 ----
  	    merge_script=1
              ;;
+ 	norandomize)
+ 	    debug="$debug --no-randomize"
+ 	    ;;
  	reuse)
  	    preserve_directory=1


------------------------------------------------------------------------------
HPCC Systems Open Source Big Data Platform from LexisNexis Risk Solutions
Find What Matters Most in Your Big Data with HPCC Systems
Open Source. Fast. Scalable. Simple. Ideal for Dirty Data.
Leverages Graph Analysis for Fast Processing & Easy Data Exploration
http://p.sf.net/sfu/hpccsystems