xmltv/grab/sd_json tv_grab_sd_json,1.18,1.19

Update of /cvsroot/xmltv/xmltv/grab/sd_json
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv12177/grab/sd_json

Modified Files:
	tv_grab_sd_json 
Log Message:
tv_grab_sd_json: Improve efficiency of date/time handling.
    
After profiling the code using NYTProf I found that the DateTime parsing
and comparisons were the top usage of execution time. Adding my own parsing
functions and converting everything to seconds from epoch for comparisons
makes a huge difference in performance. Excluding network delays (or when
everything is already in the cache) the execution time is now about cut in
half.
    
DateTime usage is actually still the number one item in the profiler output
so there may still be room for improvement here but dates and times are
hard to get right with all the timezone, DST, leap, etc. issues.


Index: tv_grab_sd_json
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/sd_json/tv_grab_sd_json,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** tv_grab_sd_json	8 Jul 2016 23:03:48 -0000	1.18
--- tv_grab_sd_json	9 Jul 2016 18:03:41 -0000	1.19
***************
*** 59,63 ****
  use JSON;
  use Digest::SHA qw(sha1_hex);
! use DateTime::Format::DateParse;
  use List::MoreUtils qw(uniq);
  use Scalar::Util qw(looks_like_number);
--- 59,63 ----
  use JSON;
  use Digest::SHA qw(sha1_hex);
! use DateTime;
  use List::MoreUtils qw(uniq);
  use Scalar::Util qw(looks_like_number);
***************
*** 167,179 ****
  }
  
! # calculate start and stop time from offset and days options
! my $dt_start = DateTime->today(time_zone => 'local');
! $dt_start->add(days => $opt->{'offset'});
! my $dt_stop = $dt_start->clone();
! $dt_stop->add(days => $opt->{'days'});
  
! # source data has times in UTC
! $dt_start->set_time_zone('UTC');
! $dt_stop->set_time_zone('UTC');
  
  my $cache_file = $conf->{'cache'}->[0];
--- 167,186 ----
  }
  
! sub get_start_stop_time {
! 	# calculate start and stop time from offset and days options
! 	my $dt_start = DateTime->today(time_zone => 'local');
! 	$dt_start->add(days => $opt->{'offset'});
! 	my $dt_stop = $dt_start->clone();
! 	$dt_stop->add(days => $opt->{'days'});
  
! 	# source data has times in UTC
! 	$dt_start->set_time_zone('UTC');
! 	$dt_stop->set_time_zone('UTC');
! 
! 	# convert DateTime to seconds from epoch which will allow for a LOT faster
! 	# comparisons than comparing DateTime objects
! 	return ($dt_start->epoch(), $dt_stop->epoch());
! }
! my ($time_start, $time_stop) = get_start_stop_time();
  
  my $cache_file = $conf->{'cache'}->[0];
***************
*** 189,192 ****
--- 196,232 ----
  }
  
+ my $dt_zone_utc = DateTime::TimeZone->new(name => 'UTC');
+ my $dt_zone_local = DateTime::TimeZone->new(name => 'local');
+ 
+ sub parse_airdate {
+ 	my @date = ($_[0] =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z/);
+ 	local $Params::Validate::NO_VALIDATION = 1;
+ 	return DateTime->new(
+ 		year       => $date[0],
+ 		month      => $date[1],
+ 		day        => $date[2],
+ 		hour       => $date[3],
+ 		minute     => $date[4],
+ 		second     => $date[5],
+ 		time_zone  => $dt_zone_utc,
+ 	);
+ }
+ 
+ # SD-JSON only specifies a date for originalAirDate. Older versions of
+ # mythtv need full date and time even though xmltv only requires date.
+ # We assume local time as mythtv expects and set the time to noon to
+ # minimize the chance of an error causing the day to be off by one.
+ sub parse_original_airdate {
+ 	my @date = ($_[0] =~ /(\d+)-(\d+)-(\d+)/);
+ 	local $Params::Validate::NO_VALIDATION = 1;
+ 	return DateTime->new(
+ 		year       => $date[0],
+ 		month      => $date[1],
+ 		day        => $date[2],
+ 		hour       => 12,
+ 		time_zone  => $dt_zone_local,
+ 	);
+ }
+ 
  sub retry {
  	my ($action) = @_;
***************
*** 787,794 ****
  		for my $schedule (values %{$cache_schedules->{$channel}}) {
  			for my $program (@{$schedule->{'programs'}}) {
! 				my $dt = DateTime::Format::DateParse->parse_datetime($program->{'airDateTime'});
! 				my $dur = DateTime::Duration->new(seconds => $program->{'duration'});
  
! 				if(($dt + $dur) > $dt_start && $dt < $dt_stop) {
  					my $id = $program->{'programID'};
  					my $cached = $cache_programs->{$id};
--- 827,835 ----
  		for my $schedule (values %{$cache_schedules->{$channel}}) {
  			for my $program (@{$schedule->{'programs'}}) {
! 				my $dt = parse_airdate($program->{'airDateTime'});
! 				my $airtime = $dt->epoch();
! 				my $dur = int($program->{'duration'});
  
! 				if(($airtime + $dur) > $time_start && $airtime < $time_stop) {
  					my $id = $program->{'programID'};
  					my $cached = $cache_programs->{$id};
***************
*** 1289,1300 ****
  	my $date = $details->{'originalAirDate'};
  	if($date) {
! 		# SD-JSON only specifies a date for originalAirDate. Older versions of
! 		# mythtv need full date and time even though xmltv only requires date.
! 		# DateParse will assume local time as mythtv expects. We set the time
! 		# to noon to minimize the chance of an error causing the day to be off
! 		# by one. Whether we actually include the time in the output depends
! 		# on the selected format in the config file.
! 		my $dt = DateTime::Format::DateParse->parse_datetime($date);
! 		$dt->add(hours => 12);
  		$previously_shown{'start'} = $dt->strftime($previously_shown_format);
  	}
--- 1330,1334 ----
  	my $date = $details->{'originalAirDate'};
  	if($date) {
! 		my $dt = parse_original_airdate($date);
  		$previously_shown{'start'} = $dt->strftime($previously_shown_format);
  	}
***************
*** 1382,1391 ****
  	my ($w, $channel, $program, $details) = @_;
  
! 	my $dt = DateTime::Format::DateParse->parse_datetime($program->{'airDateTime'});
! 	my $dur = DateTime::Duration->new(seconds => $program->{'duration'});
  
! 	if(($dt + $dur) > $dt_start && $dt < $dt_stop) {
  		my $start = $dt->strftime('%Y%m%d%H%M%S %z');
! 		$dt->add_duration($dur);
  		my $stop = $dt->strftime('%Y%m%d%H%M%S %z');
  
--- 1416,1426 ----
  	my ($w, $channel, $program, $details) = @_;
  
! 	my $dt = parse_airdate($program->{'airDateTime'});
! 	my $airtime = $dt->epoch();
! 	my $dur = int($program->{'duration'});
  
! 	if(($airtime + $dur) > $time_start && $airtime < $time_stop) {
  		my $start = $dt->strftime('%Y%m%d%H%M%S %z');
! 		$dt->add(seconds => $dur);
  		my $stop = $dt->strftime('%Y%m%d%H%M%S %z');
  


------------------------------------------------------------------------------
Attend Shape: An AT&T Tech Expo July 15-16. Meet us at AT&T Park in San
Francisco, CA to explore cutting-edge tech and listen to tech luminaries
present their vision of the future. This family event has something for
everyone, including kids. Get more information and register today.
http://sdm.link/attshape