xmltv/grab/fi_sv test.conf, 1.2, 1.3 tv_grab_fi_sv, 1.11, 1.12
Geoff <[email protected]>
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/grab/fi_sv
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv3564
Modified Files:
test.conf tv_grab_fi_sv
Log Message:
(major) unbreak after source site changes
Index: tv_grab_fi_sv
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi_sv/tv_grab_fi_sv,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** tv_grab_fi_sv 26 Jun 2011 09:22:18 -0000 1.11
--- tv_grab_fi_sv 13 Apr 2014 16:14:10 -0000 1.12
***************
*** 72,92 ****
=cut
use strict;
use DateTime;
use IO::Scalar;
! use LWP::UserAgent;
! use XML::LibXML;
use XMLTV::Ask qw/say/;
use XMLTV::Configure::Writer;
use XMLTV::Options qw/ParseOptions/;
sub t;
# Constants.
! my $DATA_SITE_ROOT = 'http://svenska.yle.fi/programguide/';
my $GRABBER_NAME = 'tv_grab_fi_sv';
! my $GRABBER_VERSION = '0.99';
! my $XML_ENCODING = 'windows-1252';
my $LANGUAGE_CODE = 'sv';
--- 72,99 ----
=cut
+ my $_version = '$Id$';
+
use strict;
use DateTime;
use IO::Scalar;
! #use LWP::UserAgent;
! #use XML::LibXML;
! use HTML::TreeBuilder;
use XMLTV::Ask qw/say/;
use XMLTV::Configure::Writer;
use XMLTV::Options qw/ParseOptions/;
+ use XMLTV::Get_nice 0.005066;
+ use XMLTV::Memoize;
+ use Encode;
sub t;
# Constants.
! # my $DATA_SITE_ROOT = 'http://svenska.yle.fi/programguide/';
! my $DATA_SITE_ROOT = 'http://ohjelmaopas.yle.fi/';
my $GRABBER_NAME = 'tv_grab_fi_sv';
! (my $GRABBER_VERSION) = $_version =~ qr/$GRABBER_NAME,v ([\d\.]*) /;
! my $XML_ENCODING = 'iso-8859-1';
my $LANGUAGE_CODE = 'sv';
***************
*** 106,111 ****
# Set up LWP::UserAgent
! my $ua = LWP::UserAgent->new;
! $ua->agent("xmltv/$XMLTV::VERSION");
# The list of channels available from the Yle Program Guide. Their
--- 113,121 ----
# Set up LWP::UserAgent
! # my $ua = LWP::UserAgent->new;
! # $ua->agent("xmltv/$XMLTV::VERSION");
!
! # Get options, including undocumented --cache option.
! XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
# The list of channels available from the Yle Program Guide. Their
***************
*** 121,132 ****
'tv1.yle.fi' => {
'id' => 'tv1.yle.fi',
! 'group' => 2,
'display-name' => [[ 'YLE TV1', $LANGUAGE_CODE ]]
},
'tv2.yle.fi' => {
'id' => 'tv2.yle.fi',
! 'group' => 2,
'display-name' => [[ 'YLE TV2', $LANGUAGE_CODE ]]
},
'mtv3.yle.fi' => {
'id' => 'mtv3.yle.fi',
--- 131,157 ----
'tv1.yle.fi' => {
'id' => 'tv1.yle.fi',
! 'group' => 3,
'display-name' => [[ 'YLE TV1', $LANGUAGE_CODE ]]
},
'tv2.yle.fi' => {
'id' => 'tv2.yle.fi',
! 'group' => 3,
'display-name' => [[ 'YLE TV2', $LANGUAGE_CODE ]]
},
+ 'fem.yle.fi' => {
+ 'id' => 'fem.yle.fi',
+ 'group' => 3,
+ 'display-name' => [[ 'YLE Fem', $LANGUAGE_CODE ]]
+ },
+ 'teema.yle.fi' => {
+ 'id' => 'teema.yle.fi',
+ 'group' => 3,
+ 'display-name' => [[ 'YLE Teema', $LANGUAGE_CODE ]]
+ },
+ 'areena.yle.fi' => {
+ 'id' => 'areena.yle.fi',
+ 'group' => 3,
+ 'display-name' => [[ 'Areena', $LANGUAGE_CODE ]]
+ },
'mtv3.yle.fi' => {
'id' => 'mtv3.yle.fi',
***************
*** 137,151 ****
'id' => 'nelonen.yle.fi',
'group' => 3,
! 'display-name' => [[ 'Nelonen (Fyran)', $LANGUAGE_CODE ]]
! },
! 'teema.yle.fi' => {
! 'id' => 'teema.yle.fi',
! 'group' => 2,
! 'display-name' => [[ 'YLE Teema', $LANGUAGE_CODE ]]
! },
! 'fst5.yle.fi' => {
! 'id' => 'fst5.yle.fi',
! 'group' => 2,
! 'display-name' => [[ 'YLE FST5', $LANGUAGE_CODE ]]
},
'subtv.yle.fi' => {
--- 162,166 ----
'id' => 'nelonen.yle.fi',
'group' => 3,
! 'display-name' => [[ 'Nelonen', $LANGUAGE_CODE ]]
},
'subtv.yle.fi' => {
***************
*** 154,157 ****
--- 169,177 ----
'display-name' => [[ 'Subtv', $LANGUAGE_CODE ]]
},
+ 'liv.yle.fi' => {
+ 'id' => 'liv.yle.fi',
+ 'group' => 3,
+ 'display-name' => [[ 'LIV', $LANGUAGE_CODE ]]
+ },
'jim.yle.fi' => {
'id' => 'jim.yle.fi',
***************
*** 159,176 ****
'display-name' => [[ 'JIM', $LANGUAGE_CODE ]]
},
!
! # The Swedish YLE web page is a bit outdated when it comes to this
! # channel; it was renamed to Nelonen Sport (4 Sport) a while
! # back... :-)
! 'nelonen.sport.yle.fi' => {
! 'id' => 'nelonen.sport.yle.fi',
'group' => 3,
! 'display-name' => [[ 'JIM', $LANGUAGE_CODE ]]
! }
};
# Map between channel names (as presented by the YLE data) and channel
# IDs, as create by us.
! my $channel_name_map =
{
'TV1' => 'tv1.yle.fi',
--- 179,193 ----
'display-name' => [[ 'JIM', $LANGUAGE_CODE ]]
},
! 'ava.yle.fi' => {
! 'id' => 'ava.yle.fi',
'group' => 3,
! 'display-name' => [[ 'AVA', $LANGUAGE_CODE ]]
! },
};
# Map between channel names (as presented by the YLE data) and channel
# IDs, as create by us.
!
! my $channel_name_map = # deprecated
{
'TV1' => 'tv1.yle.fi',
***************
*** 186,189 ****
--- 203,220 ----
'Urheilukanava' => 'nelonen.sport.yle.fi'
};
+ my $channel_id_map =
+ {
+ 'yle-tv1' => 'tv1.yle.fi',
+ 'yle-tv2' => 'tv2.yle.fi',
+ 'yle-fem' => 'fem.yle.fi',
+ 'yle-teema' => 'teema.yle.fi',
+ 'yle-areena' => 'areena.yle.fi',
+ 'MTV3' => 'mtv3.yle.fi',
+ 'Nelonen' => 'nelonen.yle.fi',
+ 'SUB' => 'subtv.yle.fi',
+ 'Liv' => 'liv.yle.fi',
+ 'Jim' => 'jim.yle.fi',
+ 'AVA' => 'ava.yle.fi',
+ };
my @ARGUMENTS = @ARGV;
***************
*** 196,200 ****
stage_sub => \&config_stage,
listchannels_sub => \&list_channels,
! version => '$Id$',
description => "Finland (Swedish)",
});
--- 227,231 ----
stage_sub => \&config_stage,
listchannels_sub => \&list_channels,
! version => $_version,
description => "Finland (Swedish)",
});
***************
*** 377,385 ****
my (@channels) = @_;
! if (!$opt->{quiet})
! {
! my $number_of_channels = scalar @channels;
! say("Will download listings for $number_of_channels configured channels\n");
! }
my $channel_groups = {};
--- 408,416 ----
my (@channels) = @_;
! my $programmes = {};
!
! say(scalar @channels ." configured channels") if !$opt->{quiet};
!
! # 2014-04-13 channel groups are deprecated
my $channel_groups = {};
***************
*** 407,414 ****
if (!$opt->{quiet})
{
! say("Downloading data for channel group $channel_group");
}
! for (my $i = $opt->{offset}; $i < $opt->{offset} + $opt->{days}; $i++)
{
# Create the URL for the schedules for this
--- 438,455 ----
if (!$opt->{quiet})
{
! # say("Downloading data for channel group $channel_group");
}
! # Get start & stop times for the grab
! my $t = $today->clone->add( days => $opt->{offset} );
! my $grab_start = $t->epoch() + $t->offset();
! $t = $t->add( days => $opt->{days} );
! my $grab_end = $t->epoch() + $t->offset();
! undef $t;
!
!
! # schedules run from 06:00-06:00 so to pass tv_validate_file we need to get the day before also
!
! for (my $i = $opt->{offset} - 1; $i < $opt->{offset} + $opt->{days}; $i++)
{
# Create the URL for the schedules for this
***************
*** 416,546 ****
my $date = $today->clone->add( days => $i );
! my $url = sprintf("%s?g=%s&d=%s", $DATA_SITE_ROOT,
! $channel_group, $date->strftime( '%Y%m%d' ));
!
! if ($opt->{debug})
! {
! say("Downloading $url");
! }
!
! # Get the HTML from the created URL.
! my $response = $ua->get($url);
! my $file_contents = $response->decoded_content;
! # Grab the table with program contents from the HTML.
! my ($program_contents) =
! ($file_contents =~ m%(<table id="programmes".+?>.+?</table>)%s);
! # We do the parsing with the XML::LibXML() library. I
! # prefer doing it like this, rather than setting up a
! # bunch of pretty silly regexps.
! my $xml = new XML::LibXML();
! # We need to enable the 'recover' option here, since the
! # data is known to contain things like standalone
! # ampersands, for example, which is not allowed in an XML
! # document.
! my $doc = $xml->parse_html_string($program_contents,
! {
! 'recover' => 1,
! 'suppress_warnings' => 1,
! 'suppress_errors' => 1
! });
! my $programs = $doc->find('//div[starts-with(@class, "programme")]');
! foreach my $program ($programs->get_nodelist())
! {
! # Get all the nodes for this program. Everything we
! # need is found below the "description" div, so we use
! # this as the foundation here.
! my @programNodes =
! $program->find('div[@class="description"]')->[0]->childNodes();
! my $description = "";
! my $title = "";
! my $subtitle = "";
! my $desc_time = "";
! # Loop over the program nodes and grab out the data
! # from the relevant nodes.
! foreach my $programNode (@programNodes)
! {
! my $nodeType = ref($programNode);
- if ($nodeType eq 'XML::LibXML::Text')
- {
- $description = $programNode->data;
- }
- else
- {
- my $class = $programNode->getAttribute("class");
! if (defined($class))
! {
! if ($class eq 'desc_title')
! {
! # We special-case certain programs to
! # be able to get a title +
! # subtitle. This makes them much
! # easier to record using PVR software
! # for example.
! if ($programNode->to_literal =~ m/^BUU-klubben: .+$/ ||
! $programNode->to_literal =~ m/^Tintin: .+$/)
! {
! ($title, $subtitle) = $programNode->to_literal =~ m/^(.+?): (.+)$/;
}
! else
{
! $title = $programNode->to_literal;
}
}
- elsif ($class eq 'desc_time')
- {
- # We don't call this variable just
- # "time" since its format looks like
- # this:
- #
- # MTV3 01.05 - 02.05
- $desc_time = $programNode->to_literal;
}
}
}
}
-
- my ($channel_name, $start_time, $end_time) =
- ($desc_time) =~ m/^(.+?) (\d{2}\.\d{2}) - (\d{2}\.\d{2})$/;
-
- # Check if this program belongs to one of the
- # configured channels. If it doesn't, ignore
- # it. (Since the data is provided to us in channel
- # groups, we cannot do it much better than this.)
- my $channel_id = $channel_name_map->{$channel_name};
- unless( exists( $wanted{ $channel_id } ) )
- {
- next;
}
- # Create the data structure for the program.
- my $program =
- {
- 'channel' => $channel_id,
- 'title' => [[ $title, $LANGUAGE_CODE ]],
- 'start' => xmltv_time($date, $start_time),
- 'stop' => xmltv_end_time($date, $start_time,
- $end_time)
- };
-
- $program->{'desc'} = [[ $description, $LANGUAGE_CODE ]] if ($description ne '');
- $program->{'sub-title'} = [[ $subtitle, $LANGUAGE_CODE ]] if ($subtitle ne '');
# All data has been gathered. We can now write the
! # program element to the output. (This could be
! # changed into a "two-pass" operation if wanted, so
! # that all the data is gathered first and written to
! # the output afterwards.)
! $writer->write_programme($program);
}
}
}
}
--- 457,593 ----
my $date = $today->clone->add( days => $i );
! # broken: my $url = sprintf("%s?g=%s&d=%s", $DATA_SITE_ROOT, $channel_group, $date->strftime( '%Y%m%d' ));
! my $url = sprintf('%stv/guide?t=%s', $DATA_SITE_ROOT, $date->strftime( '%Y-%m-%d' )); # e.g. http://ohjelmaopas.yle.fi/tv/guide?t=2014-04-14
! say("Downloading $url") if $opt->{debug};
! # Get the HTML from the created URL.
! # my $response = $ua->get($url);
! # my $file_contents = $response->decoded_content;
! # ^^ change to use Get_nice
! #
! $XMLTV::Get_nice::IncludeUnknownTags = 1;
! my $tree = get_nice_tree($url,'','UTF-8');
! #$tree->dump; exit;
!
! next if !$tree;
! my $t_guide = $tree->look_down('_tag' => 'section', 'class' => qr/guide/);
! next if !$t_guide;
! my $t_chanlists = $t_guide->look_down('_tag' => 'ul', 'class' => qr/channel-lists/);
! next if !$t_chanlists;
! my @t_channels = $t_chanlists->look_down('_tag' => 'li', sub {$_[0]->parent() == $t_chanlists} );
! say 'Found '.scalar @t_channels.' channels' if $opt->{debug};
! next if scalar @t_channels == 0;
! for (my $j = 0; $j < scalar @t_channels; $j++) { # can't use foreach because of clumpidx processing
! my $t_channel = $t_channels[$j];
! my $chan_id = $t_channel->look_down('_tag' => 'h1')->attr('id');
! my $c_chan_name = $t_channel->look_down('_tag' => 'h1')->as_text();
! # Check if this program belongs to one of the
! # configured channels. If it doesn't, ignore it.
! my $c_channel_id = $channel_id_map->{$chan_id};
! next unless( exists( $wanted{ $c_channel_id } ) );
! say "Processing $c_channel_id" if $opt->{debug};
! my $t_schedule = $t_channel->look_down('_tag' => 'ul', 'class' => qr/selected/);
! my @t_progs = $t_schedule->look_down('_tag' => 'li', 'class' => qr/program-entry/);
!
! foreach my $t_prog (@t_progs) {
! my ($t_prog_label, $t_prog_desc, $t_prog_link, $t_prog_title, $t_prog_film);
! my ($p_dtstart, $p_dtend, $p_title, $p_subtitle, $p_desc, $p_url, $p_category);
!
! $t_prog_label = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-label/);
! $t_prog_desc = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-desc/);
!
!
! # If programme is outside of requested timeframe then drop it
! my $p_start = $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime');
! $p_start = epoch_isotime($p_start);
! next if $p_start < $grab_start || $p_start >= $grab_end;
!
!
! $p_dtstart = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime') );
! $p_dtend = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtend/)->attr('datetime') );
!
! # it seems the stop times aren't to be trusted
! # - on Sub tv they seem to be the actual stop times of the programme (i.e. start + duration), i.e. excludes trailers/ads before next prog (so causing gaps)
! # - on Areena and Ava they're often 1 or 2 minutes *after* the start of the next prog (so causing overlaps).
! undef $p_dtend;
!
! $t_prog_title = $t_prog_label->look_down('_tag' => 'div', 'class' => qr/program-title/);
! $p_title = $t_prog_title->look_down('_tag' => 'a', 'class' => qr/link-grey/)->as_text();
!
! # the <div class="program-title"> may contain <span class="movie">Film</span> if prog is a film
! if ( $t_prog_film = $t_prog_title->look_down('_tag' => 'span', 'class' => qr/movie/) ) {
! $p_category = $t_prog_film->as_text();
}
!
! $p_desc = $t_prog_desc->look_down('_tag' => 'p')->as_text(); # desc is in first <p>
! $t_prog_link = $t_prog_desc->look_down('_tag' => 'a', 'class' => qr/double-angle/, sub { return $_[0]->as_text() =~ /Programinformation/ });
! $p_url = $t_prog_link->attr('href') if $t_prog_link;
!
!
! # Create the data structure for the program.
! my $program =
{
! 'channel' => $c_channel_id,
! 'title' => [[ encode('iso-8859-1', $p_title), $LANGUAGE_CODE ]],
! 'start' => $p_dtstart
! };
! $program->{'stop'} = $p_dtend if (defined $p_dtend && $p_dtend ne '');
! $program->{'desc'} = [[ encode('iso-8859-1', $p_desc), $LANGUAGE_CODE ]] if (defined $p_desc && $p_desc ne '');
! $program->{'sub-title'} = [[ encode('iso-8859-1', $p_subtitle), $LANGUAGE_CODE ]] if (defined $p_subtitle && $p_subtitle ne '');
! $program->{'category'} = [[ encode('iso-8859-1', $p_category) ]] if (defined $p_category && $p_category ne '');
! $program->{'url'} = [ $p_url ] if (defined $p_url && $p_url ne '');
!
!
! # store the programme avoiding duplicates
! # also check for deuplicate start times and set clumpidx
! if ( defined $programmes->{$c_channel_id}->{$p_dtstart} ) {
! # duplicate prog or contemporary?
! my $dup = 0;
! foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
! $dup = 1 if ( $p->{'title'}[0][0] eq $program->{'title'}[0][0] ); # duplicate
! }
! next if $dup; # ignore duplicates
! if (!$dup) {
! # contemporary programme so set clumpidx
! my $numclumps = scalar @{ $programmes->{$c_channel_id}->{$p_dtstart} } + 1;
! # set (or adjust) clumpidx of existing programmes
! my $i = 0;
! foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
! $p->{'clumpidx'} = "$i/$numclumps";
! $i++;
}
+ # set clumpidx for new programme
+ $program->{'clumpidx'} = "$i/$numclumps";
}
}
+
+ # store the programme
+ push @{ $programmes->{$c_channel_id}->{$p_dtstart} }, $program;
}
}
}
}
# All data has been gathered. We can now write the
! # program element to the output.
! #
! foreach ( keys %{$programmes} ) {
! my $ch_progs = $programmes->{$_};
! foreach ( sort keys %{$ch_progs} ) {
! my $dt_progs = $ch_progs->{$_};
! foreach (@{ $dt_progs }) {
! $writer->write_programme($_);
}
}
}
+
}
***************
*** 617,618 ****
--- 664,697 ----
return xmltv_time($date, $end_time);
}
+
+
+ # Converts a DateTime + time of the form "2014-04-12T09:00:00.000+03:00" to something suitable
+ # for XMLTV, i.e. 201404120900 +0300
+ sub xmltv_isotime ($)
+ {
+ my $time = shift;
+
+ # let's not overthink this... just use a regexp!
+
+ $time =~ s/[:-]//g;
+ $time =~ /^(\d{8})T(\d{4}).*(\+\d{4})$/;
+ return $1.$2.' '.$3;
+ }
+
+ # Converts a DateTime + time of the form "2014-04-12T09:00:00.000+03:00" to epoch time (local or utc)
+ sub epoch_isotime ($;$)
+ {
+ my $time = shift;
+ my $withoffset = shift; # set to 1 to generate local epoch (else will get utc)
+ my ($y, $m, $d, $h, $i, $s, $ms, $t, $th, $tm) = $time =~ /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d{3})([\+-])(\d{2}):(\d{2})$/;
+ $y -= 1900;
+ $m -= 1;
+ use POSIX;
+ my $epoch = mktime($s, $i, $h, $d, $m, $y);
+ if (defined $withoffset && $withoffset) {
+ my $offset = ($th * 3600) + ($tm * 60);
+ $epoch += $offset if $t eq '+';
+ $epoch -= $offset if $t eq '-';
+ }
+ return $epoch;
+ }
Index: test.conf
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi_sv/test.conf,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** test.conf 17 Nov 2010 23:10:00 -0000 1.2
--- test.conf 13 Apr 2014 16:14:10 -0000 1.3
***************
*** 1,9 ****
channel=subtv.yle.fi
! channel=fst5.yle.fi
! channel!teema.yle.fi
channel=jim.yle.fi
channel=tv2.yle.fi
channel=nelonen.yle.fi
! channel=nelonen.sport.yle.fi
channel=tv1.yle.fi
channel=mtv3.yle.fi
--- 1,11 ----
channel=subtv.yle.fi
! channel=teema.yle.fi
channel=jim.yle.fi
channel=tv2.yle.fi
channel=nelonen.yle.fi
! channel=fem.yle.fi
! channel=areena.yle.fi
channel=tv1.yle.fi
+ channel=ava.yle.fi
+ channel=liv.yle.fi
channel=mtv3.yle.fi
------------------------------------------------------------------------------
Put Bad Developers to Shame
Dominate Development with Jenkins Continuous Integration
Continuously Automate Build, Test & Deployment
Start a new project now. Try Jenkins in the cloud.
http://p.sf.net/sfu/13600_Cloudbees