xmltv/grab/fi_sv tv_grab_fi_sv,1.17,1.18
Nick Morrott via xmltv-commit <[email protected]> Thu, 07 Sep 2017 11:02:58 +0000
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/grab/fi_sv
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv4773/grab/fi_sv
Modified Files:
tv_grab_fi_sv
Log Message:
Whitespace changes only
* fix broken indentation
* remove trailing whitespace
Index: tv_grab_fi_sv
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi_sv/tv_grab_fi_sv,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** tv_grab_fi_sv 20 Jun 2015 11:28:15 -0000 1.17
--- tv_grab_fi_sv 7 Sep 2017 11:02:56 -0000 1.18
***************
*** 10,14 ****
tv_grab_fi_sv --help
!
tv_grab_fi_sv --version
--- 10,14 ----
tv_grab_fi_sv --help
!
tv_grab_fi_sv --version
***************
*** 73,77 ****
my $_version = '$Id$';
!
use strict;
--- 73,77 ----
my $_version = '$Id$';
!
use strict;
***************
*** 106,113 ****
my $xmltv_attributes =
{
! 'source-info-url' => 'http://www.yle.fi/',
! 'source-data-url' => "$DATA_SITE_ROOT/",
! 'generator-info-name' => "XMLTV/$XMLTV::VERSION, $GRABBER_NAME $GRABBER_VERSION",
! 'generator-info-url' => 'http://www.xmltv.org',
};
--- 106,113 ----
my $xmltv_attributes =
{
! 'source-info-url' => 'http://www.yle.fi/',
! 'source-data-url' => "$DATA_SITE_ROOT/",
! 'generator-info-name' => "XMLTV/$XMLTV::VERSION, $GRABBER_NAME $GRABBER_VERSION",
! 'generator-info-url' => 'http://www.xmltv.org',
};
***************
*** 129,192 ****
my $channels =
{
! 'tv1.yle.fi' => {
! 'id' => 'tv1.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE TV1', $LANGUAGE_CODE ]]
! },
! 'tv2.yle.fi' => {
! 'id' => 'tv2.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE TV2', $LANGUAGE_CODE ]]
! },
! 'fem.yle.fi' => {
! 'id' => 'fem.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE Fem', $LANGUAGE_CODE ]]
! },
! 'teema.yle.fi' => {
! 'id' => 'teema.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE Teema', $LANGUAGE_CODE ]]
! },
! 'elava-arkisto.yle.fi' => {
! 'id' => 'elava-arkisto.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Elava Arkisto', $LANGUAGE_CODE ]]
! },
! 'areena.yle.fi' => {
! 'id' => 'areena.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Areena', $LANGUAGE_CODE ]]
! },
! 'mtv3.yle.fi' => {
! 'id' => 'mtv3.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'MTV3', $LANGUAGE_CODE ]]
! },
! 'nelonen.yle.fi' => {
! 'id' => 'nelonen.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Nelonen', $LANGUAGE_CODE ]]
! },
! 'subtv.yle.fi' => {
! 'id' => 'subtv.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Subtv', $LANGUAGE_CODE ]]
! },
! 'liv.yle.fi' => {
! 'id' => 'liv.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'LIV', $LANGUAGE_CODE ]]
! },
! 'jim.yle.fi' => {
! 'id' => 'jim.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'JIM', $LANGUAGE_CODE ]]
! },
! 'ava.yle.fi' => {
! 'id' => 'ava.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'AVA', $LANGUAGE_CODE ]]
! },
};
--- 129,192 ----
my $channels =
{
! 'tv1.yle.fi' => {
! 'id' => 'tv1.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE TV1', $LANGUAGE_CODE ]]
! },
! 'tv2.yle.fi' => {
! 'id' => 'tv2.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE TV2', $LANGUAGE_CODE ]]
! },
! 'fem.yle.fi' => {
! 'id' => 'fem.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE Fem', $LANGUAGE_CODE ]]
! },
! 'teema.yle.fi' => {
! 'id' => 'teema.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'YLE Teema', $LANGUAGE_CODE ]]
! },
! 'elava-arkisto.yle.fi' => {
! 'id' => 'elava-arkisto.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Elava Arkisto', $LANGUAGE_CODE ]]
! },
! 'areena.yle.fi' => {
! 'id' => 'areena.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Areena', $LANGUAGE_CODE ]]
! },
! 'mtv3.yle.fi' => {
! 'id' => 'mtv3.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'MTV3', $LANGUAGE_CODE ]]
! },
! 'nelonen.yle.fi' => {
! 'id' => 'nelonen.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Nelonen', $LANGUAGE_CODE ]]
! },
! 'subtv.yle.fi' => {
! 'id' => 'subtv.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'Subtv', $LANGUAGE_CODE ]]
! },
! 'liv.yle.fi' => {
! 'id' => 'liv.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'LIV', $LANGUAGE_CODE ]]
! },
! 'jim.yle.fi' => {
! 'id' => 'jim.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'JIM', $LANGUAGE_CODE ]]
! },
! 'ava.yle.fi' => {
! 'id' => 'ava.yle.fi',
! 'group' => 3,
! 'display-name' => [[ 'AVA', $LANGUAGE_CODE ]]
! },
};
***************
*** 194,226 ****
# IDs, as create by us.
! my $channel_name_map = # deprecated
{
! 'TV1' => 'tv1.yle.fi',
! 'TV2' => 'tv2.yle.fi',
! 'MTV3' => 'mtv3.yle.fi',
! 'Nelonen' => 'nelonen.yle.fi',
! 'YLE Teema' => 'teema.yle.fi',
! 'FST5' => 'fst5.yle.fi',
! 'Subtv' => 'subtv.yle.fi',
! 'JIM' => 'jim.yle.fi',
! # See note above about this one.
! 'Urheilukanava' => 'nelonen.sport.yle.fi'
};
my $channel_id_map =
{
! 'yle-tv1' => 'tv1.yle.fi',
! 'yle-tv2' => 'tv2.yle.fi',
! 'yle-fem' => 'fem.yle.fi',
! 'yle-teema' => 'teema.yle.fi',
! 'yle-areena' => 'areena.yle.fi',
! 'yle-elava-arkisto' => 'elava-arkisto.yle.fi',
! 'MTV3' => 'mtv3.yle.fi',
! 'Nelonen' => 'nelonen.yle.fi',
! 'SUB' => 'subtv.yle.fi',
! 'Liv' => 'liv.yle.fi',
! 'Jim' => 'jim.yle.fi',
! 'AVA' => 'ava.yle.fi',
! 'tv-finland' => 'tvfinland.yle.fi',
};
--- 194,226 ----
# IDs, as create by us.
! my $channel_name_map = # deprecated
{
! 'TV1' => 'tv1.yle.fi',
! 'TV2' => 'tv2.yle.fi',
! 'MTV3' => 'mtv3.yle.fi',
! 'Nelonen' => 'nelonen.yle.fi',
! 'YLE Teema' => 'teema.yle.fi',
! 'FST5' => 'fst5.yle.fi',
! 'Subtv' => 'subtv.yle.fi',
! 'JIM' => 'jim.yle.fi',
! # See note above about this one.
! 'Urheilukanava' => 'nelonen.sport.yle.fi'
};
my $channel_id_map =
{
! 'yle-tv1' => 'tv1.yle.fi',
! 'yle-tv2' => 'tv2.yle.fi',
! 'yle-fem' => 'fem.yle.fi',
! 'yle-teema' => 'teema.yle.fi',
! 'yle-areena' => 'areena.yle.fi',
! 'yle-elava-arkisto' => 'elava-arkisto.yle.fi',
! 'MTV3' => 'mtv3.yle.fi',
! 'Nelonen' => 'nelonen.yle.fi',
! 'SUB' => 'subtv.yle.fi',
! 'Liv' => 'liv.yle.fi',
! 'Jim' => 'jim.yle.fi',
! 'AVA' => 'ava.yle.fi',
! 'tv-finland' => 'tvfinland.yle.fi',
};
***************
*** 230,239 ****
my ($opt, $conf) = ParseOptions(
{
! grabber_name => "tv_grab_fi_sv",
! capabilities => [qw/baseline manualconfig apiconfig/],
! stage_sub => \&config_stage,
! listchannels_sub => \&list_channels,
! version => $_version,
! description => "Finland (Swedish)",
});
--- 230,239 ----
my ($opt, $conf) = ParseOptions(
{
! grabber_name => "tv_grab_fi_sv",
! capabilities => [qw/baseline manualconfig apiconfig/],
! stage_sub => \&config_stage,
! listchannels_sub => \&list_channels,
! version => $_version,
! description => "Finland (Swedish)",
});
***************
*** 263,280 ****
sub config_stage
{
! my($stage, $conf) = shift;
! die "Unknown stage $stage" if $stage ne "start";
! # This grabber doesn't need any configuration (except for
! # possibly channel, selection), so this subroutine doesn't need
! # to do very much at all.
! my $result;
! my $writer = new XMLTV::Configure::Writer(OUTPUT => \$result,
! encoding => $XML_ENCODING);
! $writer->start({ grabber => 'tv_grab_fi_sv' });
! $writer->end('select-channels');
! return $result;
}
--- 263,280 ----
sub config_stage
{
! my($stage, $conf) = shift;
! die "Unknown stage $stage" if $stage ne "start";
! # This grabber doesn't need any configuration (except for
! # possibly channel, selection), so this subroutine doesn't need
! # to do very much at all.
! my $result;
! my $writer = new XMLTV::Configure::Writer(OUTPUT => \$result,
! encoding => $XML_ENCODING);
! $writer->start({ grabber => 'tv_grab_fi_sv' });
! $writer->end('select-channels');
! return $result;
}
***************
*** 283,312 ****
sub list_channels
{
! my ($conf, $opt) = shift;
! my $result = '';
! my $fh = new IO::Scalar \$result;
! my $oldfh = select($fh);
! # Create an XMLTV::Writer object. The important part here is that
! # the output should go to $fh (in other words, to the $result
! # string), NOT to stdout...
! my %writer_args =
! (
! encoding => $XML_ENCODING,
! OUTPUT => $fh
! );
my $writer = new XMLTV::Writer(%writer_args);
! $writer->start($xmltv_attributes);
# Loop over all channels and write them to this XMLTV::Writer.
foreach my $channel_id (keys %{ $channels })
{
! # We must remove our proprietary hash key here, otherwise
! # the XMLTV module will bark at us...
! my $channel = $channels->{$channel_id};
! delete($channel->{group});
!
$writer->write_channel($channel);
}
--- 283,312 ----
sub list_channels
{
! my ($conf, $opt) = shift;
! my $result = '';
! my $fh = new IO::Scalar \$result;
! my $oldfh = select($fh);
! # Create an XMLTV::Writer object. The important part here is that
! # the output should go to $fh (in other words, to the $result
! # string), NOT to stdout...
! my %writer_args =
! (
! encoding => $XML_ENCODING,
! OUTPUT => $fh
! );
my $writer = new XMLTV::Writer(%writer_args);
! $writer->start($xmltv_attributes);
# Loop over all channels and write them to this XMLTV::Writer.
foreach my $channel_id (keys %{ $channels })
{
! # We must remove our proprietary hash key here, otherwise
! # the XMLTV module will bark at us...
! my $channel = $channels->{$channel_id};
! delete($channel->{group});
!
$writer->write_channel($channel);
}
***************
*** 366,370 ****
{
say("Specified --offset option is not possible (0-13). "
! . "Retrieving all available listings.");
}
$opt->{offset} = 0;
--- 366,370 ----
{
say("Specified --offset option is not possible (0-13). "
! . "Retrieving all available listings.");
}
$opt->{offset} = 0;
***************
*** 396,405 ****
foreach my $channel_id (@channels)
{
! # We must remove our proprietary hash key here, otherwise the
! # XMLTV module will bark at us...
! my $channel = $channels->{$channel_id};
! my $group = $channel->{group};
! delete($channel->{group});
!
$writer->write_channel($channel);
--- 396,405 ----
foreach my $channel_id (@channels)
{
! # We must remove our proprietary hash key here, otherwise the
! # XMLTV module will bark at us...
! my $channel = $channels->{$channel_id};
! my $group = $channel->{group};
! delete($channel->{group});
!
$writer->write_channel($channel);
***************
*** 415,423 ****
my (@channels) = @_;
! my $programmes = {};
say(scalar @channels ." configured channels") if !$opt->{quiet};
! # 2014-04-13 channel groups are deprecated
my $channel_groups = {};
--- 415,423 ----
my (@channels) = @_;
! my $programmes = {};
say(scalar @channels ." configured channels") if !$opt->{quiet};
! # 2014-04-13 channel groups are deprecated
my $channel_groups = {};
***************
*** 428,461 ****
foreach my $channel (@channels)
{
! # Get the channel group for this channel and check if the group has already
! # been fetched.
! my $channel_group = $channels->{$channel}->{group};
!
! if (defined($channel_groups->{$channel_group}))
! {
! # Group has been fetched already - ignoring it.
! next;
! }
! # Mark the group as fetched.
! $channel_groups->{$channel_group} = 1;
! my $today = DateTime->today( time_zone => $TIMEZONE );
if (!$opt->{quiet})
{
! # say("Downloading data for channel group $channel_group");
}
! # Get start & stop times for the grab
! my $t = $today->clone->add( days => $opt->{offset} );
! my $grab_start = $t->epoch() + $t->offset();
! $t = $t->add( days => $opt->{days} );
! my $grab_end = $t->epoch() + $t->offset();
! undef $t;
!
!
! # schedules run from 06:00-06:00 so to pass tv_validate_file we need to get the day before also
!
for (my $i = $opt->{offset} - 1; $i < $opt->{offset} + $opt->{days}; $i++)
{
--- 428,461 ----
foreach my $channel (@channels)
{
! # Get the channel group for this channel and check if the group has already
! # been fetched.
! my $channel_group = $channels->{$channel}->{group};
! if (defined($channel_groups->{$channel_group}))
! {
! # Group has been fetched already - ignoring it.
! next;
! }
! # Mark the group as fetched.
! $channel_groups->{$channel_group} = 1;
!
! my $today = DateTime->today( time_zone => $TIMEZONE );
if (!$opt->{quiet})
{
! # say("Downloading data for channel group $channel_group");
}
! # Get start & stop times for the grab
! my $t = $today->clone->add( days => $opt->{offset} );
! my $grab_start = $t->epoch() + $t->offset();
! $t = $t->add( days => $opt->{days} );
! my $grab_end = $t->epoch() + $t->offset();
! undef $t;
!
!
! # schedules run from 06:00-06:00 so to pass tv_validate_file we need to get the day before also
!
for (my $i = $opt->{offset} - 1; $i < $opt->{offset} + $opt->{days}; $i++)
{
***************
*** 473,607 ****
# my $response = $ua->get($url);
# my $file_contents = $response->decoded_content;
! # ^^ change to use Get_nice
! #
! $XMLTV::Get_nice::IncludeUnknownTags = 1;
my $tree = get_nice_tree($url,'','UTF-8');
! #$tree->dump; exit;
!
! next if !$tree;
! my $t_guide = $tree->look_down('_tag' => 'section', 'class' => qr/guide/);
! next if !$t_guide;
! my $t_chanlists = $t_guide->look_down('_tag' => 'ul', 'class' => qr/channel-lists/);
! next if !$t_chanlists;
! my @t_channels = $t_chanlists->look_down('_tag' => 'li', sub {$_[0]->parent() == $t_chanlists} );
! say 'Found '.scalar @t_channels.' channels' if $opt->{debug};
! next if scalar @t_channels == 0;
! for (my $j = 0; $j < scalar @t_channels; $j++) { # can't use foreach because of clumpidx processing
! my $t_channel = $t_channels[$j];
! my $chan_id = $t_channel->look_down('_tag' => 'h1')->attr('id');
! my $c_chan_name = $t_channel->look_down('_tag' => 'h1')->as_text();
! # Check if this program belongs to one of the
! # configured channels. If it doesn't, ignore it.
! say "Found $chan_id" if $opt->{debug};
! my $c_channel_id = $channel_id_map->{$chan_id};
! say "UNKNOWN CHANNEL ID $chan_id" if $opt->{debug} && !$c_channel_id;
! next unless( $c_channel_id && exists( $wanted{ $c_channel_id } ) );
! say "Processing $c_channel_id" if $opt->{debug};
- my $t_schedule = $t_channel->look_down('_tag' => 'ul', 'class' => qr/selected/);
- my @t_progs = $t_schedule->look_down('_tag' => 'li', 'class' => qr/program-entry/);
-
- foreach my $t_prog (@t_progs) {
- my ($t_prog_label, $t_prog_desc, $t_prog_link, $t_prog_title, $t_prog_film);
- my ($p_dtstart, $p_dtend, $p_title, $p_subtitle, $p_desc, $p_url, $p_category);
-
- $t_prog_label = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-label/);
- $t_prog_desc = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-desc/);
-
-
- # If programme is outside of requested timeframe then drop it
- my $p_start = $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime');
- $p_start = epoch_isotime($p_start);
- next if $p_start < $grab_start || $p_start >= $grab_end;
-
-
- $p_dtstart = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime') );
- $p_dtend = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtend/)->attr('datetime') );
-
- # it seems the stop times aren't to be trusted
- # - on Sub tv they seem to be the actual stop times of the programme (i.e. start + duration), i.e. excludes trailers/ads before next prog (so causing gaps)
- # - on Areena and Ava they're often 1 or 2 minutes *after* the start of the next prog (so causing overlaps).
- undef $p_dtend;
-
- $t_prog_title = $t_prog_label->look_down('_tag' => 'div', 'class' => qr/program-title/);
- $p_title = $t_prog_title->look_down('_tag' => 'a', 'class' => qr/link-grey/)->as_text();
-
- # the <div class="program-title"> may contain <span class="movie">Film</span> if prog is a film
- if ( $t_prog_film = $t_prog_title->look_down('_tag' => 'span', 'class' => qr/movie/) ) {
- $p_category = $t_prog_film->as_text();
- }
-
- $p_desc = $t_prog_desc->look_down('_tag' => 'p')->as_text(); # desc is in first <p>
- $t_prog_link = $t_prog_desc->look_down('_tag' => 'a', 'class' => qr/double-angle/, sub { return $_[0]->as_text() =~ /Programinformation/ });
- $p_url = $t_prog_link->attr('href') if $t_prog_link;
-
-
- # occasionally it seems there's a 'null' programme (usually with overlapping time)
- # I think we should drop these
- next if $p_title eq '';
-
-
- # Create the data structure for the program.
- my $program =
- {
- 'channel' => $c_channel_id,
- 'title' => [[ encode('iso-8859-1', $p_title), $LANGUAGE_CODE ]],
- 'start' => $p_dtstart
- };
- $program->{'stop'} = $p_dtend if (defined $p_dtend && $p_dtend ne '');
- $program->{'desc'} = [[ encode('iso-8859-1', tidy( $p_desc) ), $LANGUAGE_CODE ]] if (defined $p_desc && $p_desc ne '');
- $program->{'sub-title'} = [[ encode('iso-8859-1', $p_subtitle), $LANGUAGE_CODE ]] if (defined $p_subtitle && $p_subtitle ne '');
- $program->{'category'} = [[ encode('iso-8859-1', $p_category) ]] if (defined $p_category && $p_category ne '');
- $program->{'url'} = [ $p_url ] if (defined $p_url && $p_url ne '');
-
-
- # store the programme avoiding duplicates
- # also check for deuplicate start times and set clumpidx
- if ( defined $programmes->{$c_channel_id}->{$p_dtstart} ) {
- # duplicate prog or contemporary?
- my $dup = 0;
- foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
- $dup = 1 if ( $p->{'title'}[0][0] eq $program->{'title'}[0][0] ); # duplicate
- }
- next if $dup; # ignore duplicates
- if (!$dup) {
- # contemporary programme so set clumpidx
- my $numclumps = scalar @{ $programmes->{$c_channel_id}->{$p_dtstart} } + 1;
- # set (or adjust) clumpidx of existing programmes
- my $i = 0;
- foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
- $p->{'clumpidx'} = "$i/$numclumps";
- $i++;
- }
- # set clumpidx for new programme
- $program->{'clumpidx'} = "$i/$numclumps";
- }
- }
-
- # store the programme
- push @{ $programmes->{$c_channel_id}->{$p_dtstart} }, $program;
- }
- }
- }
- }
! # All data has been gathered. We can now write the
! # program element to the output.
! #
! foreach ( keys %{$programmes} ) {
! my $ch_progs = $programmes->{$_};
! foreach ( sort keys %{$ch_progs} ) {
! my $dt_progs = $ch_progs->{$_};
! foreach (@{ $dt_progs }) {
! $writer->write_programme($_);
! }
}
}
-
}
--- 473,606 ----
# my $response = $ua->get($url);
# my $file_contents = $response->decoded_content;
! # ^^ change to use Get_nice
! #
! $XMLTV::Get_nice::IncludeUnknownTags = 1;
my $tree = get_nice_tree($url,'','UTF-8');
! #$tree->dump; exit;
! next if !$tree;
! my $t_guide = $tree->look_down('_tag' => 'section', 'class' => qr/guide/);
! next if !$t_guide;
! my $t_chanlists = $t_guide->look_down('_tag' => 'ul', 'class' => qr/channel-lists/);
! next if !$t_chanlists;
! my @t_channels = $t_chanlists->look_down('_tag' => 'li', sub {$_[0]->parent() == $t_chanlists} );
! say 'Found '.scalar @t_channels.' channels' if $opt->{debug};
! next if scalar @t_channels == 0;
! for (my $j = 0; $j < scalar @t_channels; $j++) { # can't use foreach because of clumpidx processing
! my $t_channel = $t_channels[$j];
! my $chan_id = $t_channel->look_down('_tag' => 'h1')->attr('id');
! my $c_chan_name = $t_channel->look_down('_tag' => 'h1')->as_text();
+ # Check if this program belongs to one of the
+ # configured channels. If it doesn't, ignore it.
+ say "Found $chan_id" if $opt->{debug};
+ my $c_channel_id = $channel_id_map->{$chan_id};
+ say "UNKNOWN CHANNEL ID $chan_id" if $opt->{debug} && !$c_channel_id;
+ next unless( $c_channel_id && exists( $wanted{ $c_channel_id } ) );
+ say "Processing $c_channel_id" if $opt->{debug};
+ my $t_schedule = $t_channel->look_down('_tag' => 'ul', 'class' => qr/selected/);
+ my @t_progs = $t_schedule->look_down('_tag' => 'li', 'class' => qr/program-entry/);
+
+ foreach my $t_prog (@t_progs) {
+ my ($t_prog_label, $t_prog_desc, $t_prog_link, $t_prog_title, $t_prog_film);
+ my ($p_dtstart, $p_dtend, $p_title, $p_subtitle, $p_desc, $p_url, $p_category);
! $t_prog_label = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-label/);
! $t_prog_desc = $t_prog->look_down('_tag' => 'div', 'class' => qr/program-desc/);
!
!
! # If programme is outside of requested timeframe then drop it
! my $p_start = $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime');
! $p_start = epoch_isotime($p_start);
! next if $p_start < $grab_start || $p_start >= $grab_end;
!
!
! $p_dtstart = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtstart/)->attr('datetime') );
! $p_dtend = xmltv_isotime( $t_prog_label->look_down('_tag' => 'time', 'class' => qr/dtend/)->attr('datetime') );
!
! # it seems the stop times aren't to be trusted
! # - on Sub tv they seem to be the actual stop times of the programme (i.e. start + duration), i.e. excludes trailers/ads before next prog (so causing gaps)
! # - on Areena and Ava they're often 1 or 2 minutes *after* the start of the next prog (so causing overlaps).
! undef $p_dtend;
!
! $t_prog_title = $t_prog_label->look_down('_tag' => 'div', 'class' => qr/program-title/);
! $p_title = $t_prog_title->look_down('_tag' => 'a', 'class' => qr/link-grey/)->as_text();
!
! # the <div class="program-title"> may contain <span class="movie">Film</span> if prog is a film
! if ( $t_prog_film = $t_prog_title->look_down('_tag' => 'span', 'class' => qr/movie/) ) {
! $p_category = $t_prog_film->as_text();
! }
!
! $p_desc = $t_prog_desc->look_down('_tag' => 'p')->as_text(); # desc is in first <p>
! $t_prog_link = $t_prog_desc->look_down('_tag' => 'a', 'class' => qr/double-angle/, sub { return $_[0]->as_text() =~ /Programinformation/ });
! $p_url = $t_prog_link->attr('href') if $t_prog_link;
!
!
! # occasionally it seems there's a 'null' programme (usually with overlapping time)
! # I think we should drop these
! next if $p_title eq '';
!
!
! # Create the data structure for the program.
! my $program =
! {
! 'channel' => $c_channel_id,
! 'title' => [[ encode('iso-8859-1', $p_title), $LANGUAGE_CODE ]],
! 'start' => $p_dtstart
! };
! $program->{'stop'} = $p_dtend if (defined $p_dtend && $p_dtend ne '');
! $program->{'desc'} = [[ encode('iso-8859-1', tidy( $p_desc) ), $LANGUAGE_CODE ]] if (defined $p_desc && $p_desc ne '');
! $program->{'sub-title'} = [[ encode('iso-8859-1', $p_subtitle), $LANGUAGE_CODE ]] if (defined $p_subtitle && $p_subtitle ne '');
! $program->{'category'} = [[ encode('iso-8859-1', $p_category) ]] if (defined $p_category && $p_category ne '');
! $program->{'url'} = [ $p_url ] if (defined $p_url && $p_url ne '');
!
!
! # store the programme avoiding duplicates
! # also check for deuplicate start times and set clumpidx
! if ( defined $programmes->{$c_channel_id}->{$p_dtstart} ) {
! # duplicate prog or contemporary?
! my $dup = 0;
! foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
! $dup = 1 if ( $p->{'title'}[0][0] eq $program->{'title'}[0][0] ); # duplicate
! }
! next if $dup; # ignore duplicates
! if (!$dup) {
! # contemporary programme so set clumpidx
! my $numclumps = scalar @{ $programmes->{$c_channel_id}->{$p_dtstart} } + 1;
! # set (or adjust) clumpidx of existing programmes
! my $i = 0;
! foreach my $p ( @{ $programmes->{$c_channel_id}->{$p_dtstart} } ) {
! $p->{'clumpidx'} = "$i/$numclumps";
! $i++;
! }
! # set clumpidx for new programme
! $program->{'clumpidx'} = "$i/$numclumps";
! }
! }
!
! # store the programme
! push @{ $programmes->{$c_channel_id}->{$p_dtstart} }, $program;
! }
! }
! }
! }
!
!
! # All data has been gathered. We can now write the
! # program element to the output.
! #
! foreach ( keys %{$programmes} ) {
! my $ch_progs = $programmes->{$_};
! foreach ( sort keys %{$ch_progs} ) {
! my $dt_progs = $ch_progs->{$_};
! foreach (@{ $dt_progs }) {
! $writer->write_programme($_);
! }
}
}
}
***************
*** 615,619 ****
# Remove bad chars from an element
sub tidy( $ ) {
! return $_[0] if !defined $_[0];
$_[0] =~ s/(\s)\xA0/$1/og; # replace 'space- ' with 'space'
$_[0] =~ s/\xA0/ /og; # replace any remaining with space
--- 614,618 ----
# Remove bad chars from an element
sub tidy( $ ) {
! return $_[0] if !defined $_[0];
$_[0] =~ s/(\s)\xA0/$1/og; # replace 'space- ' with 'space'
$_[0] =~ s/\xA0/ /og; # replace any remaining with space
***************
*** 679,684 ****
if ($end_time lt $start_time)
{
! # This program is spanning across a date boundary. We need to increase the
! # date part so that the end time gets produced correctly.
# Clone to ensure that this adjustment is only for this timestamp!
$date=$date->clone->add(days => 1);
--- 678,683 ----
if ($end_time lt $start_time)
{
! # This program is spanning across a date boundary. We need to increase the
! # date part so that the end time gets produced correctly.
# Clone to ensure that this adjustment is only for this timestamp!
$date=$date->clone->add(days => 1);
***************
*** 694,703 ****
{
my $time = shift;
!
! # let's not overthink this... just use a regexp!
!
! $time =~ s/[:-]//g;
! $time =~ /^(\d{8})T(\d{6}).*(\+\d{4})$/;
! return $1.$2.' '.$3;
}
--- 693,702 ----
{
my $time = shift;
!
! # let's not overthink this... just use a regexp!
!
! $time =~ s/[:-]//g;
! $time =~ /^(\d{8})T(\d{6}).*(\+\d{4})$/;
! return $1.$2.' '.$3;
}
***************
*** 706,720 ****
{
my $time = shift;
! my $withoffset = shift; # set to 1 to generate local epoch (else will get utc)
! my ($y, $m, $d, $h, $i, $s, $ms, $t, $th, $tm) = $time =~ /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d{3})([\+-])(\d{2}):(\d{2})$/;
! $y -= 1900;
! $m -= 1;
! use POSIX;
! my $epoch = mktime($s, $i, $h, $d, $m, $y);
! if (defined $withoffset && $withoffset) {
! my $offset = ($th * 3600) + ($tm * 60);
! $epoch += $offset if $t eq '+';
! $epoch -= $offset if $t eq '-';
! }
! return $epoch;
}
--- 705,719 ----
{
my $time = shift;
! my $withoffset = shift; # set to 1 to generate local epoch (else will get utc)
! my ($y, $m, $d, $h, $i, $s, $ms, $t, $th, $tm) = $time =~ /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})\.(\d{3})([\+-])(\d{2}):(\d{2})$/;
! $y -= 1900;
! $m -= 1;
! use POSIX;
! my $epoch = mktime($s, $i, $h, $d, $m, $y);
! if (defined $withoffset && $withoffset) {
! my $offset = ($th * 3600) + ($tm * 60);
! $epoch += $offset if $t eq '+';
! $epoch -= $offset if $t eq '-';
! }
! return $epoch;
}
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot