xmltv/grab/il tv_grab_il, 1.27, 1.28 test.conf, 1.3, 1.4
Nick Morrott <[email protected]> Sun, 28 Jun 2015 08:40:33 +0000
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/grab/il
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv24198/grab/il
Modified Files:
tv_grab_il test.conf
Log Message:
Unbreak tv_grab_il after site changes. Available channels have changed, please re-configure
Index: test.conf
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/il/test.conf,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** test.conf 22 Sep 2009 00:49:27 -0000 1.3
--- test.conf 28 Jun 2015 08:40:31 -0000 1.4
***************
*** 1,217 ****
! channel=126.tv.walla.co.il
! channel=128.tv.walla.co.il
! channel=129.tv.walla.co.il
! channel=13.tv.walla.co.il
! channel=130.tv.walla.co.il
! channel=134.tv.walla.co.il
! channel=135.tv.walla.co.il
! channel=136.tv.walla.co.il
! channel!137.tv.walla.co.il
! channel!138.tv.walla.co.il
! channel!139.tv.walla.co.il
! channel!141.tv.walla.co.il
! channel!142.tv.walla.co.il
! channel!143.tv.walla.co.il
! channel!144.tv.walla.co.il
! channel!145.tv.walla.co.il
! channel!146.tv.walla.co.il
! channel!147.tv.walla.co.il
! channel!148.tv.walla.co.il
! channel!149.tv.walla.co.il
! channel!15.tv.walla.co.il
! channel!150.tv.walla.co.il
! channel!151.tv.walla.co.il
! channel!152.tv.walla.co.il
! channel!153.tv.walla.co.il
! channel!154.tv.walla.co.il
! channel!155.tv.walla.co.il
! channel!156.tv.walla.co.il
! channel!157.tv.walla.co.il
! channel!158.tv.walla.co.il
! channel!159.tv.walla.co.il
! channel!160.tv.walla.co.il
! channel!161.tv.walla.co.il
! channel!162.tv.walla.co.il
! channel!163.tv.walla.co.il
! channel!164.tv.walla.co.il
! channel!169.tv.walla.co.il
! channel!170.tv.walla.co.il
! channel!171.tv.walla.co.il
! channel!172.tv.walla.co.il
! channel!174.tv.walla.co.il
! channel!175.tv.walla.co.il
! channel!18.tv.walla.co.il
! channel!180.tv.walla.co.il
! channel!181.tv.walla.co.il
! channel!182.tv.walla.co.il
! channel!184.tv.walla.co.il
! channel!185.tv.walla.co.il
! channel!188.tv.walla.co.il
! channel!189.tv.walla.co.il
! channel!191.tv.walla.co.il
! channel!193.tv.walla.co.il
! channel!194.tv.walla.co.il
! channel!198.tv.walla.co.il
! channel!199.tv.walla.co.il
! channel!200.tv.walla.co.il
! channel!201.tv.walla.co.il
! channel!202.tv.walla.co.il
! channel!204.tv.walla.co.il
! channel!205.tv.walla.co.il
! channel!206.tv.walla.co.il
! channel!207.tv.walla.co.il
! channel!208.tv.walla.co.il
! channel!209.tv.walla.co.il
! channel!210.tv.walla.co.il
! channel!211.tv.walla.co.il
! channel!212.tv.walla.co.il
! channel!213.tv.walla.co.il
! channel!214.tv.walla.co.il
! channel!215.tv.walla.co.il
! channel!216.tv.walla.co.il
! channel!217.tv.walla.co.il
! channel!218.tv.walla.co.il
! channel!219.tv.walla.co.il
! channel!22.tv.walla.co.il
! channel!220.tv.walla.co.il
! channel!221.tv.walla.co.il
! channel!222.tv.walla.co.il
! channel!223.tv.walla.co.il
! channel!225.tv.walla.co.il
! channel!228.tv.walla.co.il
! channel!229.tv.walla.co.il
! channel!230.tv.walla.co.il
! channel!231.tv.walla.co.il
! channel!232.tv.walla.co.il
! channel!233.tv.walla.co.il
! channel!234.tv.walla.co.il
! channel!235.tv.walla.co.il
! channel!236.tv.walla.co.il
! channel!237.tv.walla.co.il
! channel!238.tv.walla.co.il
! channel!239.tv.walla.co.il
! channel!244.tv.walla.co.il
! channel!245.tv.walla.co.il
! channel!247.tv.walla.co.il
! channel!248.tv.walla.co.il
! channel!252.tv.walla.co.il
! channel!253.tv.walla.co.il
! channel!255.tv.walla.co.il
! channel!258.tv.walla.co.il
! channel!259.tv.walla.co.il
! channel!260.tv.walla.co.il
! channel!261.tv.walla.co.il
! channel!263.tv.walla.co.il
! channel!264.tv.walla.co.il
! channel!265.tv.walla.co.il
! channel!266.tv.walla.co.il
! channel!267.tv.walla.co.il
! channel!269.tv.walla.co.il
! channel!270.tv.walla.co.il
! channel!271.tv.walla.co.il
! channel!272.tv.walla.co.il
! channel!273.tv.walla.co.il
! channel!274.tv.walla.co.il
! channel!275.tv.walla.co.il
! channel!277.tv.walla.co.il
! channel!279.tv.walla.co.il
! channel!281.tv.walla.co.il
! channel!284.tv.walla.co.il
! channel!285.tv.walla.co.il
! channel!289.tv.walla.co.il
! channel!290.tv.walla.co.il
! channel!292.tv.walla.co.il
! channel!293.tv.walla.co.il
! channel!294.tv.walla.co.il
! channel!295.tv.walla.co.il
! channel!297.tv.walla.co.il
! channel!299.tv.walla.co.il
! channel!30.tv.walla.co.il
! channel!301.tv.walla.co.il
! channel!302.tv.walla.co.il
! channel!3045.tv.walla.co.il
! channel!3046.tv.walla.co.il
! channel!3047.tv.walla.co.il
! channel!3048.tv.walla.co.il
! channel!3049.tv.walla.co.il
! channel!305.tv.walla.co.il
! channel!3050.tv.walla.co.il
! channel!3051.tv.walla.co.il
! channel!3052.tv.walla.co.il
! channel!3053.tv.walla.co.il
! channel!3054.tv.walla.co.il
! channel!3055.tv.walla.co.il
! channel!3056.tv.walla.co.il
! channel!3061.tv.walla.co.il
! channel!3062.tv.walla.co.il
! channel!3063.tv.walla.co.il
! channel!3064.tv.walla.co.il
! channel!312.tv.walla.co.il
! channel!314.tv.walla.co.il
! channel!315.tv.walla.co.il
! channel!316.tv.walla.co.il
! channel!317.tv.walla.co.il
! channel!324.tv.walla.co.il
! channel!329.tv.walla.co.il
! channel!33.tv.walla.co.il
! channel!331.tv.walla.co.il
! channel!332.tv.walla.co.il
! channel!333.tv.walla.co.il
! channel!335.tv.walla.co.il
! channel!336.tv.walla.co.il
! channel!339.tv.walla.co.il
! channel!34.tv.walla.co.il
! channel!340.tv.walla.co.il
! channel!341.tv.walla.co.il
! channel!343.tv.walla.co.il
! channel!344.tv.walla.co.il
! channel!345.tv.walla.co.il
! channel!346.tv.walla.co.il
! channel!349.tv.walla.co.il
! channel!350.tv.walla.co.il
! channel!351.tv.walla.co.il
! channel!353.tv.walla.co.il
! channel!355.tv.walla.co.il
! channel!356.tv.walla.co.il
! channel!357.tv.walla.co.il
! channel!361.tv.walla.co.il
! channel!362.tv.walla.co.il
! channel!363.tv.walla.co.il
! channel!365.tv.walla.co.il
! channel!366.tv.walla.co.il
! channel!367.tv.walla.co.il
! channel!368.tv.walla.co.il
! channel!369.tv.walla.co.il
! channel!37.tv.walla.co.il
! channel!370.tv.walla.co.il
! channel!373.tv.walla.co.il
! channel!374.tv.walla.co.il
! channel!375.tv.walla.co.il
! channel!39.tv.walla.co.il
! channel!40.tv.walla.co.il
! channel!41.tv.walla.co.il
! channel!44.tv.walla.co.il
! channel!45.tv.walla.co.il
! channel!49.tv.walla.co.il
! channel!50.tv.walla.co.il
! channel!505.tv.walla.co.il
! channel!509.tv.walla.co.il
! channel!511.tv.walla.co.il
! channel!514.tv.walla.co.il
! channel!515.tv.walla.co.il
! channel!517.tv.walla.co.il
! channel!519.tv.walla.co.il
! channel!520.tv.walla.co.il
! channel!540.tv.walla.co.il
! channel!541.tv.walla.co.il
! channel!542.tv.walla.co.il
! channel!545.tv.walla.co.il
! channel!548.tv.walla.co.il
! channel!55.tv.walla.co.il
! channel!559.tv.walla.co.il
! channel!560.tv.walla.co.il
! channel!66.tv.walla.co.il
! channel!76.tv.walla.co.il
! channel!77.tv.walla.co.il
! channel!8.tv.walla.co.il
! channel!94.tv.walla.co.il
--- 1,42 ----
! channel=210.tv-guide.walla.co.il
! channel=193.tv-guide.walla.co.il
! channel=200.tv-guide.walla.co.il
! channel=211.tv-guide.walla.co.il
! channel=199.tv-guide.walla.co.il
! channel!301.tv-guide.walla.co.il
! channel!3547.tv-guide.walla.co.il
! channel!547.tv-guide.walla.co.il
! channel!238.tv-guide.walla.co.il
! channel!292.tv-guide.walla.co.il
! channel!293.tv-guide.walla.co.il
! channel!294.tv-guide.walla.co.il
! channel!297.tv-guide.walla.co.il
! channel!295.tv-guide.walla.co.il
! channel!208.tv-guide.walla.co.il
! channel!3588.tv-guide.walla.co.il
! channel!255.tv-guide.walla.co.il
! channel!349.tv-guide.walla.co.il
! channel!315.tv-guide.walla.co.il
! channel!316.tv-guide.walla.co.il
! channel!3595.tv-guide.walla.co.il
! channel!559.tv-guide.walla.co.il
! channel!223.tv-guide.walla.co.il
! channel!204.tv-guide.walla.co.il
! channel!314.tv-guide.walla.co.il
! channel!317.tv-guide.walla.co.il
! channel!353.tv-guide.walla.co.il
! channel!373.tv-guide.walla.co.il
! channel!225.tv-guide.walla.co.il
! channel!201.tv-guide.walla.co.il
! channel!289.tv-guide.walla.co.il
! channel!3599.tv-guide.walla.co.il
! channel!94.tv-guide.walla.co.il
! channel!274.tv-guide.walla.co.il
! channel!275.tv-guide.walla.co.il
! channel!290.tv-guide.walla.co.il
! channel!3046.tv-guide.walla.co.il
! channel!3045.tv-guide.walla.co.il
! channel!3594.tv-guide.walla.co.il
! channel!3555.tv-guide.walla.co.il
! channel!3551.tv-guide.walla.co.il
! channel!3641.tv-guide.walla.co.il
Index: tv_grab_il
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/il/tv_grab_il,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** tv_grab_il 24 May 2014 15:51:33 -0000 1.27
--- tv_grab_il 28 Jun 2015 08:40:31 -0000 1.28
***************
*** 1,7 ****
#!/usr/bin/perl -w
- eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
- if 0; # not running under some shell
-
=pod
--- 1,4 ----
***************
*** 37,41 ****
Output TV listings in XMLTV format for many channels available in Israel.
! The data comes from tv.walla.co.il.
First you must run B<tv_grab_il --configure> to choose which channels
--- 34,38 ----
Output TV listings in XMLTV format for many channels available in Israel.
! The data comes from tv-guide.walla.co.il.
First you must run B<tv_grab_il --configure> to choose which channels
***************
*** 81,88 ****
=head1 ERROR HANDLING
! If the grabber fails to download data for some channel on a specific day,
it will print an errormessage to STDERR and then continue with the other
! channels and days. The grabber will exit with a status code of 1 to indicate
! that the data is incomplete.
=head1 ENVIRONMENT VARIABLES
--- 78,85 ----
=head1 ERROR HANDLING
! If the grabber fails to download data for some channel on a specific day,
it will print an errormessage to STDERR and then continue with the other
! channels and days. The grabber will exit with a status code of 1 to indicate
! that the data is incomplete.
=head1 ENVIRONMENT VARIABLES
***************
*** 94,98 ****
=head1 SUPPORTED CHANNELS
! For information on supported channels, see http://tv.walla.co.il/
=head1 AUTHOR
--- 91,95 ----
=head1 SUPPORTED CHANNELS
! For information on supported channels, see http://tv-guide.walla.co.il/
=head1 AUTHOR
***************
*** 121,125 ****
##$XMLTV::Get_nice::Delay = 0;
##use HTTP::Cache::Transparent;
! ##HTTP::Cache::Transparent::init( {
## BasePath => '/root/.xmltv/cache',
## NoUpdate => 60*60, # cache time in seconds
--- 118,122 ----
##$XMLTV::Get_nice::Delay = 0;
##use HTTP::Cache::Transparent;
! ##HTTP::Cache::Transparent::init( {
## BasePath => '/root/.xmltv/cache',
## NoUpdate => 60*60, # cache time in seconds
***************
*** 129,143 ****
! my $channel_link_regexp = "\\?w=\\/[0-9]\\/\\/[0-9]*\\/\\/[A-Za-z]*\\/1";
! my $channel_link_id = "\\?w=\\/[0-9]\\/\\/([0-9]*)\\/\\/[A-Za-z]*\\/1";
# Use XMLTV::Options::ParseOptions to parse the options and take care of the basic capabilities that a tv_grabber should
! my ($opt, $conf) = ParseOptions({
grabber_name => "tv_grab_il",
capabilities => [qw/baseline manualconfig apiconfig/],
stage_sub => \&config_stage,
! listchannels_sub => \&fetch_channels,
! version => '$Id$',
! description => "Israel (tv.walla.co.il)",
extra_options => [qw/slow/], # grab descriptions from sub-page
});
--- 126,140 ----
! my $channel_link_regexp = "\\?w=\\/\\/\\/[0-9]*\\/\\/[A-Za-z]*\\/1";
! my $channel_link_id = "\\?w=\\/\\/\\/([0-9]*)\\/\\/[A-Za-z]*\\/1";
# Use XMLTV::Options::ParseOptions to parse the options and take care of the basic capabilities that a tv_grabber should
! my ($opt, $conf) = ParseOptions({
grabber_name => "tv_grab_il",
+ version => '$Id$',
capabilities => [qw/baseline manualconfig apiconfig/],
stage_sub => \&config_stage,
! listchannels_sub => \&write_channels,
! description => "Israel (tv-guide.walla.co.il)",
extra_options => [qw/slow/], # grab descriptions from sub-page
});
***************
*** 158,162 ****
my ($opt, $conf) = @_;
- my $result;
my $channels = {};
--- 155,158 ----
***************
*** 166,171 ****
}) unless ($opt->{quiet} || $opt->{debug});
! # Get the page containing the list of channels
! my $tree = XMLTV::Get_nice::get_nice_tree('http://tv.walla.co.il/?w=/4');
my @channels = $tree->look_down("_tag", "a",
"href", qr/$channel_link_regexp/,
--- 162,167 ----
}) unless ($opt->{quiet} || $opt->{debug});
! # Get the page containing the list of channels
! my $tree = XMLTV::Get_nice::get_nice_tree('http://tv-guide.walla.co.il', undef, 'windows-1255');
my @channels = $tree->look_down("_tag", "a",
"href", qr/$channel_link_regexp/,
***************
*** 181,188 ****
# Browse through the downloaded list of channels and map them to a hash XMLTV::Writer would understand
- # Kludge the hash keys so Writer outputs them in the same order as on the source website :-)
- my $i=0;
foreach my $channel (@channels) {
- $i++; my $i2 = substr("0000$i", -5);
if ($channel->as_text()) {
my ($id) = $channel->attr('href') =~ /$channel_link_id/;
--- 177,181 ----
***************
*** 195,204 ****
$icon = $icon->attr('src') if $icon;
! $channels->{"$i2.$id.tv.walla.co.il"} = {
! id => "$id.tv.walla.co.il",
! 'display-name' => [[ encode( 'utf-8', decode( 'windows-1255', $channel->as_text())), 'he' ]],
url => [ $channel->attr('href') ]
};
! $channels->{"$i2.$id.tv.walla.co.il"}->{icon} = [ {src => ($icon || '')} ] if ($icon)
}
--- 188,197 ----
$icon = $icon->attr('src') if $icon;
! $channels->{"$id.tv-guide.walla.co.il"} = {
! id => "$id.tv-guide.walla.co.il",
! 'display-name' => [[ encode( 'utf-8', $channel->as_text()) ]],
url => [ $channel->attr('href') ]
};
! $channels->{"$id.tv-guide.walla.co.il"}->{icon} = [ {src => ($icon || '')} ] if ($icon)
}
***************
*** 215,219 ****
--- 208,221 ----
}) unless ($opt->{quiet} || $opt->{debug});
+ $bar->update() && $bar->finish() if defined $bar;
+
+ return $channels;
+ }
+
+ sub write_channels {
+ my $channels = fetch_channels($opt, $conf);
+
# Let XMLTV::Writer format the results as a valid xmltv file
+ my $result;
my $writer = new XMLTV::Writer(OUTPUT => \$result, encoding => 'utf-8');
$writer->start({'generator-info-name' => 'tv_grab_il'});
***************
*** 221,235 ****
$writer->end();
- $bar->update() && $bar->finish() if defined $bar;
-
return $result;
}
# Fetch the channels again to see what's available
! my $data = XMLTV::parse(&fetch_channels($opt, $conf));
! my ($encoding, $credits, $channels, $programmes) = @{$data};
! # Create a new hash for the channels so that channels without programmes won't appear in the final XML
my $w_channels = {};
# Progress Bar :)
--- 223,240 ----
$writer->end();
return $result;
}
# Fetch the channels again to see what's available
! my $channels = fetch_channels($opt, $conf);
! # Configure initial elements for XMLTV::Writer
! #
! # Create a new hash for the channels so that channels without programmes
! # won't appear in the final XML
! my $encoding = 'UTF-8';
! my $credits = {'generator-info-name' => 'tv_grab_il'};
my $w_channels = {};
+ my $programmes = [];
# Progress Bar :)
***************
*** 245,255 ****
if ($channels->{$channel_id}) {
my ($walla_id) = ($channel_id =~ /^([0-9]*)\..*$/);
!
# Now grab listings for each channel on each day, according to the options in $opt
for (my $i=$opt->{offset}; $i < ($opt->{offset} + $opt->{days}); $i++) {
my $theday = DateTime->today()->add (days => $i)->set_time_zone('Asia/Jerusalem');
! my $url = "http://tv.walla.co.il/?w=/4//$walla_id//" . $theday->day_name() . "/1";
! my $tree = XMLTV::Get_nice::get_nice_tree($url);
if ($tree) {
--- 250,260 ----
if ($channels->{$channel_id}) {
my ($walla_id) = ($channel_id =~ /^([0-9]*)\..*$/);
!
# Now grab listings for each channel on each day, according to the options in $opt
for (my $i=$opt->{offset}; $i < ($opt->{offset} + $opt->{days}); $i++) {
my $theday = DateTime->today()->add (days => $i)->set_time_zone('Asia/Jerusalem');
! my $url = "http://tv-guide.walla.co.il/?w=/4//$walla_id//" . $theday->day_name() . "/1";
! my $tree = XMLTV::Get_nice::get_nice_tree($url, undef, 'windows-1255');
if ($tree) {
***************
*** 259,263 ****
my $title = $show->look_down('_tag', 'a', 'class', 'w3b');
my $show_hour_element = $show->look_down('_tag', 'span', 'class', 'w3b txt-w');
!
my @show_hour = split(/:/, $show_hour_element->as_text());
my $show_time = $theday->clone();
--- 264,268 ----
my $title = $show->look_down('_tag', 'a', 'class', 'w3b');
my $show_hour_element = $show->look_down('_tag', 'span', 'class', 'w3b txt-w');
!
my @show_hour = split(/:/, $show_hour_element->as_text());
my $show_time = $theday->clone();
***************
*** 267,277 ****
$show_time->set(hour => $show_hour[0], minute => $show_hour[1], second => 0);
!
# If user wants descriptions then get them from the sub-page
# (can also get stop time while we're there!)
my ($start, $stop, $desc) = ('', '', ''); my $stop_time;
if ( defined $opt->{slow} && $opt->{slow} ) {
!
! if ( my $detailtree = XMLTV::Get_nice::get_nice_tree('http://tv.walla.co.il/'.$title->attr('href')) ) {
if ( my $table = $detailtree->look_down('_tag', 'table', 'class', 'wp-0-b') ) { # get 1st "wp-0-b" table
if ( my $td = $table->look_down('_tag', 'td', 'class', 'w2b', sub{ !$_[0]->look_down('_tag', 'h2') } ) ) {
--- 272,282 ----
$show_time->set(hour => $show_hour[0], minute => $show_hour[1], second => 0);
!
# If user wants descriptions then get them from the sub-page
# (can also get stop time while we're there!)
my ($start, $stop, $desc) = ('', '', ''); my $stop_time;
if ( defined $opt->{slow} && $opt->{slow} ) {
!
! if ( my $detailtree = XMLTV::Get_nice::get_nice_tree('http://tv-guide.walla.co.il/'.$title->attr('href'), undef, 'windows-1255') ) {
if ( my $table = $detailtree->look_down('_tag', 'table', 'class', 'wp-0-b') ) { # get 1st "wp-0-b" table
if ( my $td = $table->look_down('_tag', 'td', 'class', 'w2b', sub{ !$_[0]->look_down('_tag', 'h2') } ) ) {
***************
*** 289,304 ****
$stop_time->set(hour => $stop_hour[0], minute => $stop_hour[1], second => 0);
}
!
}
!
my $prog= {
start => $show_time->strftime("%Y%m%d%H%M%S %z"),
! title => [[ encode( 'utf-8', decode( 'windows-1255', $title->as_text())), 'he']],
channel => $channel_id
};
$prog->{'stop'} = $stop_time->strftime("%Y%m%d%H%M%S %z") if defined $stop_time;
! $prog->{'desc'} = [[ encode( 'utf-8', decode( 'windows-1255', $desc)), 'he']] if $desc ne '';
push @{$programmes}, $prog;
!
}
--- 294,309 ----
$stop_time->set(hour => $stop_hour[0], minute => $stop_hour[1], second => 0);
}
!
}
!
my $prog= {
start => $show_time->strftime("%Y%m%d%H%M%S %z"),
! title => [[ encode( 'utf-8', $title->as_text()) ]],
channel => $channel_id
};
$prog->{'stop'} = $stop_time->strftime("%Y%m%d%H%M%S %z") if defined $stop_time;
! $prog->{'desc'} = [[ encode( 'utf-8', $desc) ]] if $desc ne '';
push @{$programmes}, $prog;
!
}
***************
*** 326,329 ****
--- 331,337 ----
}
+ my $data = [];
+ $data->[0] = $encoding;
+ $data->[1] = $credits;
$data->[2] = $w_channels;
$data->[3] = $programmes;
------------------------------------------------------------------------------
Monitor 25 network devices or servers for free with OpManager!
OpManager is web-based network management software that monitors
network devices and physical & virtual servers, alerts via email & sms
for fault. Monitor 25 devices for free with no restriction. Download now
http://ad.doubleclick.net/ddm/clk/292181274;119417398;o