xmltv/grab/pt test.conf, 1.5, 1.6 tv_grab_pt, 1.49, 1.50
Geoff <[email protected]>
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/grab/pt
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv30213
Modified Files:
test.conf tv_grab_pt
Log Message:
(major) unbreak after source site changes
Index: tv_grab_pt
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/pt/tv_grab_pt,v
retrieving revision 1.49
retrieving revision 1.50
diff -C2 -d -r1.49 -r1.50
*** tv_grab_pt 11 Feb 2012 18:43:28 -0000 1.49
--- tv_grab_pt 7 May 2014 16:24:33 -0000 1.50
***************
*** 13,17 ****
tv_grab_pt [--config-file FILE] --configure [--gui OPTION]
! tv_grab_pt [--config-file FILE] [--output FILE] [--quiet] [--icons]
tv_grab_pt --list-channels
--- 13,18 ----
tv_grab_pt [--config-file FILE] --configure [--gui OPTION]
! tv_grab_pt [--config-file FILE] [--output FILE] [--days N]
! [--offset N] [--fast] [--quiet] [--icons]
tv_grab_pt --list-channels
***************
*** 38,41 ****
--- 39,50 ----
B<--configure> and read when grabbing.
+ B<--days N> Grab N days. The default is 7 days.
+
+ B<--offset N> Start N days in the future. The default is to start
+ from today.
+
+ B<--fast> Only fetch summary information for each programme. This is
+ only title, start/stop times, category, episode number.
+
B<--output FILE> Write to FILE rather than standard output.
***************
*** 43,47 ****
error.
! B<--icons> Fetches channels icons/logos
B<--version> Show the version of the grabber.
--- 52,56 ----
error.
! B<--icons> Fetches channels icons/logos [deprecated - this is now the default]
B<--version> Show the version of the grabber.
***************
*** 72,83 ****
use XMLTV::Description 'Portugal';
use Getopt::Long;
! use Date::Manip;
! use Data::Dumper;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
use IO::File;
use File::Path;
use File::Basename;
! use LWP::UserAgent;
use XMLTV;
--- 81,95 ----
use XMLTV::Description 'Portugal';
use Getopt::Long;
! #use Date::Manip;
! use DateTime;
! #use Data::Dumper;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
+ use HTTP::Cache::Transparent;
+ use Encode;
use IO::File;
use File::Path;
use File::Basename;
! #use LWP::UserAgent;
use XMLTV;
***************
*** 99,106 ****
my $DOMAIN = 'zon.pt';
# Attributes of the root element in output.
! my $HEAD = { 'source-info-url' => "http://$DOMAIN",
! 'source-data-url' => "http://www.$DOMAIN/Televisao/Programacao.aspx",
'generator-info-name' => 'XMLTV',
'generator-info-url' => 'http://xmltv.org/',
--- 111,119 ----
my $DOMAIN = 'zon.pt';
+ my $SOURCE_URL = "http://www.$DOMAIN";
# Attributes of the root element in output.
! my $HEAD = { 'source-info-url' => "http://$DOMAIN/",
! 'source-data-url' => "http://www.$DOMAIN/tv/guiaTV/",
'generator-info-name' => 'XMLTV',
'generator-info-url' => 'http://xmltv.org/',
***************
*** 120,125 ****
our ($opt_help, $opt_output,
$opt_configure, $opt_config_file, $opt_gui, $opt_quiet,
! $opt_list_channels,$opt_offset,$opt_days,$opt_fullinfo,$opt_icons);
$opt_quiet = 0; # default
GetOptions('help' => \$opt_help,
'configure' => \$opt_configure,
--- 133,143 ----
our ($opt_help, $opt_output,
$opt_configure, $opt_config_file, $opt_gui, $opt_quiet,
! $opt_list_channels, $opt_offset, $opt_days, $opt_fast, $opt_icons,
! $opt_debug);
$opt_quiet = 0; # default
+ $opt_days = 7; # default
+ $opt_offset = 0; # default
+ $opt_fast = 0; # default
+ $opt_debug = 0;
GetOptions('help' => \$opt_help,
'configure' => \$opt_configure,
***************
*** 131,147 ****
'offset=i' => \$opt_offset,
'days=i' => \$opt_days,
! 'fullinfo' => \$opt_fullinfo,
! 'icons' => \$opt_icons,
)
or usage(0);
usage(1) if $opt_help;
XMLTV::Ask::init($opt_gui);
! # --offset and --days are ignored (we return more data than was
! # requested) but at least check the user didn't ask for something
! # impossible.
! #
! $opt_days |= 0;
! if ($opt_days > 0) { $opt_days-- }
our $first_day = ($opt_offset || 0);
our $last_day = $first_day + $opt_days;
--- 149,170 ----
'offset=i' => \$opt_offset,
'days=i' => \$opt_days,
! 'fast' => \$opt_fast,
! 'icons' => \$opt_icons, # Fetches channels icons/logos [deprecated - this is now the default]
! 'debug' => \$opt_debug, # undocumented
)
or usage(0);
usage(1) if $opt_help;
+
+ # Initialise the web page cache
+ HTTP::Cache::Transparent::init( {
+ BasePath => get_default_cachedir(),
+ NoUpdate => 4*3600, # cache time in seconds
+ MaxAge => 24, # flush time in hours
+ Verbose => $opt_debug,
+ } );
+
XMLTV::Ask::init($opt_gui);
!
our $first_day = ($opt_offset || 0);
our $last_day = $first_day + $opt_days;
***************
*** 188,192 ****
# Ask about each channel.
my @chs = sort keys %channels;
! my @names = map { $channels{$_} } @chs;
my @qs = map { "add channel $_?" } @names;
my @want = ask_many_boolean(1, @qs);
--- 211,215 ----
# Ask about each channel.
my @chs = sort keys %channels;
! my @names = map { $channels{$_}->{'channel-name'} } @chs;
my @qs = map { "add channel $_?" } @names;
my @want = ask_many_boolean(1, @qs);
***************
*** 291,301 ****
scalar @channels)
if not $opt_quiet;
foreach my $ch_did (@channels) {
die if not defined $ch_did;
!
! my $ch_name=$channels{$ch_did};
! my $channel = { id => "$ch_did.$DOMAIN",
! 'display-name' => [ [ $ch_name ] ],
};
$channel->{'icon'} = [{'src' => $icons{$ch_did}}] if(defined($icons{$ch_did}));
--- 314,326 ----
scalar @channels)
if not $opt_quiet;
+
+ # write the <channels> elements
foreach my $ch_did (@channels) {
die if not defined $ch_did;
! my $ch_name=$channels{$ch_did}->{'channel-name'};
! my $channel = { 'id' => $channels{$ch_did}->{'id'},
! 'display-name' => $channels{$ch_did}->{'display-name'},
! 'icon' => $channels{$ch_did}->{'icon'},
};
$channel->{'icon'} = [{'src' => $icons{$ch_did}}] if(defined($icons{$ch_did}));
***************
*** 307,310 ****
--- 332,342 ----
my $some=0;
our $today_offset;
+
+ # time limits for grab
+ my $today_date = DateTime->today(time_zone => 'Europe/Lisbon');
+ my $grab_start = $today_date->epoch() + ($opt_offset * 86400);
+ my $grab_stop = $grab_start + ($opt_days * 86400);
+ print STDERR "\n start/end grab: $grab_start $grab_stop \n" if $opt_debug;
+
foreach my $ch_did (@channels) {
#skip legacy channels...
***************
*** 346,355 ****
*toUTF8 = sub { return $uref->toUTF8(@_); }
} else {
! *toUTF8 = sub { my $str = $_[1]; Encode::from_to($str, $_[0], "utf-8"); return $str; }
}
}
# Clean up bad characters in HTML.
! sub tidy( $ ) {
for (my $s = shift) {
# Character 150 seems to be used for 'versus' in sporting
--- 378,388 ----
*toUTF8 = sub { return $uref->toUTF8(@_); }
} else {
! # *toUTF8 = sub { my $str = $_[1]; Encode::from_to($str, $_[0], "utf-8"); return $str; }
! *toUTF8 = sub { return Encode::encode("utf-8", $_[0]); }
}
}
# Clean up bad characters in HTML.
! sub _tidy( $ ) {
for (my $s = shift) {
# Character 150 seems to be used for 'versus' in sporting
***************
*** 361,364 ****
--- 394,413 ----
}
+ # Remove bad chars from an element
+ sub tidy( $ ) {
+ return $_[0] if !defined $_[0];
+ $_[0] =~ s/(\s)\xA0/$1/og; # replace 'space- ' with 'space'
+ $_[0] =~ s/\xA0/ /og; # replace any remaining with space
+ $_[0] =~ s/\xAD//og; # delete soft hyphens
+ return $_[0];
+ }
+
+ # Remove leading & trailing spaces
+ sub trim( $ ) {
+ # Remove leading & trailing spaces
+ $_[0] =~ s/^\s+|\s+$//g;
+ return $_[0];
+ }
+
sub today_offset {
return $today_offset if defined($today_offset);
***************
*** 378,711 ****
t "Getting channel $ch_xmltv_id\n";
- #my $url = $HEAD->{"source-data-url"};
- my $url = $HEAD->{"source-info-url"};
- $url .= "/Televisao/ListaProgramas.aspx?";
- t $url;
- use LWP::UserAgent;
-
- my $ua = LWP::UserAgent->new;
- $ua->timeout(30);
-
$ch_xmltv_id =~ /(.+?)\.zon\.pt/;
! my @all_programs = ();
! for ( my $k = $first_day ; $k <= $last_day ; $k++ ) {
! my @program_data = ();
! my $ch_url_id = $ch_xmltv_id;
- $ch_url_id =~ s/\+/%2B/g;
- $ch_url_id =~ s/\s/%20/g;
! my $ch_url =
! $url
! . "cat=&channelSigla="
! . encode_entities($ch_url_id) . "&dia="
! . ( today_offset() + $k )
! . "&package=9";
! my $r = $ua->get($ch_url);
! if ( !$r->is_success ) {
! die "Could not fetch $url ($ch_xmltv_id): " . $r->status_line;
! }
! my $data = tidy( $r->content );
! if ( not defined $data ) {
! die "could not fetch $url, aborting\n";
}
! local $SIG{__WARN__} = sub {
! warn "$url: $_[0]";
! };
!
! # parse the page to a document object
! my $tree = HTML::TreeBuilder->new();
! $tree->parse($data) or die "cannot parse content of $url\n";
! $tree->eof;
!
! my %month_conv = (
! 'Janeiro' => 'Jan',
! 'Fevereiro' => 'Feb',
! 'Março' => 'Mar',
! 'Abril' => 'Apr',
! 'Maio' => 'May',
! 'Junho' => 'Jun',
! 'Julho' => 'Jul',
! 'Agosto' => 'Aug',
! 'Setembro' => 'Sep',
! 'Outubro' => 'Oct',
! 'Novembro' => 'Nov',
! 'Dezembro' => 'Dec',
! );
!
! #lets get the day
! my $span = $tree->find_by_tag_name( '_tag' => 'strong' );
! next unless $span; # ??
!
! my $date = trim( $span->as_trimmed_text() );
!
! print STDERR "$date\n" unless $opt_quiet;
!
! $date =~ /(\d+) (.+)$/;
! my ( $day, $month ) = ( $1, $2 );
! $month = $month_conv{$month};
! #lets calc the year
! my @local_time = localtime( time() );
! my $current_year = $local_time[5] + 1900;
! my $current_month = $local_time[4] + 1;
! my $year = $current_year;
! if ( $month eq 'Jan' && $current_month != 1 ) {
! $year++;
}
! my $parsed_date =
! ParseDate( sprintf( "%02d %3s %04d", $day, $month, $year ) );
! die "Parse error, could not parse date" unless $parsed_date;
! my $f_date = $parsed_date;
! $f_date =~ s/^(\d{4})(\d{2})(\d{2}).*/$1$2$3/;
!
! my $epgdiv = $tree->find_by_attribute( 'id', 'epgFrame' );
! my $table = $epgdiv->find_by_tag_name( '_tag', 'table' );
!
! # Actually time and title are required, but we don't check that.
! if ($table) {
! my @trs = $table->find_by_tag_name( "_tag" => "tr" );
!
! # Store these for later, if we want full descs
! my @inputs = $tree->find_by_tag_name( '_tag', 'input' );
! my %descform;
! foreach my $input (@inputs) {
! next unless defined $input->attr('name');
! next if $input->attr('name') eq 'programId';
! $descform{ $input->attr('name') } = $input->attr('value');
! }
!
! foreach my $tr (@trs) {
! my @tds = $tr->find_by_tag_name( "_tag" => "td" );
! last unless ( $tds[0] && $tds[1] );
!
! my $time = trim( $tds[0]->as_trimmed_text() );
! my $title =
! trim( $tds[1]->find_by_tag_name( "_tag" => "a" )
! ->as_trimmed_text() );
! my $sub_t = "";
! my $episode = "";
! my $category = "";
! my $desc = "";
!
! #Try to find if they included the subtitle in the title
! if ( $title =~ /^(.*?)\:\#\#(.*?)$/ ) {
! $title = $1;
! $sub_t = $2;
! $sub_t =~ s/^\s*?(.*?)/$1/;
! $title =~ s/(.*?)\s*?$/$1/;
! }
! elsif ( $title =~ /^(.*?)\#\#(.*?)$/ ) {
! $title = $1;
! $sub_t = $2;
! $sub_t =~ s/^\s*?(.*?)/$1/;
! $title =~ s/(.*?)\s*?$/$1/;
! }
!
! #lets get the description
! my $link =
! $tds[1]->find_by_tag_name( "_tag" => "a" )->attr('href');
! my ($program_id) = ( $link =~ /\"(\d+)\"/ );
!
! my %credits;
! my $year;
- if ( $program_id && $opt_fullinfo ) {
! my ( $base_url, undef ) = split( /\?/, $ch_url );
- $descform{programId} = $program_id;
- my $r_desc = $ua->post( $base_url, \%descform );
! my $descr = $r_desc->content();
! if ( $descr =~
! /<\/h4>.*<strong>De:<\/strong>(.*?)<\/p>\s*\n\s*<p><strong>Com:<\/strong>(.*?)<\/p>\s*\n\s*<p>(.*?)<\/p>/ms
! )
! {
! my $director = $1;
! my $actors = $2;
! $desc = $3;
- $director =~ s/ //g;
- my @directors = split( /,/, $director );
- $actors =~ s/ //g;
- my @actors = split( /,/, $actors );
- $credits{actor} = \@actors;
- $credits{director} = \@directors;
! }
! elsif ( $descr =~ /<\/h4>\s*\n\s*<p>(.*?)<\/p>/ms ) {
! $desc = $1;
}
! if ( $desc =~ /\sREPLAY:\s.*tecla verde.*/ ) {
! $desc =~ s/\sREPLAY:\s.*//g;
}
- if ( $desc =~ /\s*Ano:\s*(\d{4})\.\s*$/ ) {
- $year = $1;
}
}
! #try to find a matching category trough css classes
! my %map = (
! epgFilm => 'Film',
! epgSer => 'Series',
! epgDoc => 'Documentary',
! epgDesp => 'Sports',
! epgNot => 'News',
! );
! my $cat_class = $tds[1]->attr('class');
! if ( $cat_class && exists( $map{$cat_class} ) ) {
! $category = $map{$cat_class};
! }
!
! $time =~ s/://g;
! $time = sprintf( "%04d", $time );
!
! for ($title) { s/^\s+$//; s/\s+$// }
! my %h = (
! time => $time,
! title => $title,
! date => $f_date,
! "episode-num" => $episode,
! subtitle => $sub_t,
! category => $category,
! desc => $desc
! );
!
! $h{credits} = \%credits if %credits;
! $h{year} = $year if $year;
! push @program_data, \%h;
}
}
! if ( not @program_data ) {
warn "$url ($ch_xmltv_id) : no programmes found\n";
! last;
! }
! @all_programs = ( @all_programs, @program_data );
! $tree->delete;
}
! return () unless scalar(@all_programs);
!
! my $first = $all_programs[0];
my @r;
! foreach my $p (@all_programs) {
! push @r, make_programme_hash( $ch_xmltv_id, $p, $first );
}
return @r;
}
! sub make_programme_hash {
! my ( $ch_xmltv_id, $cur, $first ) = @_;
- my $date = $cur->{date};
! my %prog;
! $prog{channel} = "$ch_xmltv_id.$DOMAIN";
! #$prog{channel} =~ s/\s/_/g;
- $prog{title} = [ [ $cur->{title}, $LANG ] ];
- $prog{"sub-title"} = [ [ $cur->{subtitle}, $LANG ] ] if $cur->{subtitle};
- $prog{category} = [ [ $cur->{category}, $LANG ] ] if $cur->{category};
- $prog{"episode-num"} = $cur->{"episode-num"} if $cur->{"episode-num"};
! if ( ( $cur->{time} < $first->{time} )
! && ( $cur->{date} == $first->{date} ) )
! {
! t "Jumping for next day of ("
! . $cur->{time} . ","
! . $first->{time}
! . ") $date...";
! $date = nextday($date);
! t "Got $date\n";
! }
! my $time = $date . $cur->{time} . "00";
! #print STDERR "Date built = $time\n";
! $prog{start} = utc_offset( $time, '+0000' );
! t "...got $prog{start}";
! unless ( $prog{start} ) {
! warn "bad time string: $cur->{time}";
! return undef;
}
- $prog{desc} = [ [ $cur->{desc}, $LANG ] ] if $cur->{desc};
! $prog{credits} = $cur->{credits} if $cur->{credits};
! $prog{date} = $cur->{year} if $cur->{year};
! return \%prog;
! }
! # as_trimmed_text() doesn't deal with ASCII 160, non-breaking space.
! sub trim( $ ) {
! for (my $tmp = shift) {
! #tr/\240/ /;
! s/^\s+//;
! s/\s+$//;
! return $_;
! }
}
# get channel listing
sub get_channels {
! my $bar = new XMLTV::ProgressBar( 'getting list of channels', 1 )
! if not $opt_quiet;
my %channels;
! my $url = $HEAD->{'source-data-url'};
t $url;
! my $tree = get_nice_tree $url;
! my @menus = $tree->find_by_tag_name( "_tag" => "select" );
!
! foreach my $elem (@menus) {
! my $cname = $elem->attr('name');
! next unless $cname eq 'ctl00$Middle$channels';
! my @ocanals = $elem->find_by_tag_name( "_tag" => "option" );
! my $chanpos = 0;
! foreach my $sigla (@ocanals) {
! $channelnumbers{ $sigla->attr('value') } = $chanpos;
! $chanpos++;
! }
! @ocanals = sort @ocanals;
! foreach my $opt (@ocanals) {
! my $channel_id = $opt->attr('value');
!
! next if $channel_id eq 'ALL';
! my $channel_name = trim( $opt->content->[0] );
! $channel_name = toUTF8( "iso-8859-1", $channel_name );
! $channels{$channel_id} = $channel_name;
! push @ch_all,
{
'display-name' => [ [ $channel_name, $LANG ] ],
! 'id' => "$channel_id"
};
} #foreach
! } #while
die "no channels could be found" if not keys %channels;
- #we need a token
- my $t = $tree->find_by_attribute( 'name', '__VIEWSTATE' );
- $token = $t->attr('value');
- die "Could not get the token\n" unless $token;
-
update $bar if not $opt_quiet;
$tree->delete;
--- 427,723 ----
t "Getting channel $ch_xmltv_id\n";
$ch_xmltv_id =~ /(.+?)\.zon\.pt/;
+
+ # This seems like a useful link but I can't see how to get the channelindex
+ # http://www.zon.pt/_layouts/EPGGetProgramsForChannels.aspx?cIndex=1&day=1&order=grelha&category=&numChannels=1
+
+ # http://www.zon.pt/tv/guiaTV/Pages/Guia-TV-programacao.aspx?channelSigla=5
+ my $url = $HEAD->{'source-data-url'} . 'Pages/Guia-TV-programacao.aspx?channelSigla='.$ch_xmltv_id;
+ print STDERR " URL= $url \n" if $opt_debug;
+ t $url;
! my $tree = get_nice_tree($url, '', 'UTF-8');
! my $programmes = {};
!
! if ( my $h = $tree->look_down('_tag' => 'div', 'id' => 'programs-container') ) {
! if ( my @h2 = $h->look_down('_tag' => 'div', 'class' => qr/programs-day-list/) ) {
! DAY:
! foreach my $h_day (@h2) { # schedule for a day
! my ($daynum) = $h_day->attr('id') =~ /day(\d*)/;
!
! my $dt = $today_date->set_day($daynum);
! #print STDERR "\n" . $dt->strftime("%Y%m%d%H%M%S %z") . "\n";
!
! next DAY if $dt->epoch() < $grab_start || $dt->epoch() >= $grab_stop;
! if ( my @h3 = $h_day->look_down('_tag' => 'li') ) { # progs for a day
! my $j = 0;
! PROG:
! foreach my $h_prog (@h3) { # each prog
! $j++;
! #<li style="height:119px;">
! # <span style="height:55px">
! # <a class="series" id="71841" href="#" title="Anatomia de Grey T.9 Ep.22">
! # <span class="program">Anatomia de Grey T.9 Ep.22</span><br>
! # <span class="duration">02:19 - 03:03</span>
! # </a>
! # </span>
! #</li>
! my ( $p_id, $p_category, $p_title, $p_desc, $p_timespan, $p_start, $p_stop, $p_start_epoch, $p_stop_epoch, $p_episode_num );
! if ( my $a = $h_prog->look_down('_tag' => 'a') ) {
! $p_category = $a->attr('class');
! $p_id = $a->attr('id');
! $p_title = $a->attr('title');
! if ( my $s = $a->look_down('_tag' => 'span', 'class' => 'program') ) {
! $p_desc = tidy $s->as_text();
}
! if ( my $s = $a->look_down('_tag' => 'span', 'class' => 'duration') ) {
! my ($start_h, $start_m, $stop_h, $stop_m) = $s->as_text() =~ /(\d\d):(\d\d)\s-\s(\d\d):(\d\d)/;
!
! my $start = $dt->clone(); $start->set( 'hour' => $start_h, 'minute' => $start_m );
! my $stop = $dt->clone(); $stop->set ( 'hour' => $stop_h, 'minute' => $stop_m );
! if ($stop_h < $start_h) { # stop hh < start hh = assume we've gone to the next (or previous) day
! $start->subtract( days => 1 ) if $j == 1; # first programme in day
! $stop->add( days => 1 ) if $j != 1; # not first programme in day
}
! $p_start = $start->strftime("%Y%m%d%H%M%S %z");
! $p_stop = $stop->strftime("%Y%m%d%H%M%S %z");
! $p_start_epoch = $start->epoch();
! $p_stop_epoch = $stop->epoch();
! $p_timespan = $s->as_text();
! }
! # strip the SnnEnn out of the title (e.g. "Anatomia de Grey T.9 Ep.24")
! my ($p_ser, $p_ep) = ('', ''); my $p_match;
! if ( ($p_match, $p_ser, $p_ep) = $p_title =~ /.*(T\.(\d*)\sEp\.(\d*))/ ) {
! $p_episode_num = --$p_ser . ' . ' . --$p_ep . ' . ';
! } elsif ( ($p_match, $p_ep) = $p_title =~ /.*(Ep\.(\d*))/ ) {
! $p_episode_num = ' . ' . --$p_ep . ' . ';
! }
! $p_title =~ s/$p_match// if $p_match;
! trim $p_title;
! }
! next PROG if $p_start_epoch < $grab_start || $p_start_epoch >= $grab_stop;
! my %prog;
! $prog{'channel'} = "$ch_xmltv_id.$DOMAIN";
! $prog{'id'} = $p_id;
! $prog{'title'} = $p_title;
! $prog{'category'} = $p_category;
! $prog{'start'} = $p_start;
! $prog{'stop'} = $p_stop;
! $prog{'episode-num'} = $p_episode_num;
! $prog{'timespan'} = $p_timespan; # not an xmltv item: used in process_details_page()
! # if user wants details then get them from the programme page
! if (!$opt_fast) {
! process_details_page ( $ch_xmltv_id, \%prog, $tree );
}
!
! # store the programme avoiding duplicates
! # also check for duplicate start times and set clumpidx
! {
! if ( defined $programmes->{ $ch_xmltv_id }->{ $p_start_epoch } ) {
! # duplicate prog or contemporary?
! my $dup = 0; my $_P;
! foreach $_P ( @{ $programmes->{ $ch_xmltv_id }->{ $p_start_epoch } } ) {
! $dup = 1 if ( $_P->{'title'} eq $prog{'title'} ); # duplicate
! }
! next PROG if $dup; # ignore duplicates (go to next programme)
! if (!$dup) {
! # contemporary programme so set clumpidx
! my $numclumps = scalar @{ $programmes->{ $ch_xmltv_id }->{ $p_start_epoch } } + 1;
! # set (or adjust) clumpidx of existing programmes
! my $i = 0;
! foreach $_P ( @{ $programmes->{ $ch_xmltv_id }->{ $p_start_epoch } } ) {
! $_P->{'clumpidx'} = "$i/$numclumps";
! $i++;
! }
! # set clumpidx for new programme
! $prog{'clumpidx'} = "$i/$numclumps";
}
}
}
! # store the programme
! push @{ $programmes->{ $ch_xmltv_id }->{ $p_start_epoch } }, \%prog;
! } # end each prog
! } # end progs for each day
! } # schedule for a day
}
}
!
! # did we get any programmes?
! if ( scalar $programmes == 0 ) {
warn "$url ($ch_xmltv_id) : no programmes found\n";
! return;
}
! # format the programmes ready for XMLTV::Writer
my @r;
! foreach ( keys %{$programmes} ) {
! my $_ch_progs = $programmes->{$_};
! foreach ( sort keys %{$_ch_progs} ) {
! my $_dt_progs = $_ch_progs->{$_};
! foreach (@{ $_dt_progs }) {
! push @r, make_programme_hash( $ch_xmltv_id, $_ );
! }
! }
}
return @r;
}
! sub process_details_page {
! my ($ch_xmltv_id, $prog, $s_tree) = @_;
! t "Getting prog details $$prog{'id'} \n";
! # I don't know how much of this url is actually required, but let's try to follow it as far as poss (for stealth reasons)
! #
! # http://www.zon.pt/_layouts/EPGGetProgramDetails.aspx?c=SIC&p=79302&h=0&sh=03:45%20&eh=%2006:00
! #
! # jQuery code:
! # channelAcronym1 = $("#channel-logo img").attr('alt');
! # programId1 = $(this).attr('id');
! #
! # var hasDifferentTime = "";
! # if (programId1 == "-2") {
! # hasDifferentTime = ($(this).find('.duration')).text();
! # }
! #
! # var substr = ($(this).find('.duration')).text().split('-');
! #
! # $.get('/_layouts/EPGGetProgramDetails.aspx?c=' + channelAcronym1 + '&p=' + programId1+ '&h=' + $("#ctl00_PlaceHolderMain_channelProgr_HoursToAddOrRemove").val() + "&sh=" + substr[0] + "&eh=" + substr[1], function (data) {...});
! #
! my $channelAcronym1;
! if ( my $h = $s_tree->look_down('_tag' => 'div', 'id' => 'channel-logo') ) {
! if ( my $h2 = $h->look_down('_tag' => 'img') ) {
! $channelAcronym1 = $h2->attr('alt');
! }
! }
! my $HoursToAddOrRemove = 0;
! if ( my $h = $s_tree->look_down('_tag' => 'input', 'id' => 'ctl00_PlaceHolderMain_channelProgr_HoursToAddOrRemove') ) {
! $HoursToAddOrRemove = $h->attr('value');
! }
! my $programId1 = $$prog{'id'};
! my ($timespan) = $$prog{'timespan'} =~ s/ /%20/g;
! my @substr = split(/-/, $$prog{'timespan'});
! my $url = $HEAD->{'source-info-url'} . '_layouts/EPGGetProgramDetails.aspx?c='.$channelAcronym1.'&p='.$programId1.'&h='.$HoursToAddOrRemove.'&sh='.$substr[0].'&eh='.$substr[1];
! print STDERR " URL= $url \n" if $opt_debug;
! t $url;
! # This is what the page returns. Looks like an old school delimited list
! # (c.f. OnCallGetProgramDetailsComplete JS code)
! #
! # Anatomia de Grey T.9 Ep.23_#|$_O drama médico mais famoso da televisão centra a sua história nas vidas profissionais e pessoais de um grupo de médicos cirurgiões e dos seus supervisores._#|$_277055_resized_352x198.jpg_#|$_02:47 _#|$_ 03:28_#|$_RTP 1_#|$_2014-05-07T02:47:00+01:00_#|$_2014-05-07T03:28:00+01:00_#|$_false
! #
! my $content = get_nice($url);
! $content = decode('UTF-8', $content);
!
! my @data = split(/_#\|\$_/, $content);
! # We could check the title matches what we already have but why not just trust the 'id' is correct ;-)
!
! $$prog{'desc'} = $data[1]; # store the description in our prog hash
! $$prog{'icon'} = "http://images.$DOMAIN/" . $data[2]; # [2] is a uri to the programme image
! return;
}
! # reformat the data to something acceptable to xmltv:::writer
! sub make_programme_hash {
! my ( $ch_xmltv_id, $cur ) = @_;
! my %prog;
+ $prog{channel} = $cur->{'channel'};
! #$prog{channel} =~ s/\s/_/g;
!
! $prog{'title'} = [ [ toUTF8( $cur->{'title'} ), $LANG ] ];
! $prog{'sub-title'} = [ [ toUTF8( $cur->{'subtitle'} ), $LANG ] ] if $cur->{'subtitle'};
! $prog{'category'} = [ [ toUTF8( $cur->{'category'} ), $LANG ] ] if $cur->{'category'};
! $prog{'episode-num'} = [[ $cur->{'episode-num'}, 'xmltv_ns' ]] if $cur->{'episode-num'};
! $prog{'start'} = $cur->{'start'} if $cur->{'start'};
! $prog{'stop'} = $cur->{'stop'} if $cur->{'stop'};
! $prog{'desc'} = [ [ toUTF8( $cur->{'desc'} ), $LANG ] ] if $cur->{'desc'};
! $prog{'icon'} = [ { 'src' => $cur->{'icon'} } ] if $cur->{'icon'};
! $prog{'credits'} = $cur->{'credits'} if $cur->{'credits'};
! $prog{'date'} = $cur->{'year'} if $cur->{'year'};
!
! return \%prog;
}
# get channel listing
sub get_channels {
! my $bar = new XMLTV::ProgressBar( 'getting list of channels', 1 ) if not $opt_quiet;
my %channels;
!
! # retrieve channels via a dummy call to the schedule page
! # http://www.zon.pt/tv/guiaTV/Pages/GuiaTV.aspx
! my $url = $HEAD->{'source-data-url'} . 'Pages/GuiaTV.aspx';
t $url;
! my $tree = get_nice_tree($url, '', 'UTF-8');
! # <div id="channels-list-container">
! # <ul id="channels-list-slider">
! # <li><span class="channel-number">001</span>
! # <span class="channel-logo">
! # <a href='/tv/guiaTV/Pages/Guia-TV-programacao.aspx?channelSigla=5' title='RTP 1'>
! # <img src='/EPGChannelImages/RTP1.png' alt='logótipo RTP 1' />
! # </a>
! # </span>
! # </li>
! #
! if ( my $h = $tree->look_down('_tag' => 'div', 'id' => 'channels-list-container') ) {
! if ( my $h2 = $h->look_down('_tag' => 'ul', 'id' => 'channels-list-slider') ) {
! my @h3 = $h2->look_down('_tag' => 'li');
! foreach my $elem (@h3) {
! my ($channel_id, $channel_name, $channel_logo);
! if ( my $h4 = $elem->look_down('_tag' => 'a') ) {
! $channel_name = toUTF8( $h4->attr('title') );
! ($channel_id) = $h4->attr('href') =~ /channelSigla=(\d*)/;
! }
! if ( my $h4 = $elem->look_down('_tag' => 'img') ) {
! $channel_logo = $SOURCE_URL . $h4->attr('src');
! }
! # store the channel
! my $ch =
{
+ 'channel-name' => $channel_name,
'display-name' => [ [ $channel_name, $LANG ] ],
! 'id' => $channel_id.'.'.$DOMAIN,
! 'icon' => [ { 'src' => $channel_logo } ],
};
+ $channels{$channel_id} = $ch;
+ push @ch_all, $ch;
+
} #foreach
! }
! }
die "no channels could be found" if not keys %channels;
update $bar if not $opt_quiet;
$tree->delete;
***************
*** 720,724 ****
}
! sub get_icons {
my %icons;
my $url= $HEAD->{"source-info-url"}."/Televisao/ListaProgramas.aspx?dia=0&package=9&cat=&channelSigla=";
--- 732,736 ----
}
! sub get_icons { # deprecated
my %icons;
my $url= $HEAD->{"source-info-url"}."/Televisao/ListaProgramas.aspx?dia=0&package=9&cat=&channelSigla=";
***************
*** 754,755 ****
--- 766,782 ----
}
+
+ # Get the user's home directory
+ sub get_default_dir {
+ my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH}
+ if defined( $ENV{HOMEDRIVE} )
+ and defined( $ENV{HOMEPATH} );
+
+ my $home = $ENV{HOME} || $winhome || ".";
+ return $home;
+ }
+
+ # Set default cache dir = $HOME/.xmltv/cache
+ sub get_default_cachedir {
+ return get_default_dir() . "/.xmltv/cache";
+ }
Index: test.conf
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/pt/test.conf,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** test.conf 25 Jan 2010 04:33:24 -0000 1.5
--- test.conf 7 May 2014 16:24:33 -0000 1.6
***************
*** 1,122 ****
! channel 10.tvcabo.pt
! channel 100.tvcabo.pt
! channel 101.tvcabo.pt
! channel 105.tvcabo.pt
! channel 108.tvcabo.pt
! #channel 109.tvcabo.pt
! #channel 11.tvcabo.pt
! #channel 111.tvcabo.pt
! #channel 112.tvcabo.pt
! #channel 113.tvcabo.pt
! #channel 114.tvcabo.pt
! #channel 115.tvcabo.pt
! #channel 116.tvcabo.pt
! #channel 117.tvcabo.pt
! #channel 118.tvcabo.pt
! #channel 119.tvcabo.pt
! #channel 120.tvcabo.pt
! #channel 121.tvcabo.pt
! #channel 123.tvcabo.pt
! #channel 124.tvcabo.pt
! #channel 125.tvcabo.pt
! #channel 127.tvcabo.pt
! #channel 129.tvcabo.pt
! #channel 13.tvcabo.pt
! #channel 130.tvcabo.pt
! #channel 131.tvcabo.pt
! #channel 132.tvcabo.pt
! #channel 133.tvcabo.pt
! #channel 134.tvcabo.pt
! #channel 135.tvcabo.pt
! #channel 136.tvcabo.pt
! #channel 137.tvcabo.pt
! #channel 138.tvcabo.pt
! #channel 140.tvcabo.pt
! #channel 141.tvcabo.pt
! #channel 142.tvcabo.pt
! #channel 143.tvcabo.pt
! #channel 144.tvcabo.pt
! #channel 145.tvcabo.pt
! #channel 146.tvcabo.pt
! #channel 147.tvcabo.pt
! #channel 148.tvcabo.pt
! #channel 149.tvcabo.pt
! #channel 16.tvcabo.pt
! #channel 160.tvcabo.pt
! #channel 161.tvcabo.pt
! #channel 162.tvcabo.pt
! #channel 163.tvcabo.pt
! #channel 164.tvcabo.pt
! #channel 165.tvcabo.pt
! #channel 166.tvcabo.pt
! #channel 167.tvcabo.pt
! #channel 168.tvcabo.pt
! #channel 169.tvcabo.pt
! #channel 17.tvcabo.pt
! #channel 171.tvcabo.pt
! #channel 172.tvcabo.pt
! #channel 174.tvcabo.pt
! #channel 175.tvcabo.pt
! #channel 176.tvcabo.pt
! #channel 177.tvcabo.pt
! #channel 178.tvcabo.pt
! #channel 18.tvcabo.pt
! #channel 19.tvcabo.pt
! #channel 20.tvcabo.pt
! #channel 22.tvcabo.pt
! #channel 23.tvcabo.pt
! #channel 25.tvcabo.pt
! #channel 26.tvcabo.pt
! #channel 27.tvcabo.pt
! #channel 3.tvcabo.pt
! #channel 31.tvcabo.pt
! #channel 32.tvcabo.pt
! #channel 34.tvcabo.pt
! #channel 35.tvcabo.pt
! #channel 36.tvcabo.pt
! #channel 37.tvcabo.pt
! #channel 38.tvcabo.pt
! #channel 39.tvcabo.pt
! #channel 40.tvcabo.pt
! #channel 41.tvcabo.pt
! #channel 43.tvcabo.pt
! #channel 44.tvcabo.pt
! #channel 45.tvcabo.pt
! #channel 48.tvcabo.pt
! #channel 49.tvcabo.pt
! #channel 5.tvcabo.pt
! #channel 50.tvcabo.pt
! #channel 51.tvcabo.pt
! #channel 52.tvcabo.pt
! #channel 53.tvcabo.pt
! #channel 54.tvcabo.pt
! #channel 57.tvcabo.pt
! #channel 59.tvcabo.pt
! #channel 61.tvcabo.pt
! #channel 64.tvcabo.pt
! #channel 66.tvcabo.pt
! #channel 7.tvcabo.pt
! #channel 71.tvcabo.pt
! #channel 76.tvcabo.pt
! #channel 77.tvcabo.pt
! #channel 78.tvcabo.pt
! #channel 79.tvcabo.pt
! #channel 8.tvcabo.pt
! #channel 80.tvcabo.pt
! #channel 81.tvcabo.pt
! #channel 84.tvcabo.pt
! #channel 85.tvcabo.pt
! #channel 86.tvcabo.pt
! #channel 87.tvcabo.pt
! #channel 88.tvcabo.pt
! #channel 89.tvcabo.pt
! #channel 9.tvcabo.pt
! #channel 90.tvcabo.pt
! #channel 91.tvcabo.pt
! #channel 92.tvcabo.pt
! #channel 93.tvcabo.pt
! #channel 94.tvcabo.pt
! #channel 95.tvcabo.pt
! #channel 97.tvcabo.pt
! #channel 98.tvcabo.pt
! #channel 99.tvcabo.pt
--- 1,178 ----
! #channel 10.zon.pt
! #channel 100.zon.pt
! #channel 101.zon.pt
! #channel 105.zon.pt
! #channel 108.zon.pt
! #channel 109.zon.pt
! #channel 11.zon.pt
! #channel 111.zon.pt
! #channel 112.zon.pt
! #channel 113.zon.pt
! #channel 114.zon.pt
! #channel 115.zon.pt
! #channel 116.zon.pt
! #channel 117.zon.pt
! #channel 118.zon.pt
! #channel 119.zon.pt
! #channel 120.zon.pt
! #channel 121.zon.pt
! #channel 123.zon.pt
! #channel 124.zon.pt
! #channel 125.zon.pt
! #channel 127.zon.pt
! #channel 129.zon.pt
! #channel 13.zon.pt
! #channel 130.zon.pt
! #channel 132.zon.pt
! #channel 133.zon.pt
! #channel 134.zon.pt
! #channel 135.zon.pt
! #channel 136.zon.pt
! #channel 137.zon.pt
! #channel 138.zon.pt
! #channel 139.zon.pt
! #channel 14.zon.pt
! #channel 141.zon.pt
! #channel 142.zon.pt
! #channel 143.zon.pt
! #channel 144.zon.pt
! #channel 145.zon.pt
! #channel 148.zon.pt
! #channel 149.zon.pt
! #channel 15.zon.pt
! channel 16.zon.pt
! #channel 160.zon.pt
! #channel 161.zon.pt
! #channel 162.zon.pt
! #channel 163.zon.pt
! #channel 164.zon.pt
! #channel 165.zon.pt
! #channel 167.zon.pt
! #channel 168.zon.pt
! #channel 169.zon.pt
! #channel 17.zon.pt
! #channel 171.zon.pt
! #channel 172.zon.pt
! #channel 174.zon.pt
! #channel 175.zon.pt
! #channel 176.zon.pt
! #channel 177.zon.pt
! #channel 178.zon.pt
! #channel 179.zon.pt
! #channel 18.zon.pt
! #channel 180.zon.pt
! #channel 181.zon.pt
! #channel 182.zon.pt
! #channel 183.zon.pt
! #channel 184.zon.pt
! #channel 185.zon.pt
! #channel 186.zon.pt
! #channel 187.zon.pt
! #channel 188.zon.pt
! #channel 19.zon.pt
! #channel 190.zon.pt
! #channel 191.zon.pt
! #channel 194.zon.pt
! #channel 196.zon.pt
! #channel 20.zon.pt
! #channel 200.zon.pt
! #channel 201.zon.pt
! #channel 202.zon.pt
! #channel 203.zon.pt
! #channel 21.zon.pt
! #channel 22.zon.pt
! #channel 221.zon.pt
! #channel 222.zon.pt
! #channel 223.zon.pt
! #channel 224.zon.pt
! #channel 226.zon.pt
! #channel 229.zon.pt
! #channel 23.zon.pt
! #channel 230.zon.pt
! #channel 232.zon.pt
! #channel 233.zon.pt
! #channel 234.zon.pt
! #channel 235.zon.pt
! #channel 236.zon.pt
! #channel 237.zon.pt
! #channel 24.zon.pt
! #channel 242.zon.pt
! #channel 243.zon.pt
! #channel 244.zon.pt
! #channel 245.zon.pt
! #channel 246.zon.pt
! #channel 247.zon.pt
! #channel 248.zon.pt
! #channel 249.zon.pt
! #channel 25.zon.pt
! #channel 250.zon.pt
! #channel 251.zon.pt
! #channel 252.zon.pt
! #channel 26.zon.pt
! #channel 27.zon.pt
! #channel 28.zon.pt
! #channel 29.zon.pt
! channel 3.zon.pt
! #channel 30.zon.pt
! #channel 31.zon.pt
! #channel 33.zon.pt
! #channel 34.zon.pt
! #channel 35.zon.pt
! #channel 36.zon.pt
! #channel 37.zon.pt
! channel 38.zon.pt
! #channel 39.zon.pt
! #channel 4.zon.pt
! #channel 40.zon.pt
! #channel 41.zon.pt
! #channel 42.zon.pt
! #channel 44.zon.pt
! #channel 45.zon.pt
! #channel 46.zon.pt
! #channel 47.zon.pt
! #channel 48.zon.pt
! #channel 49.zon.pt
! channel 5.zon.pt
! #channel 50.zon.pt
! #channel 51.zon.pt
! #channel 52.zon.pt
! #channel 53.zon.pt
! #channel 54.zon.pt
! #channel 55.zon.pt
! #channel 57.zon.pt
! #channel 58.zon.pt
! #channel 59.zon.pt
! #channel 60.zon.pt
! #channel 61.zon.pt
! #channel 62.zon.pt
! #channel 64.zon.pt
! #channel 65.zon.pt
! #channel 66.zon.pt
! #channel 67.zon.pt
! #channel 69.zon.pt
! #channel 7.zon.pt
! #channel 70.zon.pt
! #channel 71.zon.pt
! #channel 72.zon.pt
! #channel 74.zon.pt
! #channel 75.zon.pt
! #channel 76.zon.pt
! #channel 77.zon.pt
! #channel 78.zon.pt
! #channel 79.zon.pt
! #channel 8.zon.pt
! #channel 80.zon.pt
! #channel 81.zon.pt
! #channel 82.zon.pt
! #channel 83.zon.pt
! #channel 84.zon.pt
! #channel 85.zon.pt
! #channel 86.zon.pt
! #channel 9.zon.pt
! #channel 90.zon.pt
! #channel 91.zon.pt
! #channel 92.zon.pt
! #channel 93.zon.pt
! #channel 94.zon.pt
! #channel 95.zon.pt
! #channel 98.zon.pt
------------------------------------------------------------------------------
Is your legacy SCM system holding you back? Join Perforce May 7 to find out:
• 3 signs your SCM is hindering your productivity
• Requirements for releasing software faster
• Expert tips and advice for migrating your SCM now
http://p.sf.net/sfu/perforce
_______________________________________________
xmltv-commit mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/xmltv-commit