xmltv/grab/es_laguiatv test.conf, 1.6, 1.7 tv_grab_es_laguiatv, 1.20, 1.21
Geoff <[email protected]>
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/grab/es_laguiatv
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv6157
Modified Files:
test.conf tv_grab_es_laguiatv
Log Message:
(major) Fix broken grabber
Index: tv_grab_es_laguiatv
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/es_laguiatv/tv_grab_es_laguiatv,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** tv_grab_es_laguiatv 1 Jan 2012 18:45:43 -0000 1.20
--- tv_grab_es_laguiatv 2 Feb 2014 12:21:32 -0000 1.21
***************
*** 88,93 ****
--- 88,95 ----
use HTML::Entities; # parse entities
use IO::File;
+ use DateTime;
use LWP::Simple;
+ use Encode;
use XMLTV;
***************
*** 97,101 ****
use XMLTV::Config_file;
use XMLTV::DST;
! use XMLTV::Get_nice;
use XMLTV::Mode;
use XMLTV::Date;
--- 99,103 ----
use XMLTV::Config_file;
use XMLTV::DST;
! use XMLTV::Get_nice 0.005065;
use XMLTV::Mode;
use XMLTV::Date;
***************
*** 113,141 ****
# Attributes of the root element in output.
! my $HEAD = { 'source-info-url' => 'http://www.laguiatv.com/programacion.php',
! 'source-data-url' => "http://www.laguiatv.com/programacion_vertical.php",
'generator-info-name' => 'XMLTV',
'generator-info-url' => 'http://xmltv.org/',
};
! # Whether zero-length programmes should be included in the output.
! my $WRITE_ZERO_LENGTH = 0;
my $DO_SLOWER_DESC_GET = 0;
my $CONFIG_VERSION = 1; # default to v1 (v1 doesnt have version info)
! my $EXPECTED_CONFIG_VERSION = 2;
# default language
my $LANG="es";
# Global channel_data
our @ch_all;
- # debug print function
- sub debug_print
- {
- # my ($str) = @_;
- # print "$str";
- }
-
my @hide_channels = (
"canal-bar.a", # currently gives 404 not found
--- 115,140 ----
# Attributes of the root element in output.
! my $HEAD = { 'source-info-url' => 'http://www.laguiatv.com/programacion/',
! 'source-data-url' => "http://www.laguiatv.com/programacion/",
'generator-info-name' => 'XMLTV',
'generator-info-url' => 'http://xmltv.org/',
};
! my $WRITE_ZERO_LENGTH = 0; # whether zero-length programmes should be included in the output.
my $DO_SLOWER_DESC_GET = 0;
my $CONFIG_VERSION = 1; # default to v1 (v1 doesnt have version info)
! my $EXPECTED_CONFIG_VERSION = 3;
! my $CONFIG_USECACHE = 0; # whether to use a disc cache for web pages
! my $CONFIG_CACHEDIR; # directory to store cached web pages
# default language
my $LANG="es";
+ # default web page encoding
+ my $WEB_ENCODING = 'iso-8859-15';
+
# Global channel_data
our @ch_all;
my @hide_channels = (
"canal-bar.a", # currently gives 404 not found
***************
*** 150,157 ****
my ($opt_days, $opt_offset, $opt_help, $opt_output,
$opt_configure, $opt_config_file, $opt_gui,
! $opt_quiet, $opt_list_channels);
! $opt_days = 1; # default
$opt_offset = 0; # default
$opt_quiet = 0; # default
GetOptions('days=i' => \$opt_days,
'offset=i' => \$opt_offset,
--- 149,157 ----
my ($opt_days, $opt_offset, $opt_help, $opt_output,
$opt_configure, $opt_config_file, $opt_gui,
! $opt_quiet, $opt_list_channels, $opt_debug);
! $opt_days = 4; # default
$opt_offset = 0; # default
$opt_quiet = 0; # default
+ $opt_debug = 0; # default
GetOptions('days=i' => \$opt_days,
'offset=i' => \$opt_offset,
***************
*** 162,177 ****
'output=s' => \$opt_output,
'quiet' => \$opt_quiet,
! 'list-channels' => \$opt_list_channels
)
or usage(0);
# Force days to be 1, since we get all days at once
! $opt_days = 1;
die 'number of days must not be negative'
if (defined $opt_days && $opt_days < 0);
usage(1) if $opt_help;
XMLTV::Ask::init($opt_gui);
my $mode = XMLTV::Mode::mode('grab', # default
$opt_configure => 'configure',
--- 162,196 ----
'output=s' => \$opt_output,
'quiet' => \$opt_quiet,
! 'list-channels' => \$opt_list_channels,
! 'debug' => \$opt_debug,
)
or usage(0);
# Force days to be 1, since we get all days at once
! # $opt_days = 1;
die 'number of days must not be negative'
if (defined $opt_days && $opt_days < 0);
usage(1) if $opt_help;
+ # [mod Jan 2014 - max days is 4
+ die 'max days available is 4 (today + 3)'
+ if ( $opt_offset + $opt_days > 4 );
+
XMLTV::Ask::init($opt_gui);
+
+ # Although we use HTTP::Cache::Transparent, this undocumented --cache
+ # option for debugging is still useful since it will _always_ use a
+ # cached copy of a page, without contacting the server at all.
+ #
+ use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
+
+
+ # debug print function
+ sub debug_print
+ {
+ print STDERR $_[0]."\n" if $opt_debug;
+ }
+
my $mode = XMLTV::Mode::mode('grab', # default
$opt_configure => 'configure',
***************
*** 196,200 ****
# Whatever we are doing, we need the channels data.
! my %channels; # sets @ch_all
my @channels;
--- 215,219 ----
# Whatever we are doing, we need the channels data.
! my %channels = get_channels(); # sets @ch_all
my @channels;
***************
*** 217,223 ****
open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
! print CONF "configversion 2\n";
# Ask about getting descs
! my $getdescs = ask_boolean("Do you want to get descriptions (very slow)");
warn("cannot read input, using default")
if not defined $getdescs;
--- 236,261 ----
open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
! print CONF "configversion 3\n";
!
! # Ask about using a cache
! my $usecache = ask_boolean("Do you want to use a cache for web pages (recommended)", 'yes');
! warn("cannot read input, using default")
! if not defined $usecache;
!
! print CONF "usecache ";
! print CONF "yes\n" if $usecache;
! print CONF "no\n" if not $usecache;
!
! my $cachedir = "$ENV{HOME}/.xmltv/cache";
! if ($usecache)
! {
! my $cachedir = ask("Directory for cache (default=$cachedir)");
! warn("cannot read input, using default")
! if not defined $cachedir;
! }
! print CONF "cachedir ".$cachedir."\n";
!
# Ask about getting descs
! my $getdescs = ask_boolean("Do you want to get descriptions (very slow)", 'yes');
warn("cannot read input, using default")
if not defined $getdescs;
***************
*** 256,260 ****
# else
# {
! print CONF "channel $_ $name\n";
# }
# TODO don't store display-name in config file.
--- 294,298 ----
# else
# {
! print CONF "channel $_ ".encode($WEB_ENCODING, $name)."\n";
# }
# TODO don't store display-name in config file.
***************
*** 282,286 ****
$w_args{OUTPUT} = $fh;
}
! $w_args{encoding} = 'ISO-8859-1';
my $writer = new XMLTV::Writer(%w_args);
$writer->start($HEAD);
--- 320,324 ----
$w_args{OUTPUT} = $fh;
}
! $w_args{encoding} = 'UTF-8';
my $writer = new XMLTV::Writer(%w_args);
$writer->start($HEAD);
***************
*** 306,309 ****
--- 344,358 ----
$CONFIG_VERSION = $1;
}
+ elsif (/usecache:?\s+(\S+)/)
+ {
+ if($1 eq "yes")
+ {
+ $CONFIG_USECACHE = 1;
+ }
+ }
+ elsif (/cachedir:?\s+(\S+)/)
+ {
+ $CONFIG_CACHEDIR = $1;
+ }
elsif (/getdescriptions:?\s+(\S+)/)
{
***************
*** 335,339 ****
my $ch_name = $2;
! #debug_print "Got channel $ch_name icon $ch_icon\n";
$ch_name =~ s/\s*$//;
push @channels, $ch_did;
--- 384,388 ----
my $ch_name = $2;
! debug_print "Fetching channel $ch_name";
$ch_name =~ s/\s*$//;
push @channels, $ch_did;
***************
*** 345,348 ****
--- 394,412 ----
}
+
+
+ if ($CONFIG_USECACHE) {
+ use HTTP::Cache::Transparent;
+ HTTP::Cache::Transparent::init( {
+ BasePath => $CONFIG_CACHEDIR,
+ NoUpdate => 60*60, # cache time in seconds
+ MaxAge => 4, # flush time in hours
+ Verbose => $opt_debug,
+ } );
+ }
+
+
+
+
######################################################################
# begin main program
***************
*** 377,386 ****
'display-name' => [ [ $ch_name ] ] });
# }
! my $day=UnixDate($now,'%Q');
! for (my $i=0;$i<$opt_days;$i++) {
! push @to_get, [ $day, $ch_xid, $ch_did ];
! #for each day
! $day=nextday($day); die if not defined $day;
! }
update $iconbar if not $opt_quiet;
}
--- 441,455 ----
'display-name' => [ [ $ch_name ] ] });
# }
!
! # [Jan 2014] - current website offers a fixed 4 days of data
! # my $day=UnixDate($now,'%Q');
! # for (my $i=0;$i<$opt_days;$i++) {
! # push @to_get, [ $day, $ch_xid, $ch_did ];
! # #for each day
! # $day=nextday($day); die if not defined $day;
! # }
! #
! push @to_get, [ '', $ch_xid, $ch_did ];
!
update $iconbar if not $opt_quiet;
}
***************
*** 392,396 ****
if not $opt_quiet;
foreach (@to_get) {
! debug_print("process $_->[0], $_->[1], $_->[2]\n");
foreach (process_table($_->[0], $_->[1], $_->[2])) {
$writer->write_programme($_);
--- 461,465 ----
if not $opt_quiet;
foreach (@to_get) {
! debug_print "process $_->[0], $_->[1], $_->[2]\n";
foreach (process_table($_->[0], $_->[1], $_->[2])) {
$writer->write_programme($_);
***************
*** 431,438 ****
my ($date, $ch_xmltv_id, $ch_es_id) = @_;
my $ch_conv_id = convert_id_to_laguiatvid($ch_es_id);
my $today = UnixDate($date, '%d/%m/%Y');
! my $url = "http://www.laguiatv.com/programacion/$ch_es_id";
debug_print "Getting $url\n";
t $url;
--- 500,508 ----
my ($date, $ch_xmltv_id, $ch_es_id) = @_;
+
my $ch_conv_id = convert_id_to_laguiatvid($ch_es_id);
my $today = UnixDate($date, '%d/%m/%Y');
! my $url = 'http://www.laguiatv.com/programacion/'.$ch_es_id.'.html';
debug_print "Getting $url\n";
t $url;
***************
*** 443,447 ****
# parse the page to a document object
! my $tree = get_nice_tree $url;
my @program_data = get_program_data($tree);
my $bump_start_day=0;
--- 513,527 ----
# parse the page to a document object
! my $tree;
! # HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages!
! # Save the page and run it again and you don't get the warning!
! # You can't even supress the warning! What a crock.
! {
! local $SIG{__WARN__} = sub {
! warn @_ unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/);
! };
! $tree = get_nice_tree($url,'',$WEB_ENCODING);
! }
!
my @program_data = get_program_data($tree);
my $bump_start_day=0;
***************
*** 474,495 ****
my ($date, $ch_xmltv_id, $ch_es_id, $cur, $next) = @_;
my %prog;
$prog{channel}=$ch_xmltv_id;
! $prog{title}=[ [ $cur->{title}, $LANG ] ];
! $prog{"sub-title"}=[ [ $cur->{subtitle}, $LANG ] ] if defined $cur->{subtitle};
# $prog{category}=[ [ $cur->{category}, $LANG ] ];
! t "turning local time $cur->{time}, on date $date, into UTC";
! eval { $prog{start}=utc_offset("$date $cur->{time}", '+0100') };
! if ($@) {
! warn "bad time string: $cur->{time}";
! return undef;
}
! t "...got $prog{start}";
! # FIXME: parse description field further
- $prog{desc}=[ [ $cur->{desc}, $LANG ] ] if defined $cur->{desc};
- $prog{category}=[ [ $cur->{category}, $LANG ] ] if defined $cur->{category};
return \%prog;
}
--- 554,596 ----
my ($date, $ch_xmltv_id, $ch_es_id, $cur, $next) = @_;
+ #require Data::Dumper; debug_print Data::Dumper::Dumper($cur);
+
my %prog;
$prog{channel}=$ch_xmltv_id;
! $prog{title}=[ [ encode( 'UTF-8', $cur->{title} ), $LANG ] ];
! $prog{"sub-title"}=[ [ encode( 'UTF-8', $cur->{subtitle} ), $LANG ] ] if defined $cur->{subtitle};
# $prog{category}=[ [ $cur->{category}, $LANG ] ];
+ $prog{start}=$cur->{stime};
+ $prog{stop} =$cur->{etime} if defined $cur->{etime};
+ $prog{desc}=[ [ encode( 'UTF-8', $cur->{desc} ), $LANG ] ] if defined $cur->{desc};
+ # $prog{category}=[ [ encode( 'UTF-8', $cur->{category} ), $LANG ] ] if defined $cur->{category};
+ $prog{'date'} = $cur->{year} if defined $cur->{year};
+ $prog{'star-rating'} = [ $cur->{rating} . '/5' ] if defined $cur->{rating};
+ $prog{'rating'} = [[ $cur->{classification}, '' ]] if defined $cur->{classification};
! if (defined $cur->{genres})
! {
! foreach ( @{ $cur->{genres} } )
! {
! push @{$prog{'category'}}, [ encode('UTF-8', $_), $LANG ] if $_ ne '';
! }
! }
! if (defined $cur->{directors})
! {
! foreach ( @{ $cur->{directors} } )
! {
! push @{$prog{'credits'}{'director'}}, encode('UTF-8', $_) if $_ ne '';
! }
! }
! if (defined $cur->{actors})
! {
! foreach ( @{ $cur->{actors} } )
! {
! push @{$prog{'credits'}{'actor'}}, encode('UTF-8', $_) if $_ ne '';
}
! }
!
return \%prog;
}
***************
*** 515,570 ****
my @data;
# find schedule table
! my @divs = $tree->find_by_tag_name("_tag"=>"div");
foreach my $div (@divs)
{
! my $class = $div->attr('class');
! if ($class && $class eq "clip")
{
- debug_print("Got clip\n");
- my $p_category = "";
! my @litems = $div->find_by_tag_name("_tag"=>"li");
! foreach my $litem (@litems)
{
! $p_category = $litem->attr('class');
}
! my @links = $div->find_by_tag_name("_tag"=>"a");
! foreach my $link (@links)
{
- my @txts = get_txt_elems($link);
- my $p_title = $txts[0];
- my ($p_stime, $ignore, $p_etime) = split(/\ /, $txts[1]);
- my $p_description = "";
! debug_print("title: $p_title start: $p_stime end: $p_etime cat: $p_category c2: " . $categories{$p_category} . "\n");
! if($p_title && $p_title ne "" && $p_stime && $p_stime ne "")
{
! my %h = (time => $p_stime,
! title=> $p_title);
! if($p_category && $p_category ne "")
{
! my $cat = $categories{$p_category};
! if($cat && $cat ne "")
{
! $h{category} = $cat;
}
}
$h{desc} = $p_description if $p_description ne "";
push @data, \%h;
- $p_title = "";
- $p_stime = "";
- $p_category = "";
- }
}
}
}
return @data;
--- 616,874 ----
my @data;
+ my $today = DateTime->today->set_time_zone('Europe/Madrid');
+
+ # - current website offers a fixed 4 days of data
+ # ignore any programmes outside requested range
+ my $startgrab = $today->clone->add('days' => $opt_offset)->epoch();
+ my $stopgrab = $today->clone->add('days' => ($opt_offset + $opt_days))->epoch();
+ debug_print 'Grab times: start: '.DateTime->from_epoch(epoch=>$startgrab)->strftime("%Y %m %d %H%M %S %z").' stop: '.DateTime->from_epoch(epoch=>$stopgrab)->strftime("%Y %m %d %H%M %S %z");
# find schedule table
! # the following could could do with some error checking but I don't have time to do that right now :-(
!
! my @divs = $tree->look_down('_tag' => 'div', 'id' => qr/dia1|nad2|nad3|nad4/);
foreach my $div (@divs)
{
! my ($i) = $div->attr('id') =~ /(?:dia|nad)(\d)/;
! #'debugtime' debug_print "i= $i ".$div->attr('id');
! #'debugtime' debug_print 'today: '.$today->strftime("%Y %m %d %H%M %S %z");
! my $theday = $today->clone->add(days => ($i - 1));
! #'debugtime' debug_print 'theday: '.$theday->strftime("%Y %m %d %H%M %S %z");
! my @trs = $div->look_down('_tag' => 'tr');
!
! foreach my $tr (@trs)
{
! my $stime = $tr->look_down('_tag' => 'th')->as_text;
! trim($stime);
!
! my $p_div = $tr->look_down('_tag' => 'div', 'class' => 'programa');
! next if !$p_div;
!
! my $a = $p_div->look_down('_tag' => 'a');
!
! my $p_url = $a->attr('href');
! my $p_title = $a->as_text;
!
! my $p_times = $p_div->look_down('_tag' => 'p')->as_text;
!
! my ($h, $i, $h2, $i2) = $p_times =~ /(\d*):(\d*) a (\d*):(\d*)/;
!
! my $showtime = $theday->clone->set(hour => $h, minute => $i, second => 0);
!
! # - current website offers a fixed 4 days of data
! # ignore any programmes outside requested range
! #'debugtime' debug_print 'this: '.$showtime->strftime("%Y %m %d %H%M %S %z");
! next if ( $showtime->epoch() < $startgrab ) || ( $showtime->epoch() >= $stopgrab );
!
! my $p_stime = $theday->clone->set(hour => $h, minute => $i, second => 0)->strftime("%Y%m%d%H%M%S %z");
!
! my $p_etime;
! # this will probably fail around DST times
! if (defined $h2 && $h2 >= 0)
{
! $showtime->add(days => 1) if $h2 < $h;
! eval { # try
! $showtime->set(hour => $h2, minute => $i2, second => 0);
! $p_etime = $showtime->strftime("%Y%m%d%H%M%S %z");
! } or do { # catch
! # no output prog 'stop' time
}
+ }
+
! # get descriptions? Kinda compulsory now since there is no longer *any* description on the schedule page
! #
! my ($p_description, $p_rating, $p_classification, $p_year, @p_genres, @p_actors, @p_directors) = ('', '', '', '', (), (), ());
! #
! { # begin code block
! if ($DO_SLOWER_DESC_GET) # get descriptions
{
! my $url = $p_url;
! debug_print "Getting $url";
! t $url;
!
! last if $url eq 'javascript:void(0);' ;
!
! # parse the page to a document object
! # HTML::Parse keeps reporting "Parsing of undecoded UTF-8 will give garbage when decoding entities" yet I can see no UTF8 in the pages!
! # Often on the http://hoycinema.abc.es/ pages. (Could be due to the <script> they have *before* the <meta Content-Type> ?)
! {
! local $SIG{__WARN__} = sub {
! warn @_ unless (defined $_[0] && $_[0] =~ /^Parsing of undecoded UTF-/);
! };
! $tree = get_nice_tree($url,'',$WEB_ENCODING);
! }
!
! my $div = $tree->look_down('_tag' => 'div', 'id' => 'contenedor'); # container
! $tree->dump if !$div;
! exit if !$div;
!
! # see if the title has a year
! # <h1 itemprop="name">«Fin de semana al desnudo»
! # <span><a href="/peliculas/1974.html" title="PelÃculas del año 1974">(1974)</a></span>
! # </h1>
! my $h1 = $div->look_down('_tag' => 'h1', 'itemprop' => 'name');
! if ($h1)
! {
! my $a = $h1->look_down('_tag' => 'a');
! ($p_year) = $a->as_text =~ /^\((19\d\d|20\d\d)\)$/ if $a;
! }
!
! $div = $div->look_down('_tag' => 'div', 'class' => 'modulo');
! last if !$div;
!
! # get the various <dl> blocks
! my @dls = $div->look_down('_tag' => 'dl');
! foreach my $dl (@dls)
! {
! my $dt = $dl->look_down('_tag' => 'dt');
! next if !$dt;
!
! if ($dt->as_text =~ /Informaci.n/)
{
! #<dl class="datos">
! # <dt>Información:</dt>
! # <dd class="calificacion">SC</dd>
! # <dd itemprop="genre">Comedia</dd>
! # <dd itemprop="duration">93 minutos</dd>
! # <dd>Sin especificar</dd> OR e.g. <dd>EE.UU.</dd>
! #</dl>
! my $t = $dl->look_down('_tag' => 'dd', 'class' => 'calificacion');
! $p_classification = $t->as_text if $t;
! #
! $t = $dl->look_down('_tag' => 'dd', 'itemprop' => 'genre');
! @p_genres = split(/,/, $t->as_text) if $t;
! }
!
! if ($dt->as_text =~ /Director/)
! {
! #<dl>
! # <dt>Director:</dt>
! # <dd itemprop="director" itemscope itemtype="http://schema.org/Person">
! # <a href="/perfil-cine/peter-webber-97721/" title="Peter Webber"><span itemprop="name">Peter Webber</span></a>
! # <a href="/perfil-cine/mariano-ozores-20092/" title="Mariano Ozores"><span itemprop="name">Mariano Ozores</span></a>
! # </dd>
! #</dl>
! my @t = $dl->look_down('_tag' => 'span', 'itemprop' => 'name');
! foreach (@t)
! {
! push @p_directors, $_->as_text;
! }
! }
!
! if ($dt->as_text =~ /Int.rpretes/)
! {
! #<dl class="interpretes" itemprop="actor" itemscope itemtype="http://schema.org/Person">
! # <dt>Intérpretes:</dt>
! # <dd>
! # <a href="/perfil-cine/mandy-patinkin-4279/" title="Mandy Patinkin"><span itemprop="name">Mandy Patinkin</span></a>,
! # <a href="/perfil-cine/alfredo-landa-15923/" title="Alfredo Landa"><span itemprop="name">Alfredo Landa</span></a>,
! # <a href="/perfil-cine/thomas-gibson-18922/" title="Thomas Gibson"><span itemprop="name">Thomas Gibson</span></a>,
! # </dd>
! # <dd class="enlace"><a href="/peliculas/1974/fin-de-semana-al-desnudo-9814/reparto.html" title="Reparto completo">Reparto completo</a></dd>
! #</dl>
! my @t = $dl->look_down('_tag' => 'span', 'itemprop' => 'name');
! foreach (@t)
! {
! push @p_actors, $_->as_text;
! }
! # note: we could follow the "Reparto completo" link for the complete cast list
! }
!
! if ($dt->as_text =~ /Descripci.n/)
! {
! #<dl>
! # <dt>Descripción:</dt>
! # <dd>Programa que repasa todas las noticias de interés general, nacionales, internacionales, asà como deportivas. Incluye El tiempo. Presentador: Albert MartÃnez.</dd>
! #</dl>
! my $t = $dl->look_down('_tag' => 'dd');
! $p_description = $t->as_text if $t;
! }
!
! if ($dt->as_text =~ /Sinopsis/)
! {
! #<dl class="sinopsis">
! # <dt>Sinopsis:</dt>
! # <dd itemprop="description">La historia de la psicóloga Virginia Johnson (Lizzy Caplan) y el tÃmido ginecólogo William Masters (Michael Sheen),...<a href="/series/masters-of-sex-25058/sinopsis.html" title="Leer sinopsis completa">Leer sinopsis completa</a>.</dd>
! #</dl>
! #
! # For a *really* long description we could follow the 'Leer sinopsis completa' link but we won't do that until someone asks for it! ;-)
! #
! if ($p_description eq '')
! {
! my $t = $dl->look_down('_tag' => 'dd', 'itemprop' => 'description');
! if ($t)
! {
! my $a = $t->look_down('_tag' => 'a');
! $a->detach if $a;
! $p_description = $t->as_text;
! }
! }
! }
!
! }
!
! # if no description then try for a synopis
! # <aside id="sinopsis"><h2>Sinopsis</h2><p itemprop="description">...
! if ($p_description eq '')
! {
! my $h2 = $tree->look_down('_tag' => 'h2', sub { $_[0]->as_text =~ /Sinopsis/ } );
! if ($h2)
{
! my $p = $h2->right();
! $p_description = $p->as_text if ( $p->tag() eq 'p' && $p->attr('itemprop') eq 'description' );
! #
! # website sometimes has invalid html (nested <p>) which treebuilder flattens
! # so append <p> siblings
! while (1)
! {
! $p = $p->right();
! last if ( $p->tag() ne 'p' );
! $p_description .= $p->as_text if ( $p->tag() eq 'p' );
! }
! }
! }
!
! # rating (x/5)
! # <meta itemprop="ratingValue" content="2.2"/>
! my $meta = $div->look_down('_tag' => 'meta', 'itemprop' => 'ratingValue');
! if ($meta)
{
! $p_rating = $meta->attr('content');
}
+
+
}
+ } # end code block
+
+ #debug_print("title: $p_title start: $p_stime end: $p_etime cat: $p_category c2: " . $categories{$p_category} . "\n");
+ debug_print("title: $p_title start: $p_stime end: $p_etime");
+
+ if($p_title && $p_title ne "" && $p_stime && $p_stime ne "")
+ {
+ my %h = ('stime' => $p_stime,
+ 'etime' => $p_etime,
+ 'title' => $p_title,
+ );
+ $h{year} = $p_year if defined $p_year && $p_year ne "";
+ $h{rating} = $p_rating if $p_rating ne "";
$h{desc} = $p_description if $p_description ne "";
+ $h{classification} = $p_classification if $p_classification ne "";
+ $h{directors} = \@p_directors if scalar @p_directors > 0;
+ $h{actors} = \@p_actors if scalar @p_actors > 0;
+ $h{genres} = \@p_genres if scalar @p_genres > 0;
push @data, \%h;
}
+
}
+
}
return @data;
***************
*** 672,701 ****
if not $opt_quiet;
my %channels;
! my $url="http://www.laguiatv.com/programacion";
t $url;
my $channel_id;
my $channel_name;
! my $elem;
!
! my $tree = get_nice_tree $url;
! my @uls = $tree->find_by_tag_name("_tag"=>"ul");
! foreach my $ul (@uls)
! {
! my $uclass = $ul->attr('class');
! if ($uclass && $uclass eq 'channel-list')
! {
! my @links = $ul->find_by_tag_name("_tag"=>"a");
! # <a href="/programacion/intereconomia" title="IntereconomÃa">
! foreach my $link (@links)
{
! $channel_name = $link->attr('title');
! $channel_id = $link->attr('href');
! $channel_id =~ s/^\/programacion\///;
my $hide = 0;
foreach my $hide_id (@hide_channels)
--- 976,1003 ----
if not $opt_quiet;
my %channels;
!
! # the front page is very big and slow to parse, so we'll
! # get channels via a dummy call to TVE 1 and then parse out the channel selector
! my $url="http://www.laguiatv.com/programacion/tve-1-807.html";
t $url;
my $channel_id;
my $channel_name;
! my $channel_num;
! my $tree = get_nice_tree($url,'',$WEB_ENCODING);
! my @options = $tree->look_down('_tag' => 'select', 'id' => 'cadenas_programacion')->look_down('_tag' => 'option');
! foreach my $option (@options)
{
! next if !$option->attr('value');
! # <option value="tve-1-807">TVE 1</option>
! $channel_name = $option->as_text;
! $channel_id = $option->attr('value');
! ($channel_num) = $option->attr('value') =~ /.*?-(\d+)$/;
+ # remove channels that should not be listed
my $hide = 0;
foreach my $hide_id (@hide_channels)
***************
*** 710,721 ****
{
$channels{$channel_id}=$channel_name;
! print("Got channel $channel_name with id $channel_id\n");
! }
! }
! }
! }
! # remove channels that should not be listed
die "no channels could be found" if not keys %channels;
--- 1012,1026 ----
{
$channels{$channel_id}=$channel_name;
! debug_print "Got channel $channel_name with id $channel_id" if $opt_list_channels;
! my $coded_chan_name=encode("utf-8",$channel_name);
! push @ch_all, {
! 'display-name' => [[ $coded_chan_name, $LANG ],[$channel_num]],
! 'channel-num' => $channel_num,
! 'id'=> "$channel_id.laguiatv"
! };
! }
+ }
die "no channels could be found" if not keys %channels;
***************
*** 759,760 ****
--- 1064,1076 ----
return UnixDate($n, '%Q');
}
+
+ sub trim {
+ # Remove leading & trailing spaces
+ $_[0] =~ s/^\s+|\s+$//g;
+ }
+
+ sub utf8 {
+ # Catch the error:
+ # "Parsing of undecoded UTF-8 will give garbage when decoding entities
+ return decode('UTF-8', $_[0]);
+ }
Index: test.conf
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/es_laguiatv/test.conf,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** test.conf 1 Jan 2012 18:45:43 -0000 1.6
--- test.conf 2 Feb 2014 12:21:32 -0000 1.7
***************
*** 1,167 ****
! configversion 2
getdescriptions yes
! #channel 13tv 13TV
! #channel 40-latino 40 Latino
! #channel 40tv 40TV
! #channel 7rm 7RM
! #channel axn AXN
! #channel axn-hd AXN HD
! #channel al-jazeera-english Al Jazeera English
! #channel andalucia-tv Andalucía TV
! #channel animax Animax
! channel antena-3 Antena 3
! #channel aragon-tv Aragón TV
! #channel bbc-world BBC World
! #channel baby-tv Baby TV
! #channel biography Biography
! #channel bloomberg Bloomberg
! #channel boing Boing
! #channel boomerang Boomerang
! #channel cmt-1 CMT 1
! #channel cmt-2 CMT 2
! #channel cnbc CNBC
! #channel cnn-int CNN Int
! #channel cnnplus CNN+
! #channel calle-13 Calle 13
! #channel canal-plus Canal +
! #channel canal-plus-30 Canal + ...30
! #channel canal-plus-2 Canal + 2
! #channel canal-plus-3d Canal + 3D
! #channel canal-plus-accion Canal + Acción
! #channel canal-plus-accion-30 Canal + Acción 30
! #channel canal-plus-accion-hd Canal + Acción HD
! #channel canal-plus-comedia Canal + Comedia
! #channel canal-plus-comedia-hd Canal + Comedia HD
! #channel canal-plus-dcine Canal + DCine
! #channel canal-plus-dcine-hd Canal + DCine HD
! #channel canal-plus-deportes Canal + Deportes
! #channel canal-plus-deportes-hd Canal + Deportes HD
! #channel canal-plus-eventos Canal + Eventos
! #channel canal-plus-futbol Canal + Fútbol
! #channel canal-plus-futbol-hd Canal + Fútbol HD
! #channel canal-plus-hd Canal + HD
! #channel canal-plus-liga-hd Canal + Liga HD
! #channel canal-plus-xtra Canal + XTRA
! #channel canal-plus-xtra-hd Canal + Xtra HD
! #channel canal-33 Canal 33
! #channel canal-club Canal Club
! #channel canal-cocina Canal Cocina
! #channel canal-extremadura Canal Extremadura
! #channel canal-fox Canal Fox
! #channel canal-sur Canal Sur
! #channel canal-sur-2 Canal Sur 2
! #channel canal-de-las-estrellas Canal de las Estrellas
! #channel canalplus-deportes-2-hd Canal+ Deportes 2 HD
! #channel canalplus-deportes-mixto Canal+ Deportes Mixto
! #channel canalplus-liga-2 Canal+ Liga 2
! #channel canalplus-liga-2-a Canal+ Liga 2 A
! #channel canalplus-liga-2-b Canal+ Liga 2 B
! #channel canalplus-liga-2-c Canal+ Liga 2 C
! #channel canalplus-liga-2-d Canal+ Liga 2 D
! #channel canalplus-liga-2-e Canal+ Liga 2 E
! #channel cartoon-network Cartoon Network
! #channel cartoonito Cartoonito
! #channel caza-y-pesca Caza y Pesca
! #channel clan Clan
! #channel cosmopolitan Cosmopolitan
! #channel cosmopolitan-hd Cosmopolitan HD
! channel cuatro Cuatro
! #channel cubavision Cubavisión
! #channel d-cine-espanol D Cine Español
! #channel decasa Decasa
! #channel discovery-channel Discovery Channel
! #channel disney-channel Disney Channel
! #channel disney-cinemagic Disney Cinemagic
! #channel disney-cinemagic-hd Disney Cinemagic HD
! #channel disney-xd Disney XD
! #channel divinity Divinity
! #channel ehs EHS
! #channel etb-1 ETB 1
! #channel etb-2 ETB 2
! #channel etb-3 ETB 3
! #channel eurosport Eurosport
! #channel eurosport-hd Eurosport HD
! #channel fdf FDF
! #channel france-24 FRANCE 24
! #channel fox-crime Fox Crime
! #channel fox-hd Fox HD
! #channel fox-news Fox News
! #channel golfplus Golf+
! #channel historia Historia
! #channel hogar-util Hogar Útil
! #channel hollywood Hollywood
! #channel hollywood-hd Hollywood HD
! #channel ib3 IB3
! #channel infometeo InfoMeteo
! #channel intereconomia Intereconomía
! #channel la-10 La 10
! channel la-2 La 2
! #channel la-sexta La Sexta
! #channel la-sexta-2 La Sexta 2
! #channel la-sexta-3 La Sexta 3
! #channel la-siete La Siete
! #channel mtv MTV
! #channel mtv-2 MTV 2
! #channel mtv-esp MTV ESP
! #channel mtv-hd MTV HD
! #channel marca-tv Marca TV
! #channel mezzo Mezzo
! #channel mezzo-live-hd Mezzo Live HD
! #channel multi-x--1- Multi-X (1)
! #channel multi-x--2- Multi-X (2)
! #channel multi-x--3- Multi-X (3)
! #channel multideporte-1 Multideporte 1
! #channel multideporte-2 Multideporte 2
! #channel multideporte-3 Multideporte 3
! #channel multideporte-4 Multideporte 4
! #channel multideporte-5 Multideporte 5
! #channel multideporte-6 Multideporte 6
! #channel multideporte-7 Multideporte 7
! #channel nhk-world NHK World
! #channel nick-jr NICK JR
! #channel nat-geo-wild-hd Nat Geo Wild HD
! #channel nat-geographic-hd Nat Geographic HD
! #channel nat-geographic-wild Nat Geographic Wild
! #channel national-geographic National Geographic
! #channel neox Neox
! #channel nickelodeon Nickelodeon
! #channel nitro Nitro
! #channel nou Nou
! #channel nou-2 Nou 2
! #channel nova Nova
! #channel odisea Odisea
! #channel panda Panda
! #channel paramount-comedy Paramount Comedy
! #channel playboy-tv Playboy TV
! #channel playhouse-disney Playhouse Disney
! #channel rt-espanol RT Español
! #channel radios Radios.
! #channel real-madrid-tv Real Madrid TV
! #channel russia-today Russia Today
! #channel syfy-hd SYFY HD
! #channel sci-fi Sci-Fi
! #channel sony-entertaiment-tv Sony Entertaiment TV
! #channel sportmania Sportmanía
! #channel super-3 Super 3
! #channel tcm TCM
! #channel tcm-clasico TCM Clásico
! #channel tnt TNT
! #channel tpa TPA
! #channel tv-canaria TV Canaria
! #channel tv-record TV RECORD
! #channel tv3 TV3
! #channel tv5 TV5
! channel tve-1 TVE 1
! #channel tvg TVG
! #channel tvod TVOD
! #channel taquilla-hd Taquilla HD
! channel tele-5 Tele 5
! #channel teledeporte Teledeporte
! #channel telemadrid Telemadrid
! #channel telemadrid-sat-laotra Telemadrid SAT/LaOtra
! #channel telesur Telesur
! #channel televisio-catalunya-int Televisió Catalunya Int.
! #channel vh1 VH1
! #channel viajar Viajar
! #channel viajar-hd Viajar HD
! #channel laotra laOtra
--- 1,179 ----
! configversion 3
! usecache no
! cachedir /root/.xmltv/cache
getdescriptions yes
! #channel 13tv-943 13TV
! #channel 40-latino-759 40 Latino
! #channel 40tv-866 40TV
! #channel 7rm-935 7RM
! #channel axn-867 AXN
! #channel axn-hd-868 AXN HD
! #channel al-jazeera-english-772 Al Jazeera English
! #channel andalucia-tv-783 Andalucía TV
! #channel animax-779 Animax
! channel antena-3-839 Antena 3
! #channel aragon-tv-919 Aragón TV
! #channel bbc-world-776 BBC World
! #channel baby-tv-790 Baby TV
! #channel barcatv-764 BarçaTV
! #channel biography-771 Biography
! #channel bloomberg-774 Bloomberg
! #channel boing-869 Boing
! #channel cmt-1-929 CMT 1
! #channel cmt-2-931 CMT 2
! #channel cnbc-778 CNBC
! #channel cnn-int-773 CNN Int
! #channel calle-13-870 Calle 13
! #channel canal-865 Canal +
! #channel canal-2-871 Canal + 2
! #channel canal-3d-803 Canal + 3D
! #channel canal-accion-710 Canal + Acción
! #channel canal-accion-30-872 Canal + Acción 30
! #channel canal-accion-hd-908 Canal + Acción HD
! #channel canal-comedia-711 Canal + Comedia
! #channel canal-comedia-hd-909 Canal + Comedia HD
! #channel canal-dcine-712 Canal + DCine
! #channel canal-dcine-hd-873 Canal + DCine HD
! #channel canal-deportes-715 Canal + Deportes
! #channel canal-deportes-hd-791 Canal + Deportes HD
! #channel canal-eventos-716 Canal + Eventos
! #channel canal-futbol-713 Canal + Fútbol
! #channel canal-futbol-hd-714 Canal + Fútbol HD
! #channel canal-hd-874 Canal + HD
! #channel canal-liga-hd-875 Canal + Liga HD
! #channel canal-xtra-877 Canal + XTRA
! #channel canal-xtra-hd-718 Canal + Xtra HD
! #channel canal-33-951 Canal 33
! #channel canal-club-787 Canal Club
! #channel canal-cocina-767 Canal Cocina
! #channel canal-extremadura-939 Canal Extremadura
! #channel canal-fox-731 Canal Fox
! #channel canal-sur-863 Canal Sur
! #channel canal-sur-2-864 Canal Sur 2
! #channel canal-de-las-estrellas-786 Canal de las Estrellas
! #channel canal-1-30-876 Canal+ 1...30
! #channel canal-deportes-2-hd-755 Canal+ Deportes 2 HD
! #channel canal-deportes-mixto-910 Canal+ Deportes Mixto
! #channel canal-liga-2-792 Canal+ Liga 2
! #channel canal-liga-2-a-735 Canal+ Liga 2 A
! #channel canal-liga-2-b-736 Canal+ Liga 2 B
! #channel canal-liga-2-c-737 Canal+ Liga 2 C
! #channel canal-liga-2-d-738 Canal+ Liga 2 D
! #channel canal-liga-2-e-739 Canal+ Liga 2 E
! #channel cartoon-network-878 Cartoon Network
! #channel cartoonito-793 Cartoonito
! #channel caza-y-pesca-769 Caza y Pesca
! #channel clan-756 Clan
! #channel cosmopolitan-879 Cosmopolitan
! #channel cosmopolitan-hd-717 Cosmopolitan HD
! #channel crimen-investigacion-989 Crimen & Investigación
! channel cuatro-895 Cuatro
! #channel cubavision-805 Cubavisión
! #channel d-cine-espanol-730 D Cine Español
! #channel decasa-770 Decasa
! #channel discovery-channel-766 Discovery Channel
! #channel discovery-max-995 Discovery MAX
! #channel disney-channel-880 Disney Channel
! #channel disney-channel-hd-965 Disney Channel HD
! #channel disney-cinemagic-881 Disney Cinemagic
! #channel disney-cinemagic-hd-719 Disney Cinemagic HD
! #channel disney-junior-983 Disney Junior
! #channel disney-xd-794 Disney XD
! #channel divinity-925 Divinity
! #channel ehs-788 EHS
! #channel etb-1-855 ETB 1
! #channel etb-2-856 ETB 2
! #channel etb-3-927 ETB 3
! #channel etb-sat-975 ETB Sat
! #channel energy-963 Energy
! #channel eurosport-882 Eurosport
! #channel eurosport-hd-751 Eurosport HD
! #channel fdf-734 FDF
! #channel france-24-775 FRANCE 24
! #channel fox-crime-732 Fox Crime
! #channel fox-hd-883 Fox HD
! #channel fox-news-777 Fox News
! #channel galicia-tv-977 Galicia TV
! #channel golf-761 Golf+
! #channel historia-768 Historia
! #channel hogar-util-912 Hogar Útil
! #channel hollywood-728 Hollywood
! #channel hollywood-hd-795 Hollywood HD
! #channel ib3-937 IB3
! #channel infometeo-729 InfoMeteo
! #channel intereconomia-959 Intereconomía
! #channel la-10-923 La 10
! #channel la-2-823 La 2
! #channel la-sexta-782 La Sexta
! #channel la-sexta-2-887 La Sexta 2
! #channel la-sexta-3-888 La Sexta 3
! #channel la-siete-720 La Siete
! #channel mtv-2-802 MTV 2
! #channel mtv-esp-740 MTV ESP
! #channel mtv-hd-752 MTV HD
! #channel marca-tv-884 Marca TV
! #channel mezzo-760 Mezzo
! #channel mezzo-live-hd-721 Mezzo Live HD
! #channel multi-x-1-796 Multi-X (1)
! #channel multi-x-2-797 Multi-X (2)
! #channel multi-x-3-798 Multi-X (3)
! #channel multideporte-1-742 Multideporte 1
! #channel multideporte-2-743 Multideporte 2
! #channel multideporte-3-744 Multideporte 3
! #channel multideporte-4-745 Multideporte 4
! #channel multideporte-5-746 Multideporte 5
! #channel multideporte-6-747 Multideporte 6
! #channel multideporte-7-748 Multideporte 7
! #channel nhk-world-799 NHK World
! #channel nick-jr-741 NICK JR
! #channel nat-geo-wild-hd-913 Nat Geo Wild HD
! #channel nat-geographic-hd-753 Nat Geographic HD
! #channel nat-geographic-wild-904 Nat Geographic Wild
! #channel national-geographic-886 National Geographic
! #channel neox-722 Neox
! #channel nickelodeon-905 Nickelodeon
! #channel nitro-800 Nitro
! #channel nou-860 Nou
! #channel nou-2-861 Nou 2
! #channel nova-723 Nova
! #channel nueve-993 Nueve
! #channel odisea-765 Odisea
! #channel panda-754 Panda
! #channel paramount-channel-979 Paramount Channel
! #channel paramount-comedy-733 Paramount Comedy
! #channel playboy-tv-757 Playboy TV
! #channel playhouse-disney-758 Playhouse Disney
! #channel rt-espanol-801 RT Español
! #channel radios-789 Radios.
! #channel real-madrid-tv-763 Real Madrid TV
! #channel russia-today-749 Russia Today
! #channel syfy-750 SYFY
! #channel syfy-hd-906 SYFY HD
! #channel sony-entertaiment-tv-780 Sony Entertaiment TV
! #channel sportmania-889 Sportmanía
! #channel super-3-859 Super 3
! #channel tcm-890 TCM
! #channel tcm-autor-971 TCM Autor
! #channel tcm-clasico-726 TCM Clásico
! #channel tl-novelas-europa-973 TL Novelas Europa
! #channel tnt-727 TNT
! #channel tpa-941 TPA
! #channel tv-canaria-933 TV Canaria
! #channel tv-record-806 TV RECORD
! #channel tv3-858 TV3
! #channel tv3cat-785 TV3Cat
! #channel tv5-891 TV5
! channel tve-1-807 TVE 1
! #channel tvg-862 TVG
! #channel tvod-915 TVOD
! #channel taquilla-hd-804 Taquilla HD
! channel tele-5-847 Tele 5
! #channel teledeporte-762 Teledeporte
! #channel telemadrid-857 Telemadrid
! #channel telemadrid-sat-laotra-784 Telemadrid SAT/LaOtra
! #channel telesur-725 Telesur
! #channel vh1-892 VH1
! #channel viajar-893 Viajar
! #channel viajar-hd-724 Viajar HD
! #channel xplora-981 Xplora
! #channel laotra-953 laOtra
------------------------------------------------------------------------------
WatchGuard Dimension instantly turns raw network data into actionable
security intelligence. It gives you real-time visual feedback on key
security issues and trends. Skip the complicated setup - simply import
a virtual appliance and go from zero to informed in seconds.
http://pubads.g.doubleclick.net/gampad/clk?id=123612991&iu=/4140/ostg.clktrk
_______________________________________________
xmltv-commit mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/xmltv-commit