xmltv/grab/fi/fi/source foxtv.pm, 2.04, 2.05 yle.pm, 2.11, 2.12
Stefan Becker via xmltv-commit <[email protected]> Wed, 12 Jul 2017 19:33:04 +0000
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
--===============1519798338137332678==
Content-Type: text/plain
Update of /cvsroot/xmltv/xmltv/grab/fi/fi/source
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv27680/grab/fi/fi/source
Modified Files:
foxtv.pm yle.pm
Log Message:
- foxtv: fix programme start time parsing
- common: work around meta-in-body bug in HTML::TreeBuilder
- yle: updated for site changes - NOTE: IDs for yle.fi have changed!
Index: foxtv.pm
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi/fi/source/foxtv.pm,v
retrieving revision 2.04
retrieving revision 2.05
diff -C2 -d -r2.04 -r2.05
*** foxtv.pm 15 May 2016 16:58:19 -0000 2.04
--- foxtv.pm 12 Jul 2017 19:33:02 -0000 2.05
***************
*** 78,82 ****
if ($start && $title) {
if (my($hour, $minute) =
! $start->as_text() =~ /^(\d{2}):(\d{2})$/) {
my $desc = $programme->find("p");
my $extra = $programme->find("h4");
--- 78,82 ----
if ($start && $title) {
if (my($hour, $minute) =
! $start->as_text() =~ /^(\d{2})[:.](\d{2})$/) {
my $desc = $programme->find("p");
my $extra = $programme->find("h4");
Index: yle.pm
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi/fi/source/yle.pm,v
retrieving revision 2.11
retrieving revision 2.12
diff -C2 -d -r2.11 -r2.12
*** yle.pm 15 May 2016 15:16:16 -0000 2.11
--- yle.pm 12 Jul 2017 19:33:02 -0000 2.12
***************
*** 26,31 ****
my %languages = (
! "fi" => "ohjelmaopas",
! "sv" => "programguide",
);
--- 26,31 ----
my %languages = (
! "fi" => [ "areena", "opas" ],
! "sv" => [ "arenan", "guide" ],
);
***************
*** 37,42 ****
foreach my $code (sort keys %languages) {
! # Fetch & parse HTML
! my $root = fetchTree("http://$languages{$code}.yle.fi/tv/opas");
if ($root) {
--- 37,43 ----
foreach my $code (sort keys %languages) {
! # Fetch & parse HTML (do not ignore HTML5 <time>)
! my $root = fetchTree("http://$languages{$code}[0].yle.fi/tv/$languages{$code}[1]",
! undef, undef, 1);
if ($root) {
***************
*** 44,64 ****
# Channel list can be found from this list:
#
! # <ul class="channel-lists ...">
! # <li><h1 id="yle-tv1">Yle TV1...</h1>...</li>
! # <li><h1 id="yle-tv2">Yle TV2...</h1>...</li>
! # ...
! # </ul>
#
! if (my $container = $root->look_down("class" => qr/^channel-lists\s+/)) {
! if (my @headers = $container->find("h1")) {
! debug(2, "Source ${code}.yle.fi found " . scalar(@headers) . " channels");
! foreach my $header (@headers) {
! my $id = $header->attr("id");
! my $name = $header->as_text();
! if (defined($id) && length($id) && length($name)) {
! debug(3, "channel '$name' ($id)");
! $channels{"${id}.${code}.yle.fi"} = "$code $name";
! }
}
}
--- 45,71 ----
# Channel list can be found from this list:
#
! # <ul class="guide-channels">
! # <li class="guide-channels__channel">
! # <h2 class="channel-header">
! # <a>...<div class="channel-header__logo " ... aria-label="Yle TV1"></div></a>
! # </h2>
! # ...
! # </li>
! # ...
! # </ul>
#
! if (my @divs = $root->look_down("_tag" => "div",
! "aria-label" => qr/^.+$/)) {
! debug(2, "Source ${code}.yle.fi found " . scalar(@divs) . " channels");
! foreach my $div (@divs) {
! my $name = $div->attr("aria-label");
! if (defined($name) && length($name)) {
! # replace space with hyphen
! my $id;
! ($id = $name) =~ s/ /-/g;
!
! debug(3, "channel '$name' ($id)");
! $channels{"${id}.${code}.yle.fi"} = "$code $name";
}
}
***************
*** 83,89 ****
# Get channel number from XMLTV id
return unless my($channel, $code) = ($id =~ /^([^.]+)\.([^.]+)\.yle\.fi$/);
# Fetch & parse HTML (do not ignore HTML5 <time>)
! my $root = fetchTree("http://$languages{$code}.yle.fi/tv/opas?t=" . $today->ymdd(),
undef, undef, 1);
if ($root) {
--- 90,97 ----
# Get channel number from XMLTV id
return unless my($channel, $code) = ($id =~ /^([^.]+)\.([^.]+)\.yle\.fi$/);
+ $channel =~ s/-/ /g;
# Fetch & parse HTML (do not ignore HTML5 <time>)
! my $root = fetchTree("http://$languages{$code}[0].yle.fi/tv/$languages{$code}[1]?t=" . $today->ymdd(),
undef, undef, 1);
if ($root) {
***************
*** 93,166 ****
# Each programme can be found in a separate <li> node
#
! # <ul class="channel-lists ...">
! # <li>
! # <h1 id="yle-tv1">Yle TV1...</h1>
! # <ul>
! # <li class="program-entry ...">
! # <div class="program-label">
! # <time class="dtstart" datetime="2014-06-15T01:30:00.000+03:00">01:30</time>
! # <time class="dtend" datetime="2014-06-15T04:30:00.000+03:00"></time>
! # <div class="program-title">
! # ...
! # <a class="link-grey" href="...">Suunnistuksen Jukolan viesti</a>
! # <span class="label movie">Elokuva</span>
! # ...
! # </div>
! # </div>
! # ...
! # <div class="program-desc">
! # <p>66. Jukolan viesti. Kolmas, neljäs ja viides osuus...
! # ...
! # </p>
! # </div>
! # </li>
! # ...
! # </ul>
# ...
# </li>
! # </ul>
#
! if (my $container = $root->look_down("class" => qr/^channel-lists\s+/)) {
! if (my $header = $container->look_down("_tag" => "h1",
! "id" => $channel)) {
! if (my $parent = $header->parent()) {
! if (my @programmes = $parent->look_down("class" => qr/^program-entry\s+/)) {
! foreach my $programme (@programmes) {
! my $start = $programme->look_down("class", "dtstart");
! my $end = $programme->look_down("class", "dtend");
! my $title = $programme->look_down("class", "program-title");
! my $desc = $programme->look_down("class", "program-desc");
! if ($start && $end && $title && $desc) {
! $start = UnixDate($start->attr("datetime"), "%s");
! $end = UnixDate($end->attr("datetime"), "%s");
! my $link = $title->find("a");
! my $category = $title->look_down("class" => "label movie") ? "elokuvat" : undef;
! # NOTE: entries with same start and end time are invalid
! if ($start && $end && $link && ($start != $end)) {
! $title = $link->as_text();
! $title =~ s/^\s+//;
! $title =~ s/\s+$//;
! if (length($title)) {
! $desc = $desc->find("p");
! $desc = $desc ? $desc->as_text() : "";
! $desc =~ s/^\s+//;
! $desc =~ s/\s+$//;
! debug(3, "List entry $channel ($start -> $end) $title");
! debug(4, $desc);
! debug(4, $category) if defined $category;
! # Create program object
! my $object = fi::programme->new($id, $code, $title, $start, $end);
! $object->category($category);
! $object->description($desc);
! push(@objects, $object);
! }
}
}
--- 101,164 ----
# Each programme can be found in a separate <li> node
#
! # <ul class="guide-channels">
! # <li class="guide-channels__channel">
! # <h2 class="channel-header">
! # <a>...<div class="channel-header__logo " ... aria-label="Yle TV1"></div></a>
! # </h2>
! # <ul class="schedule-list">
! # <li class="schedule-card ..." ... itemtype="http://schema.org/Movie">
! # ...
! # <time datetime="2017-07-11T06:25:00+03:00" itemprop="startDate">06.25</time>
! # <time datetime="2017-07-11T06:55:00+03:00" itemprop="endDate"></time>
! # ...
! # <span itemprop="name">Mikä meitä lihottaa?</span>
! # ...
! # <span itemprop="description">1/8. Lihavuusepidemia. ...</span>
! # ...
! # </li>
# ...
+ # </ul>
# </li>
! # ...
! # </ul>
#
! if (my $div = $root->look_down("_tag" => "div",
! "aria-label" => qr/^${channel}$/)) {
! if (my $parent = $div->look_up("class" => "guide-channels__channel")) {
! if (my @programmes = $parent->look_down("class" => qr/^schedule-card\s+/)) {
! foreach my $programme (@programmes) {
! my $start = $programme->look_down("itemprop", "startDate");
! my $end = $programme->look_down("itemprop", "endDate");
! my $title = $programme->look_down("itemprop", "name");
! my $desc = $programme->look_down("itemprop", "description");
! if ($start && $end && $title && $desc) {
! $start = UnixDate($start->attr("datetime"), "%s");
! $end = UnixDate($end->attr("datetime"), "%s");
! my $category = $programme->attr("itemtype") =~ /Movie/ ? "elokuvat" : undef;
! # NOTE: entries with same start and end time are invalid
! if ($start && $end && ($start != $end)) {
! $title = $title->as_text();
! $title =~ s/^\s+//;
! $title =~ s/\s+$//;
! if (length($title)) {
! $desc = $desc->as_text();
! $desc =~ s/^\s+//;
! $desc =~ s/\s+$//;
! debug(3, "List entry $channel ($start -> $end) $title");
! debug(4, $desc);
! debug(4, $category) if defined $category;
! # Create program object
! my $object = fi::programme->new($id, $code, $title, $start, $end);
! $object->category($category);
! $object->description($desc);
! push(@objects, $object);
}
}
--===============1519798338137332678==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
--===============1519798338137332678==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
_______________________________________________
xmltv-commit mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/xmltv-commit
--===============1519798338137332678==--