xmltv/grab/fi/fi/source telkku.pm,2.05,2.06
Stefan Becker <[email protected]> Sat, 20 Aug 2016 16:55:15 +0000
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
--===============6072044288655543137==
Update of /cvsroot/xmltv/xmltv/grab/fi/fi/source
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv7076/grab/fi/fi/source
Modified Files:
telkku.pm
Log Message:
- telkku: rewritten for update web page contents (bug #510)
- NOTE: identifiers have changed; you need to run configure again!
- test.conf: updated with new telkku.com identifiers
- programme: updated one season/episode extractor regex
Index: telkku.pm
===================================================================
RCS file: /cvsroot/xmltv/xmltv/grab/fi/fi/source/telkku.pm,v
retrieving revision 2.05
retrieving revision 2.06
diff -C2 -d -r2.05 -r2.06
*** telkku.pm 21 Jun 2014 16:36:15 -0000 2.05
--- telkku.pm 20 Aug 2016 16:55:13 -0000 2.06
***************
*** 13,16 ****
--- 13,18 ----
use strict;
use warnings;
+ use Date::Manip qw(UnixDate);
+ use JSON qw();
BEGIN {
***************
*** 25,73 ****
sub description { 'telkku.com' }
! # Grab channel list
! sub channels {
! # Fetch & parse HTML
! my $root = fetchTree("http://www.telkku.com/channel");
! if ($root) {
! my %channels;
#
! # Channel list can be found from the left sidebar
#
! # <div class="l-wrap l-grid--16" id="channelContainer">
! # ...
! # <ul>
! # <li><a href="http://www.telkku.com/channel/list/8/20101218">4 Sport</a></li>
! # <li><a href="http://www.telkku.com/channel/list/24/20101218">4 Sport Pro</a></li>
! # ...
! # <li><a href="http://www.telkku.com/channel/list/87/20101218">Viron ETV</a></li>
! # <li><a href="http://www.telkku.com/channel/list/10/20101218">YLE Teema</a></li>
! # </ul>
! # </div>
#
! if (my $container = $root->look_down("id" => "channelContainer")) {
! if (my @list = $container->find("li")) {
! debug(2, "Source telkku.com found " . scalar(@list) . " channels");
! foreach my $list_entry (@list) {
! if (my $link = $list_entry->find("a")) {
! my $href = $link->attr("href");
! my $name = $link->as_text();
! if (defined($href) && length($name) &&
! (my($channel_no) = ($href =~ m,channel/list/(\d+)/,))) {
! debug(3, "channel '$name' ($channel_no)");
! $channels{"${channel_no}.telkku.com"} = "fi $name";
! }
! }
}
}
}
-
- # Done with the HTML tree
- $root->delete();
-
- debug(2, "Source telkku.com parsed " . scalar(keys %channels) . " channels");
- return(\%channels);
}
--- 27,72 ----
sub description { 'telkku.com' }
! my %categories = (
! SPORTS => "urheilu",
! MOVIE => "elokuvat",
! );
! # Fetch raw HTML and extract & parse JSON
! sub _getJSON($$$) {
! my($date, $page, $keys) = @_;
+ # Fetch raw text
+ my $text = fetchRaw("http://www.telkku.com/tv-ohjelmat/$date/patch/koko-paiva");
+ if ($text) {
#
! # All data is encoded in JSON in a script node
#
! # <script>
! # window.__INITIAL_STATE__ = {...};
! # </script>
#
! my($match) = ($text =~ /window.__INITIAL_STATE__ = ({.+});/);
! if ($match) {
! my $decoded = JSON->new->decode($match);
!
! if (ref($decoded) eq "HASH") {
! my $data = $decoded;
!
! #debug(5, JSON->new->pretty->encode($decoded));
!
! # step through hashes using key sequence
! foreach my $key (@{$keys}) {
! debug(5, "Looking for JSON key $key");
! return unless exists $data->{$key};
! $data = $data->{$key};
}
+ debug(5, "Found JSON data");
+
+ #debug(5, JSON->new->pretty->encode($data));
+ #debug(5, "KEYS: ", join(", ", sort keys %{$data}));
+ return($data);
}
}
}
***************
*** 75,138 ****
}
! #
! # http://www.telkku.com/movie contains information about (all?) movies for
! # today and the next 7 days, i.e. offsets 0 to 7. We extract the URL to the
! # detailed programme information (http://www.telkku.com/program/show/......)
! # that can then be used to identify movies when processing programme entries.
! #
! {
! my %ids;
! sub _getMovieIDsForOffset($) {
! my($offset) = @_;
! # There is only data for the next 7 days
! return({}) if $offset > 7;
! # Reuse cached data
! return(\%ids) if %ids;
! # In order to reduce website traffic, we only try this once
! $ids{__DUMMY_ID_THAT_NEVER_MATCHES__}++;
! # Fetch & parse HTML (do not ignore HTML5 <section>)
! # This is entirely optional, so please don't abort on failure...
! my $root = fetchTree("http://www.telkku.com/movie", undef, 1, 1);
! if ($root) {
! my $test;
! #
! # Document structure for movie entries:
! #
! # <li class="l-embed theme-hr program">
! # ...
! # <section class="l-embed__bd program__content">
! # <a href="http://www.telkku.com/program/show/2014061910151">
! # ...
! # </a>
! # </section>
! # </li>
! #
! debug(2, "Source telkku.com provided movie data");
! if (my @list = $root->look_down("class" => qr/program__content/)) {
! debug(2, "Source telkku.com found " . scalar(@list) . " movies");
! foreach my $list_entry (@list) {
! if (my $link = $list_entry->find("a")) {
! my $href = $link->attr("href");
! if (defined($href) && length($href)) {
! debug(3, "movie ID: " . $href);
! $ids{$href}++;
}
}
}
}
-
- # Done with the HTML tree
- $root->delete();
}
! debug(2, "Source telkku.com parsed " . (scalar(keys %ids) - 1) . " movies");
! return(\%ids);
}
}
--- 74,145 ----
}
! # Grab channel list
! sub channels {
! # Fetch & extract JSON sub-part
! my $data = _getJSON("tanaan", "peruskanavat",
! ["channelGroups",
! "channelGroupsArray"]);
! #
! # Channels data has the following structure
! #
! # [
! # {
! # slug => "peruskanavat",
! # channels => [
! # {
! # id => "yle-tv1",
! # name => "Yle TV1",
! # ...
! # },
! # ...
! # ],
! # ...
! # },
! # ...
! # ]
! #
! if (ref($data) eq "ARRAY") {
! my %channels;
! my %duplicates;
! foreach my $item (@{$data}) {
! if ((ref($item) eq "HASH") &&
! (exists $item->{slug}) &&
! (exists $item->{channels}) &&
! (ref($item->{channels}) eq "ARRAY")) {
! my $group = $item->{slug};
! my $channels = $item->{channels};
! if (defined($group) && length($group) &&
! (ref($channels) eq "ARRAY")) {
! debug(2, "Source telkku.com found group '$group' with " . scalar(@{$channels}) . " channels");
! foreach my $channel (@{$channels}) {
! if (ref($channel) eq "HASH") {
! my $id = $channel->{id};
! my $name = $channel->{name};
! if (defined($id) && length($id) &&
! (not exists $duplicates{$id}) &&
! length($name)) {
! debug(3, "channel '$name' ($id)");
! $channels{"${id}.${group}.telkku.com"} = "fi $name";
!
! # Same ID can appear in multiple groups - avoid duplicates
! $duplicates{$id}++;
! }
}
}
}
}
}
! debug(2, "Source telkku.com parsed " . scalar(keys %channels) . " channels");
! return(\%channels);
}
+
+ return;
}
***************
*** 142,229 ****
# Get channel number from XMLTV id
! return unless my($channel) = ($id =~ /^(\d+)\.telkku\.com$/);
! # Fetch & parse HTML
! my $root = fetchTree("http://www.telkku.com/channel/list/$channel/$today");
! if ($root) {
! my $movie_ids = _getMovieIDsForOffset($offset);
! #
! # All program info is contained within a unsorted list with class "programList"
! #
! # <ul class="l-stack programList">
! # <li>
! # <a class="program" href="http://www.telkku.com/program/show/2012100920451">
! # <div class="theme-hr program__content">
! # <div class="program__desc">
! # <div class="h4 program__title">23:45 Uutisikkuna</div>
! # <div class="progrram__desc">...</div>
! # </div>
! # </div>
! # </a>
! # </li>
! # ...
! # </ul>
! #
! my $opaque = startProgrammeList($id, "fi");
! if (my $container = $root->look_down("class" => "l-stack programList")) {
! if (my @list = $container->find("li")) {
! foreach my $list_entry (@list) {
! my $link = $list_entry->look_down("class", "program");
! my $title = $list_entry->look_down("class", "h4 program__title");
! my $desc = $list_entry->look_down("class", "progrram__desc");
! if ($link && $title && $desc) {
! my $date;
! # Extract texts from HTML elements. Entities are already decoded.
! $date = $title->as_text();
! $desc = $desc->as_text();
! # Use "." to match character (it's not included in \s?)
! if (my($hour, $minute, $title) =
! $date =~ /^(\d{2}):(\d{2}).(.+)/) {
! my $href = $link->attr("href");
! my $category = (defined($href) && exists($movie_ids->{$href})) ?
! "elokuvat" : undef;
! debug(3, "List entry $channel ($hour:$minute) $title");
! debug(4, $desc);
! debug(4, $category) if defined $category;
! # Only record entry if title isn't empty
! if (length($title) > 0) {
! my $object = appendProgramme($opaque, $hour, $minute, $title);
! $object->category($category);
! $object->description($desc);
! }
! }
! }
}
}
}
! # Done with the HTML tree
! $root->delete();
! # Each page on telkku.com contains the program information
! # for one channel for one whole day.
! #
! # Example (compiled from several pages for illustration):
! #
! # /- start time (day)
! # | /- program title
! # | |
! # [23:45 Uutisikkuna (yesterday)]
! # 00:10 Uutisikkuna (today )
! # ...
! # 23:31 Uusi päivä (today )
! # 00:00 Kova laki (tomorrow )
! # [00:40 Piilosana (tomorrow )]
! # [01:00 Tellus-tietovisa (tomorrow )]
! #
! # The lines in [] don't appear on every page.
! #
! # Convert list to program objects
! return(convertProgrammeList($opaque, $yesterday, $today, $tomorrow));
}
--- 149,227 ----
# Get channel number from XMLTV id
! return unless my($channel, $group) = ($id =~ /^([\w-]+)\.(\w+)\.telkku\.com$/);
! # Fetch & extract JSON sub-part
! my $data = _getJSON($today, $group,
! ["offeringByChannelGroup",
! $group,
! "offering",
! "publicationsByChannel"]);
! #
! # Programme data has the following structure
! #
! # [
! # {
! # channel => {
! # id => "yle-tv1",
! # ...
! # },
! # publications => [
! # {
! # startTime => "2016-08-18T06:25:00.000+03:00",
! # endTime => "2016-08-18T06:55:00.000+03:00",
! # title => "Helil kyläs",
! # description => "Osa 9/10. Asiaohjelma, mikä ...",
! # programFormat => "MOVIE",
! # ...
! # },
! # ...
! # ]
! # },
! # ...
! # ]
! #
! if (ref($data) eq "ARRAY") {
! my @objects;
! foreach my $item (@{$data}) {
! if ((ref($item) eq "HASH") &&
! (ref($item->{channel}) eq "HASH") &&
! (ref($item->{publications}) eq "ARRAY") &&
! ($item->{channel}->{id} eq $channel)) {
! foreach my $programme (@{$item->{publications}}) {
! my($start, $end, $title, $desc) =
! @{$programme}{qw(startTime endTime title description)};
! #debug(5, JSON->new->pretty->encode($programme));
! if ($start && $end && $title && $desc) {
! $start = UnixDate($start, "%s");
! $end = UnixDate($end, "%s");
!
! # NOTE: entries with same start and end time are invalid
! if ($start && $end && ($start != $end)) {
! my $category = $categories{$programme->{programFormat}};
!
! debug(3, "List entry $channel.$group ($start -> $end) $title");
! debug(4, $desc);
! debug(4, $category) if defined $category;
!
! # Create program object
! my $object = fi::programme->new($id, "fi", $title, $start, $end);
! $object->category($category);
! $object->description($desc);
! push(@objects, $object);
! }
! }
}
}
}
! # Fix overlapping programmes
! fi::programme->fixOverlaps(\@objects);
! return(\@objects);
}
--===============6072044288655543137==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
------------------------------------------------------------------------------
--===============6072044288655543137==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
_______________________________________________
xmltv-commit mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/xmltv-commit
--===============6072044288655543137==--