xmltv/lib IMDB.pm,1.60,1.61
Geoff <[email protected]>
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/lib
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv15427
Modified Files:
IMDB.pm
Log Message:
add plot keywords (Patch #14)
Index: IMDB.pm
===================================================================
RCS file: /cvsroot/xmltv/xmltv/lib/IMDB.pm,v
retrieving revision 1.60
retrieving revision 1.61
diff -C2 -d -r1.60 -r1.61
*** IMDB.pm 2 May 2014 06:12:54 -0000 1.60
--- IMDB.pm 2 May 2014 06:17:13 -0000 1.61
***************
*** 42,47 ****
# = moviedb.dat directors and actors list no longer include repeated names (which mostly
# occured in episodic tv programs (reported by Alexy Khrabrov)
#
! our $VERSION = '0.8';
sub new
--- 42,48 ----
# = moviedb.dat directors and actors list no longer include repeated names (which mostly
# occured in episodic tv programs (reported by Alexy Khrabrov)
+ # .9 = added keywords data
#
! our $VERSION = '0.9';
sub new
***************
*** 57,60 ****
--- 58,62 ----
$self->{replaceTitles}=0 if ( !defined($self->{replaceTitles}));
$self->{replaceCategories}=0 if ( !defined($self->{replaceCategories}));
+ $self->{replaceKeywords}=0 if ( !defined($self->{replaceKeywords}));
$self->{replaceURLs}=0 if ( !defined($self->{replaceURLs}));
$self->{replaceDirectors}=1 if ( !defined($self->{replaceDirectors}));
***************
*** 68,71 ****
--- 70,74 ----
$self->{updateCategories}=1 if ( !defined($self->{updateCategories}));
$self->{updateCategoriesWithGenres}=1 if ( !defined($self->{updateCategoriesWithGenres}));
+ $self->{updateKeywords}=0 if ( !defined($self->{updateKeywords})); # default is to NOT add keywords
$self->{updateURLs}=1 if ( !defined($self->{updateURLs}));
$self->{updateDirectors}=1 if ( !defined($self->{updateDirectors}));
***************
*** 163,167 ****
if ( $1 == 0 && $2 == 3 ) {
# 0.2 -> 0.3 upgrade requires prepStage 5 to be re-run
! return("imdbDir index db requires major reindexing, rerun --prepStage 2 and new prepStages 5,6 and 7\n");
}
if ( $1 == 0 && $2 == 4 ) {
--- 166,170 ----
if ( $1 == 0 && $2 == 3 ) {
# 0.2 -> 0.3 upgrade requires prepStage 5 to be re-run
! return("imdbDir index db requires major reindexing, rerun --prepStage 2 and new prepStages 5,6,7 and 8\n");
}
if ( $1 == 0 && $2 == 4 ) {
***************
*** 189,193 ****
--- 192,198 ----
"database index isn't readable");
+ my $verbose = $self->{verbose}; $self->{verbose} = 0;
my $res=$self->getMovieMatches($title, $year);
+ $self->{verbose} = $verbose; undef $verbose;
if ( !defined($res) ) {
$self->closeMovieIndex();
***************
*** 483,487 ****
chop();
if ( s/^$id:// ) {
! my ($directors, $actors, $genres, $ratingDist, $ratingVotes, $ratingRank)=split('\t', $_);
if ( $directors ne "<>" ) {
for my $name (split('\|', $directors)) {
--- 488,492 ----
chop();
if ( s/^$id:// ) {
! my ($directors, $actors, $genres, $ratingDist, $ratingVotes, $ratingRank, $keywords)=split('\t', $_);
if ( $directors ne "<>" ) {
for my $name (split('\|', $directors)) {
***************
*** 523,526 ****
--- 528,534 ----
$results->{ratingVotes}=$ratingVotes if ( $ratingVotes ne "<>" );
$results->{ratingRank}=$ratingRank if ( $ratingRank ne "<>" );
+ if ( $keywords ne "<>" ) {
+ push(@{$results->{keywords}}, split(',', $keywords));
+ }
}
else {
***************
*** 1138,1142 ****
}
if ( defined($prog->{category}) ) {
- my $found=0;
for my $value (@{$prog->{category}}) {
my $found=0;
--- 1146,1149 ----
***************
*** 1155,1158 ****
--- 1162,1195 ----
}
+ if ( $self->{updateKeywords} ) {
+ my @keywords;
+ if ( defined($details->{keywords}) ) {
+ for (@{$details->{keywords}}) {
+ push(@keywords, [$_, 'en']);
+ }
+ }
+
+ if ( $self->{replaceKeywords} ) {
+ if ( defined($prog->{keywords}) ) {
+ $self->debug("replacing (all) 'keywords'");
+ delete($prog->{keywords});
+ }
+ }
+ if ( defined($prog->{keyword}) ) {
+ for my $value (@{$prog->{keyword}}) {
+ my $found=0;
+ for my $k (@keywords) {
+ if ( lc($k->[0]) eq lc($value->[0]) ) {
+ $found=1;
+ }
+ }
+ if ( !$found ) {
+ push(@keywords, $value);
+ }
+ }
+ }
+ $prog->{keyword}=\@keywords;
+ }
+
return($prog);
}
***************
*** 1302,1308 ****
mkdir $listsDir, 0777 or die "cannot mkdir $listsDir: $!";
}
CHECK_FILES:
my %missingListFiles; # maps 'movies' to filename ...movies.gz
! for ('movies', 'actors', 'actresses', 'directors', 'genres', 'ratings') {
my $filename="$listsDir/$_.list";
my $filenameGz="$filename.gz";
--- 1339,1349 ----
mkdir $listsDir, 0777 or die "cannot mkdir $listsDir: $!";
}
+
+ $self->{optionalStages} = { 'keywords' => 7 }; # list of optional stages - no need to download files for these
+
CHECK_FILES:
my %missingListFiles; # maps 'movies' to filename ...movies.gz
! for ('movies', 'actors', 'actresses', 'directors', 'genres', 'ratings', 'keywords') {
! my $file=$_;
my $filename="$listsDir/$_.list";
my $filenameGz="$filename.gz";
***************
*** 1326,1331 ****
--- 1367,1376 ----
# Just report one of the filenames, keep the message simple.
warn "$filenameGz does not exist\n";
+ if ( $self->{optionalStages}{$file} ) {
+ warn "$file will not be added to database\n";
+ } else {
$missingListFiles{$_}=$filenameGz;
}
+ }
elsif ( not $filenameExists and $filenameGzExists ) {
$self->{imdbListFiles}->{$_}=$filenameGz;
***************
*** 1412,1415 ****
--- 1457,1462 ----
$self->{moviedbOffline}="$self->{imdbDir}/moviedb.offline";
+ $self->{stageLast} = 8; # set the final stage in the build - i.e. the one which builds the final database
+
bless($self, $type);
return($self);
***************
*** 1860,1863 ****
--- 1907,2002 ----
}
+ sub readPlotKeywords($$$$)
+ {
+ my ($self, $countEstimate, $file)=@_;
+ my $startTime=time();
+ my $lineCount=0;
+
+ my $fh = openMaybeGunzip($file) || return(-2);
+ while(<$fh>) {
+ $lineCount++;
+
+ if ( m/THE KEYWORDS LIST/ ) {
+ if ( !($_=<$fh>) || !m/^===========/o ) {
+ $self->error("missing ======= after \"THE KEYWORDS LIST\" at line $lineCount");
+ closeMaybeGunzip($file, $fh);
+ return(-1);
+ }
+ if ( !($_=<$fh>) || !m/^\s*$/o ) {
+ $self->error("missing empty line after ======= at line $lineCount");
+ closeMaybeGunzip($file, $fh);
+ return(-1);
+ }
+ if ( !($_=<$fh>) || !m/^.*\s+\S+\s*$/o ) {
+ $self->error("missing title/keyword pairs after ======= at line $lineCount");
+ closeMaybeGunzip($file, $fh);
+ return(-1);
+ }
+ last;
+ }
+ elsif ( $lineCount > 70000 ) {
+ $self->error("$file: stopping at line $lineCount, didn't see \"THE KEYWORDS LIST\" line");
+ closeMaybeGunzip($file, $fh);
+ return(-1);
+ }
+ }
+
+ my $progress=Term::ProgressBar->new({name => "parsing keywords",
+ count => $countEstimate,
+ ETA => 'linear'})
+ if Have_bar;
+
+ $progress->minor(0) if Have_bar;
+ $progress->max_update_rate(1) if Have_bar;
+ my $next_update=0;
+
+ my $count=0;
+ while(<$fh>) {
+ $lineCount++;
+ my $line=$_;
+ chomp($line);
+ next if ($line =~ m/^\s*$/);
+ my ($title, $keyword) = ($line =~ m/^(.*)\s+(\S+)\s*$/);
+ if ( defined($title) and defined($keyword) ) {
+ # there are some strange titles, fix them:
+ $title =~ s/.*\s+{(.*)}/$1/;
+
+ # ignore anything which is an episode and not a main title (e.g. "Doctor Who (#10.22)" "(1986-09-18)" )
+ if ( ( $title !~ m/\(#\d{1,3}\.?\d{0,5}\)/ )
+ && ( $title !~ m/^\(\d{4}-\d{2}-\d{2}\)$/ ) )
+ {
+ if ( defined($self->{movies}{$title}) ) {
+ $self->{movies}{$title}.=",".$keyword;
+ } else {
+ $self->{movies}{$title}=$keyword;
+ # returned count is number of unique titles found
+ $count++;
+ }
+ }
+
+ if (Have_bar) {
+ # re-adjust target so progress bar doesn't seem too wonky
+ if ( $count > $countEstimate ) {
+ $countEstimate = $progress->target($count+1000);
+ $next_update=$progress->update($count);
+ }
+ elsif ( $count > $next_update ) {
+ $next_update=$progress->update($count);
+ }
+ }
+ } else {
+ $self->error("$file:$lineCount: unrecognized format \"$line\"");
+ $next_update=$progress->update($count) if Have_bar;
+ }
+ }
+ $progress->update($countEstimate) if Have_bar;
+
+ $self->status(sprintf("parsing \"keywords\" found $count titles and ".
+ "$lineCount lines in %d seconds",time()-$startTime));
+
+ closeMaybeGunzip($file, $fh);
+ return($count);
+ }
+
sub stageComplete($)
{
***************
*** 2287,2293 ****
}
elsif ( $stage == 7 ) {
my $tab=sprintf("\t");
! $self->status("indexing all previous stage's data for stage 7..");
$self->status("parsing stage 1 data (movie list)..");
--- 2426,2483 ----
}
elsif ( $stage == 7 ) {
+ $self->status("parsing Plot Keywords list for stage $stage..");
+ my $countEstimate=410000;
+ my $num=$self->readPlotKeywords($countEstimate, "$self->{imdbListFiles}->{keywords}");
+ if ( $num < 0 ) {
+ if ( $num == -2 ) {
+ $self->error("you need to download $self->{imdbListFiles}->{keywords} from ftp.imdb.com");
+ }
+ return(1);
+ }
+ elsif ( abs($num - $countEstimate) > $countEstimate*.05 ) {
+ $self->status("ARG estimate of $countEstimate for keywords needs updating, found $num");
+ }
+ $self->dbinfoAdd("keywords_list_file", "$self->{imdbListFiles}->{keywords}");
+ $self->dbinfoAdd("keywords_list_file_size", -s "$self->{imdbListFiles}->{keywords}");
+ $self->dbinfoAdd("db_stat_keywords_count", "$num");
+
+ $self->status("writing stage$stage data ..");
+ {
+ my $countEstimate=$self->dbinfoGet("db_stat_keywords_count", 0);
+ my $progress=Term::ProgressBar->new({name => "writing keywords",
+ count => $countEstimate,
+ ETA => 'linear'})
+ if Have_bar;
+ $progress->minor(0) if Have_bar;
+ $progress->max_update_rate(1) if Have_bar;
+ my $next_update=0;
+
+ open(OUT, "> $self->{imdbDir}/stage$stage.data") || die "$self->{imdbDir}/stage$stage.data:$!";
+
+ my $count=0;
+ for my $movie (keys %{$self->{movies}}) {
+ print OUT "$movie\t$self->{movies}->{$movie}\n";
+
+ $count++;
+ if (Have_bar) {
+ # re-adjust target so progress bar doesn't seem too wonky
+ if ( $count > $countEstimate ) {
+ $countEstimate = $progress->target($count+100);
+ $next_update=$progress->update($count);
+ }
+ elsif ( $count > $next_update ) {
+ $next_update=$progress->update($count);
+ }
+ }
+ }
+ $progress->update($countEstimate) if Have_bar;
+ close(OUT);
+ delete($self->{movies});
+ }
+ }
+ elsif ( $stage == $self->{stageLast} ) {
my $tab=sprintf("\t");
! $self->status("indexing all previous stage's data for stage ".$self->{stageLast}."..");
$self->status("parsing stage 1 data (movie list)..");
***************
*** 2571,2574 ****
--- 2761,2820 ----
}
+ $self->status("merging in stage 7 data (keywords)..");
+ if ( 1 ) {
+ my $countEstimate=$self->dbinfoGet("db_stat_keywords_count", 0);
+ my $progress=Term::ProgressBar->new({name => "merging keywords",
+ count => $countEstimate,
+ ETA => 'linear'})
+ if Have_bar;
+ $progress->minor(0) if Have_bar;
+ $progress->max_update_rate(1) if Have_bar;
+ my $next_update=0;
+
+ open(IN, "< $self->{imdbDir}/stage7.data") || die "$self->{imdbDir}/stage7.data:$!";
+ while(<IN>) {
+ chop();
+ s/^([^\t]+)\t+//o;
+ my $dbkey=$1;
+ my $keywords=$_;
+ if ( !defined($movies{$dbkey}) ) {
+ $self->error("keywords list references unidentified title '$1'");
+ next;
+ }
+ $movies{$dbkey}.=$tab.$keywords;
+
+ if (Have_bar) {
+ # re-adjust target so progress bar doesn't seem too wonky
+ if ( $. > $countEstimate ) {
+ $countEstimate = $progress->target($.+100);
+ $next_update=$progress->update($.);
+ }
+ elsif ( $. > $next_update ) {
+ $next_update=$progress->update($.);
+ }
+ }
+ }
+ $progress->update($countEstimate) if Have_bar;
+ close(IN);
+ }
+ if ( 1 ) {
+ # fill in default for movies we didn't have any keywords for
+ for my $key (keys %movies) {
+ my $val=$movies{$key};
+ #keyword is 6th entry
+ my $t = 0;
+ for my $i (0..4) {
+ $t=index($val, $tab, $t);
+ if ( $t == -1 ) {
+ die "Corrupt entry '$key' '$val'";
+ }
+ $t+=1;
+ }
+ if ( index($val, $tab, $t) == -1 ) {
+ $movies{$key}.=$tab."<>";
+ }
+ }
+ }
+
#unlink("$self->{imdbDir}/stage1.data");
#unlink("$self->{imdbDir}/stage2.data");
***************
*** 2821,2825 ****
}
else {
! $self->error("tv_imdb: invalid stage $stage: only 1-5 are valid");
return(1);
}
--- 3067,3071 ----
}
else {
! $self->error("tv_imdb: invalid stage $stage: only 1-8 are valid");
return(1);
}
***************
*** 2837,2843 ****
my ($self, $stage)=@_;
! for (my $st=1 ; $st < $stage ; $st++ ) {
if ( !$self->stageComplete($st) ) {
! $self->error("prep stages must be run in sequence..");
$self->error("prepStage $st either has never been run or failed");
$self->error("rerun tv_imdb with --prepStage=$st");
--- 3083,3092 ----
my ($self, $stage)=@_;
! if ( $stage == $self->{stageLast} ) {
! # check all the pre-requisite stages have been run
! for (my $st=1 ; $st < $self->{stageLast}; $st++ ) {
if ( !$self->stageComplete($st) ) {
! if ( ! grep { $_ == $st } values %{$self->{optionalStages}} ) {
! #$self->error("prep stages must be run in sequence..");
$self->error("prepStage $st either has never been run or failed");
$self->error("rerun tv_imdb with --prepStage=$st");
***************
*** 2845,2848 ****
--- 3094,3099 ----
}
}
+ }
+ }
if ( -f "$self->{moviedbInfo}" && $stage != 1 ) {
***************
*** 2864,2868 ****
else {
$self->status("prep stage $stage succeeded with $self->{errorCountInLog} errors in $self->{imdbDir}/stage$stage.log");
! if ( $stage == 7 && $self->{errorCountInLog} > 30 && $self->{errorCountInLog} < 80 ) {
$self->status("this stage commonly produces around 60 (or so) warnings because of imdb");
$self->status("list file inconsistancies, they can usually be safely ignored");
--- 3115,3119 ----
else {
$self->status("prep stage $stage succeeded with $self->{errorCountInLog} errors in $self->{imdbDir}/stage$stage.log");
! if ( $stage == $self->{stageLast} && $self->{errorCountInLog} > 30 && $self->{errorCountInLog} < 80 ) {
$self->status("this stage commonly produces around 60 (or so) warnings because of imdb");
$self->status("list file inconsistancies, they can usually be safely ignored");
------------------------------------------------------------------------------
"Accelerate Dev Cycles with Automated Cross-Browser Testing - For FREE
Instantly run your Selenium tests across 300+ browser/OS combos. Get
unparalleled scalability from the best Selenium testing platform available.
Simple to use. Nothing to install. Get started now for free."
http://p.sf.net/sfu/SauceLabs