xmltv/lib IMDB.pm,1.62,1.63
Geoff <[email protected]>
| Newsgroups | gmane.comp.tv.xmltv.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/xmltv/xmltv/lib
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv26478
Modified Files:
IMDB.pm
Log Message:
(minor) remove insistence that all files are downloaded even if that stage isn't being run
(minor) ensure output database is in iso-8859-1 (in case user has their default set to utf-8) (c.f. Bug #150)
(minor) minor code tidying
Index: IMDB.pm
===================================================================
RCS file: /cvsroot/xmltv/xmltv/lib/IMDB.pm,v
retrieving revision 1.62
retrieving revision 1.63
diff -C2 -d -r1.62 -r1.63
*** IMDB.pm 2 May 2014 06:21:07 -0000 1.62
--- IMDB.pm 12 May 2014 09:25:41 -0000 1.63
***************
*** 34,37 ****
--- 34,39 ----
package XMLTV::IMDB;
+ use open ':encoding(iso-8859-1)'; # try to enforce file encoding (does this work in Perl <5.8.1? )
+
#
# HISTORY
***************
*** 45,49 ****
# .10 = added plot data
#
! our $VERSION = '0.10';
sub new
--- 47,51 ----
# .10 = added plot data
#
! our $VERSION = '0.10'; # version number of database
sub new
***************
*** 157,161 ****
}
if ( $info->{db_version}=~m/^(\d+)\.(\d+)$/o ) {
! if ( $1 != $major || $minor < $2 ) {
return("imdbDir index db requires updating, rerun --prepStage all\n");
}
--- 159,163 ----
}
if ( $info->{db_version}=~m/^(\d+)\.(\d+)$/o ) {
! if ( $1 != $major || $2 < $minor ) {
return("imdbDir index db requires updating, rerun --prepStage all\n");
}
***************
*** 1322,1325 ****
--- 1324,1329 ----
use LWP;
+ use open ':encoding(iso-8859-1)'; # try to enforce file encoding (does this work in Perl <5.8.1? )
+
# Use Term::ProgressBar if installed.
use constant Have_bar => eval {
***************
*** 1355,1358 ****
--- 1359,1387 ----
die "invalid usage - no $_" if ( !defined($self->{$_}));
}
+
+ $self->{stageLast} = 9; # set the final stage in the build - i.e. the one which builds the final database
+ $self->{stages} = { 1=>'movies', 2=>'directors', 3=>'actors', 4=>'actresses', 5=>'genres', 6=>'ratings', 7=>'keywords', 8=>'plot' };
+ $self->{optionalStages} = { 'keywords' => 7, 'plot' => 8 }; # list of optional stages - no need to download files for these
+
+ $self->{moviedbIndex}="$self->{imdbDir}/moviedb.idx";
+ $self->{moviedbData}="$self->{imdbDir}/moviedb.dat";
+ $self->{moviedbInfo}="$self->{imdbDir}/moviedb.info";
+ $self->{moviedbOffline}="$self->{imdbDir}/moviedb.offline";
+
+ bless($self, $type);
+
+ if ( $self->{stageToRun} ne $self->{stageLast} ) {
+ # unless this is the last stage, check we have the necessary files
+ return(undef) if ( $self->checkFiles() != 0 );
+ }
+
+ return($self);
+ }
+
+
+ sub checkFiles () {
+
+ my ($self)=@_;
+
if ( ! -d "$self->{imdbDir}" ) {
if ( $self->{downloadMissingFiles} ) {
***************
*** 1370,1380 ****
}
- $self->{optionalStages} = { 'keywords' => 7, 'plot' => 8 }; # list of optional stages - no need to download files for these
-
CHECK_FILES:
my %missingListFiles; # maps 'movies' to filename ...movies.gz
! for ('movies', 'actors', 'actresses', 'directors', 'genres', 'ratings', 'keywords', 'plot') {
! my $file=$_;
! my $filename="$listsDir/$_.list";
my $filenameGz="$filename.gz";
my $filenameExists = -f $filename;
--- 1399,1411 ----
}
CHECK_FILES:
my %missingListFiles; # maps 'movies' to filename ...movies.gz
!
! FILES_CHECK:
! while ( my( $key, $value ) = each %{ $self->{stages} } ) {
! # don't check *all* files - only the ones we are crunching
! next FILES_CHECK if ( lc($self->{stageToRun}) ne 'all' && $key != int($self->{stageToRun}) );
! my $file=$value;
! my $filename="$listsDir/$file.list";
my $filenameGz="$filename.gz";
my $filenameExists = -f $filename;
***************
*** 1400,1411 ****
warn "$file will not be added to database\n";
} else {
! $missingListFiles{$_}=$filenameGz;
}
}
elsif ( not $filenameExists and $filenameGzExists ) {
! $self->{imdbListFiles}->{$_}=$filenameGz;
}
elsif ( $filenameExists and not $filenameGzExists ) {
! $self->{imdbListFiles}->{$_}=$filename;
}
elsif ( $filenameExists and $filenameGzExists ) {
--- 1431,1442 ----
warn "$file will not be added to database\n";
} else {
! $missingListFiles{$file}=$filenameGz;
}
}
elsif ( not $filenameExists and $filenameGzExists ) {
! $self->{imdbListFiles}->{$file}=$filenameGz;
}
elsif ( $filenameExists and not $filenameGzExists ) {
! $self->{imdbListFiles}->{$file}=$filename;
}
elsif ( $filenameExists and $filenameGzExists ) {
***************
*** 1479,1494 ****
print STDERR " see http://www.imdb.com/interfaces for details\n";
print STDERR " or try the --download option\n";
! return(undef);
}
! $self->{moviedbIndex}="$self->{imdbDir}/moviedb.idx";
! $self->{moviedbData}="$self->{imdbDir}/moviedb.dat";
! $self->{moviedbInfo}="$self->{imdbDir}/moviedb.info";
! $self->{moviedbOffline}="$self->{imdbDir}/moviedb.offline";
!
! $self->{stageLast} = 9; # set the final stage in the build - i.e. the one which builds the final database
!
! bless($self, $type);
! return($self);
}
--- 1510,1518 ----
print STDERR " see http://www.imdb.com/interfaces for details\n";
print STDERR " or try the --download option\n";
! #return(undef);
! return 1;
}
! return 0;
}
***************
*** 1532,1535 ****
--- 1556,1566 ----
}
+ sub withThousands ($)
+ {
+ my ($val) = @_;
+ $val =~ s/(\d{1,3}?)(?=(\d{3})+$)/$1,/g;
+ return $val;
+ }
+
use XMLTV::Gunzip;
use IO::File;
***************
*** 1669,1674 ****
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing $whichMoviesOrGenres found $count titles and ".
! "$lineCount lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
--- 1700,1705 ----
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing $whichMoviesOrGenres found ".withThousands($count)." titles in ".
! withThousands($lineCount)." lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
***************
*** 1807,1811 ****
$line=~s/\s*\{\{SUSPENDED\}\}//o;
! # ignore {Twelve Angry Men (1954)}
$line=~s/\s*\{[^\}]+\}//o;
--- 1838,1843 ----
$line=~s/\s*\{\{SUSPENDED\}\}//o;
! # [honir] this is wrong - this puts cast from all the episodes as though they are in the entire series!
! # ##ignore {Twelve Angry Men (1954)}
$line=~s/\s*\{[^\}]+\}//o;
***************
*** 1852,1857 ****
$progress->update($castCountEstimate) if Have_bar;
! $self->status(sprintf("parsing $whichCastOrDirector found $castNames names, ".
! "$count titles and $lineCount lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
--- 1884,1890 ----
$progress->update($castCountEstimate) if Have_bar;
! $self->status(sprintf("parsing $whichCastOrDirector found ".withThousands($castNames)." names, ".
! withThousands($count)." titles in ".withThousands($lineCount)." lines in %d seconds",time()-$startTime));
!
closeMaybeGunzip($file, $fh);
***************
*** 1933,1938 ****
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing Ratings found $count titles and ".
! "$lineCount lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
return($count);
--- 1966,1972 ----
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing Ratings found ".withThousands($count)." titles in ".
! withThousands($lineCount)." lines in %d seconds",time()-$startTime));
!
closeMaybeGunzip($file, $fh);
return($count);
***************
*** 1991,2000 ****
my ($title, $keyword) = ($line =~ m/^(.*)\s+(\S+)\s*$/);
if ( defined($title) and defined($keyword) ) {
- # there are some strange titles, fix them:
- $title =~ s/.*\s+{(.*)}/$1/;
! # ignore anything which is an episode and not a main title (e.g. "Doctor Who (#10.22)" or "(1986-09-18)" )
! if ( ( $title !~ m/\(#\d{1,3}\.?\d{0,5}\)/ )
! && ( $title !~ m/^\(\d{4}-\d{2}-\d{2}\)$/ ) )
{
if ( defined($self->{movies}{$title}) ) {
--- 2025,2033 ----
my ($title, $keyword) = ($line =~ m/^(.*)\s+(\S+)\s*$/);
if ( defined($title) and defined($keyword) ) {
! my ($episode) = $title =~ m/^.*\s+(\{.*\})$/;
!
! # ignore anything which is an episode (e.g. "{Doctor Who (#10.22)}" )
! if ( !defined $episode || $episode eq '' )
{
if ( defined($self->{movies}{$title}) ) {
***************
*** 2024,2029 ****
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing \"keywords\" found $count titles and ".
! "$lineCount lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
--- 2057,2062 ----
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing Keywords found ".withThousands($count)." titles in ".
! withThousands($lineCount)." lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
***************
*** 2076,2084 ****
chomp($line);
next if ($line =~ m/^\s*$/);
! my ($title, $episode) = ($line =~ m/^MV:\s(.*?)\s?({.*})?$/);
if ( defined($title) ) {
# ignore anything which is an episode (e.g. "{Doctor Who (#10.22)}" )
! if ( !defined $episode || $episode eq '' ) {
my $plot = '';
LOOP:
--- 2109,2118 ----
chomp($line);
next if ($line =~ m/^\s*$/);
! my ($title, $episode) = ($line =~ m/^MV:\s(.*?)\s?(\{.*\})?$/);
if ( defined($title) ) {
# ignore anything which is an episode (e.g. "{Doctor Who (#10.22)}" )
! if ( !defined $episode || $episode eq '' )
! {
my $plot = '';
LOOP:
***************
*** 2126,2131 ****
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing \"plots\" found $count plots and ".
! "$lineCount lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
--- 2160,2165 ----
$progress->update($countEstimate) if Have_bar;
! $self->status(sprintf("parsing Plots found $count ".withThousands($count)." in ".
! withThousands($lineCount)." lines in %d seconds",time()-$startTime));
closeMaybeGunzip($file, $fh);
***************
*** 2837,2841 ****
$self->status("merging in stage 5 data (genres)..");
if ( 1 ) {
! my $countEstimate=$self->dbinfoGet("db_stat_genres_count", 0);
my $progress=Term::ProgressBar->new({name => "merging genres",
count => $countEstimate,
--- 2871,2875 ----
$self->status("merging in stage 5 data (genres)..");
if ( 1 ) {
! my $countEstimate=$self->dbinfoGet("db_stat_genres_count", 1); # '1' prevents the spurious "(nothing to do)" msg
my $progress=Term::ProgressBar->new({name => "merging genres",
count => $countEstimate,
***************
*** 2890,2894 ****
$self->status("merging in stage 6 data (ratings)..");
if ( 1 ) {
! my $countEstimate=$self->dbinfoGet("db_stat_ratings_count", 0);
my $progress=Term::ProgressBar->new({name => "merging ratings",
count => $countEstimate,
--- 2924,2928 ----
$self->status("merging in stage 6 data (ratings)..");
if ( 1 ) {
! my $countEstimate=$self->dbinfoGet("db_stat_ratings_count", 1); # '1' prevents the spurious "(nothing to do)" msg
my $progress=Term::ProgressBar->new({name => "merging ratings",
count => $countEstimate,
------------------------------------------------------------------------------
"Accelerate Dev Cycles with Automated Cross-Browser Testing - For FREE
Instantly run your Selenium tests across 300+ browser/OS combos.
Get unparalleled scalability from the best Selenium testing platform available
Simple to use. Nothing to install. Get started now for free."
http://p.sf.net/sfu/SauceLabs