cvs: docweb /cron/weekly checkent /include lib_url_entities.inc.php /scripts checkent.php
[email protected] ("Sean Coates")
| Newsgroups | php.doc.web |
|---|---|
| Message-ID | <cvssean1106413743@cvsserver> |
sean Sat Jan 22 12:09:03 2005 EDT
Modified files:
/docweb/cron/weekly checkent
/docweb/include lib_url_entities.inc.php
/docweb/scripts checkent.php
Log:
more checkent cleaning && now with (optional) forking
http://cvs.php.net/diff.php/docweb/cron/weekly/checkent?r1=1.3&r2=1.4&ty=u
Index: docweb/cron/weekly/checkent
diff -u docweb/cron/weekly/checkent:1.3 docweb/cron/weekly/checkent:1.4
--- docweb/cron/weekly/checkent:1.3 Sat Jan 22 01:56:59 2005
+++ docweb/cron/weekly/checkent Sat Jan 22 12:09:03 2005
@@ -2,6 +2,8 @@
. `dirname $0`/../../build-ops
+NUMFORKS=10
+
cd ${SCRIPTSDIR}
${PHP} checkent.php phpdoc > /dev/null
${PHP} checkent.php peardoc > /dev/null
http://cvs.php.net/diff.php/docweb/include/lib_url_entities.inc.php?r1=1.2&r2=1.3&ty=u
Index: docweb/include/lib_url_entities.inc.php
diff -u docweb/include/lib_url_entities.inc.php:1.2 docweb/include/lib_url_entities.inc.php:1.3
--- docweb/include/lib_url_entities.inc.php:1.2 Sat Jan 22 10:15:26 2005
+++ docweb/include/lib_url_entities.inc.php Sat Jan 22 12:09:03 2005
@@ -19,7 +19,7 @@
| Mehdi Achour <[email protected]> |
| Sean Coates <[email protected]> |
+----------------------------------------------------------------------+
-$Id: lib_url_entities.inc.php,v 1.2 2005/01/22 15:15:26 sean Exp $
+$Id: lib_url_entities.inc.php,v 1.3 2005/01/22 17:09:03 sean Exp $
*/
// user agent
@@ -73,10 +73,30 @@
$schemes[] = 'ftp';
}
+// timeout
+define('URL_CONNECT_TIMEOUT', 10);
+
+// allow forking?
+define('URL_ALLOW_FORK', function_exists('pcntl_fork') && isset($_ENV['NUMFORKS']));
+define('NUM_ALLOWED_FORKS', URL_ALLOW_FORK ? $_ENV['NUMFORKS'] : 0);
+
// SQLite DB file
define('URL_ENT_SQLITE_FILE', SQLITE_DIR . "checkent_{$entType}.sqlite");
/**
+ * Opens a new SQLite connection
+ *
+ * @return resource SQLite connection
+ */
+function url_ent_sqlite_open()
+{
+ if (!($sqlite = sqlite_open(URL_ENT_SQLITE_FILE, 0666))) {
+ echo "Error creating database.\n";
+ }
+ return $sqlite;
+}
+
+/**
* Handles relative HTTP URLs
*
* @param string $url URL to handle
@@ -152,11 +172,17 @@
}
$port = isset($url['port']) ? $url['port'] : $port;
- if (!$fp = @fsockopen($scheme . $url['host'], $port)) {
+ $junk = '';
+ if (!$fp = @fsockopen($scheme . $url['host'], $port, $junk, $junk, URL_CONNECT_TIMEOUT)) {
return array(HTTP_CONNECT, array($num));
} else {
- fputs($fp, "HEAD {$url['path']} HTTP/1.0\r\nHost: {$url['host']}\r\nUser-agent: ". DOCWEB_CRAWLER_USER_AGENT ."\r\nConnection: close\r\n\r\n");
+ $query = "HEAD {$url['path']} HTTP/1.0\r\n"
+ ."Host: {$url['host']}\r\n"
+ ."User-agent: ". DOCWEB_CRAWLER_USER_AGENT ."\r\n"
+ ."Connection: close\r\n"
+ ."\r\n";
+ fputs($fp, $query);
$str = '';
while (!feof($fp)) {
@@ -226,8 +252,11 @@
* @param array $result result of check_url()
* @return void
*/
-function url_store_result(&$sqlite, $num, $name, $url, $result)
+function url_store_result($sqlite, $num, $name, $url, $result)
{
+ if (!$sqlite) {
+ $sqlite = url_ent_sqlite_open();
+ }
$return_val = isset($result[1][1]) ? $result[1][1] : '';
$sql = "
INSERT
http://cvs.php.net/diff.php/docweb/scripts/checkent.php?r1=1.15&r2=1.16&ty=u
Index: docweb/scripts/checkent.php
diff -u docweb/scripts/checkent.php:1.15 docweb/scripts/checkent.php:1.16
--- docweb/scripts/checkent.php:1.15 Sat Jan 22 10:15:26 2005
+++ docweb/scripts/checkent.php Sat Jan 22 12:09:03 2005
@@ -19,10 +19,11 @@
| Mehdi Achour <[email protected]> |
| Sean Coates <[email protected]> |
+----------------------------------------------------------------------+
-$Id: checkent.php,v 1.15 2005/01/22 15:15:26 sean Exp $
+$Id: checkent.php,v 1.16 2005/01/22 17:09:03 sean Exp $
*/
set_time_limit(0);
+$scriptBegin = time();
$inCli = true;
require_once '../include/init.inc.php';
@@ -50,14 +51,15 @@
require_once '../include/lib_url_entities.inc.php';
+echo "DocWeb URL Entity Checker.\n";
+echo "Using forks? ". (NUM_ALLOWED_FORKS ? 'yes: '. NUM_ALLOWED_FORKS : 'no') ."\n\n";
+
// create a new database (remove old first, if exists)
if (is_file(URL_ENT_SQLITE_FILE) && !unlink(URL_ENT_SQLITE_FILE)) {
echo "Error removing old database.\n";
die();
}
-if (!($sqlite = sqlite_open(URL_ENT_SQLITE_FILE, 0666))) {
- echo "Error creating database.\n";
-}
+$sqlite = url_ent_sqlite_open();
// Table creation
$sqlCreateMeta = "
@@ -100,7 +102,7 @@
$entity_names = $entities_found[1];
$entity_urls = $entities_found[3];
-echo "Found: ". count($entity_urls) ."URLs\n";
+echo "Found: ". count($entity_urls) ." URLs\n";
// log start time && schemes in DB
$sql = "
@@ -112,10 +114,49 @@
";
sqlite_query($sqlite, $sql);
-// Walk through entities found
-foreach ($entity_urls as $num => $entity_url) {
- echo "Checking: $entity_url\n";
- url_store_result($sqlite, $num, $entity_names[$num], $entity_url, check_url($num, $entity_url));
+if (URL_ALLOW_FORK) {
+ // use the forking method ... MUCH faster
+ declare(ticks=1);
+ $children = 0;
+ for ($num=0; $num<count($entity_urls); $num++) {
+ $url = $entity_urls[$num];
+ $name = $entity_names[$num];
+ if ($children < NUM_ALLOWED_FORKS) {
+ $pid = pcntl_fork();
+ if ($pid) {
+ // parent
+ //echo "Forked: $pid\n";
+ ++$children;
+ } else {
+ // child
+ echo "[$num] (". getmypid() .") Checking: $url\n";
+ url_store_result(FALSE, $num, $name, $url, check_url($num, $url));
+ exit();
+ }
+ } else {
+ // enough $children
+ $status = 0;
+ $child = pcntl_wait($status);
+ --$children;
+ echo "Child: $child exited with status $status ($children remain)\n";
+ }
+ }
+
+ while ($children) {
+ $status = 0;
+ $child = pcntl_wait($status);
+ --$children;
+ echo "Child: $child exited with status $status ($children remain)\n";
+ }
+
+} else {
+ // no forking
+ // walk through entities found
+ foreach ($entity_urls as $num => $entity_url) {
+ echo "[$num] Checking: $entity_url\n";
+ url_store_result($sqlite, $num, $entity_names[$num], $entity_url, check_url($num, $entity_url));
+ }
+ ++$num; // (for the count)
}
// log end time in DB
@@ -127,6 +168,10 @@
";
sqlite_query($sqlite, $sql);
-echo "Done.\n";
+$elapsed = time() - $scriptBegin;
+
+echo "\n";
+echo "Checked $num URLs.\n";
+echo "Completed in $elapsed seconds.\n";
?>