cvs: SVNROOT / run-conversion.php

http://cvs.php.net/viewvc.cgi/SVNROOT/run-conversion.php?r1=1.60&r2=1.61&diff_format=u
Index: SVNROOT/run-conversion.php
diff -u SVNROOT/run-conversion.php:1.60 SVNROOT/run-conversion.php:1.61
--- SVNROOT/run-conversion.php:1.60	Wed Dec 17 17:44:19 2008
+++ SVNROOT/run-conversion.php	Sat Jun 20 09:50:55 2009
@@ -7,7 +7,7 @@
 
 // -----------------------------------------------------------------------------------------------------------------------------
 // Constants
-$version = substr('$Revision: 1.60 $', strlen('$Revision: '), -2);
+$version = substr('$Revision: 1.61 $', strlen('$Revision: '), -2);
 
 $passes = array(
     'processcvs',       // Process CVS modules
@@ -26,6 +26,8 @@
 
 // -----------------------------------------------------------------------------------------------------------------------------
 // Utility functions
+
+// Print a verbose message if the current verbosity level allows
 function v($level, $message)
 {
     if ($GLOBALS['options']['verbose'] >= $level) {
@@ -33,12 +35,14 @@
     }
 }
 
+// Note a fatal error and exit. Just die(), really.
 function error($message, $status = 1)
 {
     print $message;
     exit($status);
 }
 
+// Returns true if either 1) We're running all passes, or 2) running the specific pass given
 function is_running_pass($pass)
 {
     return $GLOBALS['options']['pass'] === NULL || $GLOBALS['options']['pass'] == $pass;
@@ -49,6 +53,7 @@
     return !($value === '.' || $value === '..');
 }
 
+// A version of scandir() which doesn't return the . and .. entries
 function scandir_no_meta($directory)
 {
     if (func_num_args() == 3) {
@@ -61,30 +66,38 @@
     return array_filter($results, 'scandir_is_meta');
 }
 
-function run_command($command, $ignoreErrors = FALSE)
+// Run a shell command with verbose commentary. Does NOT escape its argument. Can be made to ignore errors.
+function run_command($command, $ignoreErrors = FALSE, &$exitStatus = NULL)
 {
     v(2, "Running '{$command}'...");
-    exec($command, $output, $exitstatus);
-    if ($exitstatus != 0 && !$ignoreErrors) {
+    exec($command, $output, $lExitStatus);
+    if ($lExitStatus != 0 && !$ignoreErrors) {
         error("\nAn error occurred. Exit status was {$exitstatus}. Output:\n" . implode("\n", $output) . "\n");
-    } else if ($exitstatus != 0) {
+    } else if ($lExitStatus != 0) {
         v(2, " warning: an error occurred.\n");
     } else {
         v(2, " done.\n");
     }
+    if ($exitStatus !== NULL) {
+        $exitStatus = $lExitStatus;
+    }
     return $output;
 }
 
+// Recursive premissions-preserving verbose copy which appends to temp_path/cp-output
 function cp($source, $dest, $ignoreErrors = FALSE)
 {
-    run_command("cp -Rpv " . escapeshellarg($source) . ' ' . escapeshellarg($dest) . ' >> ' . escapeshellarg("{$GLOBALS['temp_path']}/cp-output"), $ignoreErrors);
+    run_command('cp -Rpv ' . escapeshellarg($source) . ' ' . escapeshellarg($dest) .
+                    ' >> ' . escapeshellarg($GLOBALS['temp_path'] . DIR_SEP . 'cp-output'), $ignoreErrors);
 }
 
+// Rename from source to dest.
 function mv($source, $dest)
 {
     run_command("mv " . escapeshellarg($source) . ' ' . escapeshellarg($dest));
 }
 
+// Recursive forced unlink
 function rm($path)
 {
     run_command("rm -Rf " . escapeshellarg($path));
@@ -133,6 +146,12 @@
     'default' => './phpsvn.options.skel',
     'description' => 'Skeleton file to use for cvs2svn conversions.',
     'action' => 'StoreString'));
+$cmdline_parser->addOption('noclean', array(
+    'short_name' => '-d',
+    'long_name' => '--noclean',
+    'description' => 'Don\'t remove temporary files when conversion is done.',
+    'default' => FALSE,
+    'action' => 'StoreTrue'));
 
 try {
     $result = $cmdline_parser->parse();
@@ -173,7 +192,8 @@
     private     $optionsFileContents = NULL;
     private     $outputPath = NULL;
     private     $seed = 0;
-    
+
+    // Read options file, replacing temporary directory reference immediately (not variant per run). 
     public function __construct($seed = NULL)
     {
         $this->optionsFileContents = file_get_contents($GLOBALS['options']['skeleton']);
@@ -181,23 +201,32 @@
         $this->seed = ($seed === NULL ? mt_rand() : $seed);
     }
     
+    // Update the SVN repository path
     public function setOutputPath($path)
     {
         $this->outputPath = $path;
     }
     
+    // Add a CVS module to the options. Special-cases several problematic modules.
     public function addCVSModule($moduleName, $isMetaModule = false)
     {
+        // Path to the module in the original CVS repository
         $cvspath = "{$GLOBALS['temp_path']}/cvsrepo/{$moduleName}/";
+        // A meta-module is one which contains a list of other modules that need to be individually handled.
+        // Currently this includes only PEAR's modules and PECL's extensions
         if ($isMetaModule) {
+            // Prefix of module
             $modprefix = $moduleName . '/';
+            // For each submodule in the meta-module...
             foreach(scandir_no_meta($cvspath) as $module) {
+                // Special case for problematic modules
                 if (!is_dir($cvspath . $module) || 
                         count(scandir($cvspath . $module)) == 2 || $module == 'Attic' ||
                         ($moduleName == 'php/pecl' && $module == 'libextractor') ||
                         ($moduleName == 'pear/modules' && in_array($module, array('HTML_QuickForm_ComboBox', 'Services_Compete', 'XML_HTMLSax3')))) {
                     continue; // empty dir
                 }
+                // Add the submodule as a subdirectory of the meta-module. Notice the meta-module itself is never added.
                 $this->optionsFileContents .= <<<EOEXTRA
 run_options.add_project(
     '{$cvspath}{$module}',
@@ -211,6 +240,7 @@
 EOEXTRA;
             }
         } else {
+            // Add the plain old module as a root directory
             $this->optionsFileContents .= <<<EOEXTRA
 run_options.add_project(
     '{$cvspath}',
@@ -227,10 +257,13 @@
     
     public function run()
     {
+        // Replace output repository path here now that it can't change again.
         $this->optionsFileContents = str_replace('@@@OUTPUT_PATH@@@', $this->outputPath, $this->optionsFileContents);
+        // Choose a random filename for the options file, which has to exist on disk (grumble)
         $filename = $GLOBALS['temp_path'] . DIR_SEP . 'cvs2svn.options.' . $this->seed;
         file_put_contents($filename, $this->optionsFileContents);
 
+        // Run the cvs2svn command with various pipes.
         $command = "exec cvs2svn --options=" . escapeshellarg($filename);
         v(2, "Running: '{$command}'...");
         $cvs2svn_process = proc_open($command, array(
@@ -240,6 +273,7 @@
             ), $pipes, NULL, NULL);
         fclose($pipes[0]);
         $procinfo = array();
+        // Sleep and let cvs2svn do its work. 500,000us is 1/2 second. Could probably push that even higher without noticable penalty.
         do {
             usleep(500000);
             $procinfo = proc_get_status($cvs2svn_process);
@@ -285,9 +319,17 @@
 
     $converter = new CVS2SVNConverter;
     
+    // Copy the real CVSROOT to a temporary staging area, since we'll be modding it heavily.
     $cvsPath = "{$GLOBALS['temp_path']}/cvsrepo";
     cp($GLOBALS['options']['cvsroot'], $cvsPath, TRUE);
     
+    // Some renames:
+    //      cvs/phpdoc      => cvs/doc-base ; Will be moved into phpdoc/ later
+    //      cvs/phpdoc/en   => cvs/phpdoc-en ; Will be renamed to "en" and moved into phpdoc/ later
+    //      cvs/pear        => cvs/modules ; This will be moved into a "pear" directory later
+    //      cvs/TSRM        => cvs/php-src/TSRM ; Merge TSRM into php-src
+    //      cvs/ZendEngine2 => cvs/php-src/ZendEngine2 ; Merge ZE2 into php-src
+    //      cvs/pdo-specs   => cvs/pecl/pdo/specs ; This didn't belong as a separate module to begin with and will be merged into PECL's PDO
     mv("{$cvsPath}/phpdoc", "{$cvsPath}/doc-base");
     mv("{$cvsPath}/doc-base/en", "{$cvsPath}/phpdoc-en");
     mv("{$cvsPath}/pear", "{$cvsPath}/modules");
@@ -295,14 +337,17 @@
     mv("{$cvsPath}/ZendEngine2", "{$cvsPath}/php-src/ZendEngine2");
     mv("{$cvsPath}/pdo-specs", "{$cvsPath}/pecl/pdo/specs");
     
+    // There are several symlinks in php-src to PECL extensions which confuse cvs2svn, so remove them. These will be properly restored later.
     $links = array('bz2', 'hash', 'oci8', 'pdo', 'pdo_dblib', 'pdo_firebird', 'spl', 'soap', 'sqlite', 'tidy', 'xmlreader', 'xmlwriter', 'filter', 'ircg',
         'json', 'pdo_mysql', 'pdo_oci', 'pdo_odbc', 'pdo_pgsql', 'pdo_sqlite', 'simplexml', 'fileinfo');
     foreach ($links as $link) {
         rm("{$cvsPath}/php-src/ext/{$link}");
     }
-
+    
+    // Get a base list of CVS modules.
     $cvs_modules = scandir_no_meta($cvsPath);
-
+    
+    // Setup the basic directory structure in the CVS root for restructure purposes
     mkdir("{$cvsPath}/gtk");
     mkdir("{$cvsPath}/php");
     mkdir("{$cvsPath}/pear");
@@ -310,23 +355,32 @@
     mkdir("{$cvsPath}/phpdoc");
     
     foreach ($cvs_modules as $cvs_module) {
+        // CVSROOT is not converted. Period.
         if ($cvs_module == 'CVSROOT') {
             continue;
         }
+        // Raw files in the cvs repo, empty directories (count(scandir()) == 2 (./..)), and selected obsolete modules are not converted.
         if (!is_dir("{$cvsPath}/{$cvs_module}") || count(scandir("{$cvsPath}/{$cvs_module}")) == 2 ||
                 in_array($cvs_module, array('smarty', 'smarty-web', 'php4.fubar', 'php4.unused', 'peardoc.backup', 'php3', 'phpfi', 'livedocs',
                     'Zend', 'functable', 'dialin', 'docstuff', 'jpgraph', 'phpoc_de', 'phpdoc-fa_IR', 'lxr', 'portal', 'pres', 'embed', 'imapd'))) {
             rm("{$cvsPath}/{$cvs_module}");
+        // Modules belonging to PHP-GTK go into cvs/gtk/<module>
         } else if (in_array($cvs_module, array('php-gtk', 'php-gtk-doc', 'old-php-gtk-modules', 'php-gtk-web'))) {
             mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/gtk/{$cvs_module}");
             $converter->addCVSModule("gtk/{$cvs_module}");
+        // Modules belonging to the PHP base go into cvs/php/<module>. Note that this includes PECL!
+        //  PECL is a meta-module. (see CVS2SVNConverter::addCVSModule)
         } else if (in_array($cvs_module, array('ZendAPI', 'php-src', 'pecl', 'php-objc', 'php-lang',
                 'win-installer', 'bindlib_w32', 'zlib', 'pdo-specs', 'php-internals-win', 'fastcgi-isapi'))) {
             mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/php/{$cvs_module}");
             $converter->addCVSModule("php/{$cvs_module}", $cvs_module == 'pecl');
+        // Modules belonging to PEAR (especially the "modules" directory which contains all PEAR modules) go into cvs/pear/<module>
+        //  PEAR/modules is a meta-module. (see CVS2SVNConverter::addCVSModule)
         } else if (in_array($cvs_module, array('modules', 'pearbot', 'pear-core', 'peardoc', 'pearweb'))) {
             mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/pear/{$cvs_module}");
             $converter->addCVSModule("pear/{$cvs_module}", $cvs_module == 'modules');
+        // Modules belonging to websites, including the presentation, bugtracker, and pecl4win modules, go into cvs/web/<module>
+        //  All modules that end in web are matched, and added with the "-?web" suffix stripped.
         } else if (substr($cvs_module, -3) == "web" || in_array($cvs_module, array('pres2', 'presentations', 'bugtracker', 'pecl4win'))) {
             if (substr($cvs_module, -4) == '-web') {
                 $dest_cvs_module = substr($cvs_module, 0, -4);
@@ -337,6 +391,8 @@
             }
             mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/web/{$dest_cvs_module}");
             $converter->addCVSModule("web/{$dest_cvs_module}");
+        // Modules belonging to phpdoc (especially doc-base, see renamings) go into cvs/phpdoc/<module>
+        //  All modules matching "phpdoc-(.*)" are matched, and added with the "phpdoc-" prefix stripped.
         } else if (strncmp($cvs_module, "phpdoc", 6) == 0 || $cvs_module == 'doc-base') {
             if (substr($cvs_module, 0, 7) == 'phpdoc-') {
                 $dest_cvs_module = substr($cvs_module, 7);
@@ -345,11 +401,13 @@
             }
             mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/phpdoc/{$dest_cvs_module}");
             $converter->addCVSModule("phpdoc/{$dest_cvs_module}");
+        // All other modules go into cvs/<module>. There shouldn't be any of these.
         } else {
             $converter->addCVSModule($cvs_module);
         }
     }
     
+    // Strip any .svn dirs that might've been accidentally committed to CVS from other checkouts to avoid confusing the heck out of cvs2svn
     run_command('find ' . escapeshellarg("{$cvsPath}/pear/modules") . " -name '.svn' -type d -print0 | xargs -0 rm -Rf");
 }
     
@@ -397,6 +455,8 @@
 // ignores pass
 function pass_ignores()
 {
+    // Recursively processes the entire converted SVN repository, converting .cvsignore files to svn:ignore properties.
+    // XXX THIS MAY NO LONGER BE NECESSARY AS OF cvs2svn 2.2.0!
     $svnPath = "{$GLOBALS['options']['svnroot']}/repository";
     $coDir = "{$GLOBALS['temp_path']}/ignore_co";
     $dirStack = array($svnPath);
@@ -425,13 +485,17 @@
 // link pass
 function pass_link()
 {
+    // Re-link the PECL extensions to php-src using svn:externals.
+    // XXX THERE IS A BUG HERE WHICH IS CAUSING EXTENSIONS TO BE LINKED INTO BRANCHES THEY SHOULDN'T BE
     $svnPath = "{$GLOBALS['options']['svnroot']}/repository/php/php-src";
-
+    
+    // Get a complete list of tags and branches in the converted SVN repository
     $rawTagList = run_command('exec svn ls file:///' . escapeshellarg("{$svnPath}/tags"));
     $rawBranchList = run_command('exec svn ls file:///' . escapeshellarg("{$svnPath}/branches"));
     
     $oldcwd = getcwd();
 
+    // Creates a sparse checkout of the top levels ONLY of php/php-src/trunk, php/php-src/branches/*, and php/php-src/tags/*
     $sparseCoPath = "{$GLOBALS['temp_path']}/php-src-sparse-co";
     $tmpfilePath = "{$GLOBALS['temp_path']}/sparse-co-output";
     if (!file_exists($sparseCoPath)) {
@@ -456,52 +520,47 @@
     
     chdir($sparseCoPath);
     
-    $tagList = array();
-    $branchList = array();
-    foreach ($rawTagList as $tag) {
-        $realTag = substr($tag, 0, -1);
-        $xml = run_command('exec svn info --xml file:///' . escapeshellarg("{$svnPath}/tags/{$realTag}"));
-        $xml = implode("\n", $xml);
-        $tagList[$realTag] = strtotime(substr($xml, strpos($xml, '<date>') + 6, strlen('0000-00-00T00:00:00.000000Z')));
-        run_command('exec svn propdel svn:externals ' . escapeshellarg('tags' . DIR_SEP . $realTag . DIR_SEP . 'ext') . ' 2>&1');
-    }
-    foreach ($rawBranchList as $branch) {
-        $realBranch = substr($branch, 0, -1);
-        $xml = run_command('exec svn info --xml file:///' . escapeshellarg("{$svnPath}/branches/{$realBranch}"));
-        $xml = implode("\n", $xml);
-        $branchList[$realBranch] = strtotime(substr($xml, strpos($xml, '<date>') + 6, strlen('0000-00-00T00:00:00.000000Z')));
-        run_command('exec svn propdel svn:externals ' . escapeshellarg('branches' . DIR_SEP . $realBranch . DIR_SEP . 'ext') . ' 2>&1');
-    }
-    run_command('exec svn propdel svn:externals ' . escapeshellarg('trunk' . DIR_SEP . 'ext') . ' 2>&1');
-    
+    // List of extensions that should be linked from PECL.
+    //  Note: This list is accurate as of 2009-06-20
     $links = array('bz2', 'hash', 'oci8', 'pdo', 'pdo_dblib', 'pdo_firebird', 'spl', 'soap', 'sqlite', 'tidy', 'xmlreader', 'xmlwriter', 'filter', 'json',
         'pdo_mysql', 'pdo_oci', 'pdo_odbc', 'pdo_pgsql', 'pdo_sqlite', 'simplexml', 'fileinfo');
     
+    // For each link, get its creation date from the ORIGINAL CVS root.
+    //  Then, for each tag and branch, see whether the given link has a same-named directory. If so, make a link. Always make a link for trunk.
     $props = array();
-    
     foreach ($links as $link) {
-        $info = lstat("{$GLOBALS['options']['cvsroot']}/php-src/ext/{$link}"); // Use the original CVS root here so we get proper timestamps
-        $createDate = $info['ctime'];
-        foreach ($tagList as $tag => $tagDate) {
-            $path = 'tags' . DIR_SEP . $tag . DIR_SEP . 'ext';
-            if ($createDate <= $tagDate) {
-                $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/tags/{$tag} {$link}\n";
+        foreach ($rawTagList as $tag) {
+            $tag = substr($tag, 0, -1);
+            $externalPath = $svnPath . "/php/pecl/{$link}/tags/{$tag}";
+            run_command("exec svn ls file:///" . escapeshellarg($externalPath) . " 2>/dev/null", TRUE, $status);
+            if ($status === 0) {
+                v(2, "Tag {$tag} exists in ^/php/pecl/{$link}/tags. Adding external.\n");
+                $path = 'tags' . DIR_SEP . $tag . DIR_SEP . 'ext';
+                $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/tags/{$tag} {$link}";
+            } else {
+                v(2, "Tag {$tag} does NOT exist in ^/php/pecl/{$link}/tags. NOT adding external.\n");
             }
         }
-        foreach ($branchList as $branch => $branchDate) {
-            $path = 'branches' . DIR_SEP . $branch . DIR_SEP . 'ext';
-            if ($createDate <= $branchDate) {
-                $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/branches/{$branch} {$link}\n";
+        foreach ($rawBranchList as $branch) {
+            $branch = substr($branch, 0, -1);
+            $externalPath = $svnPath . "/php/pecl/{$link}/branches/{$branch}";
+            run_command("exec svn ls file:///" . escapeshellarg($externalPath) . " 2>/dev/null", TRUE, $status);
+            if ($status === 0) {
+                v(2, "Branch {$branch} exists in ^/php/pecl/{$link}/branches. Adding external.\n");
+                $path = 'branches' . DIR_SEP . $branch . DIR_SEP . 'ext';
+                $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/branches/{$branch} {$link}";
+            } else {
+                v(2, "Branch {$branch} does NOT exist in ^/php/pecl/{$link}/branches. NOT adding external.\n");
             }
         }
         $path = 'trunk' . DIR_SEP . 'ext';
-        $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/trunk {$link}\n";
+        $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/trunk {$link}";
     }
-
+    
     foreach ($props as $path => $value) {
         run_command('exec svn propset svn:externals ' . escapeshellarg($value) . ' ' . escapeshellarg($path));
     }
-    run_command('exec svn commit -m "[SVN CONVERSION] Reorganization in repository php-src."');
+    run_command('exec svn commit -m "[SVN CONVERSION] Reorganization in repository php-src (committing externals in php-src)."');
     
     chdir($oldcwd);
 }
@@ -532,7 +591,7 @@
 // -----------------------------------------------------------------------------------------------------------------------------
 // Cleanup
 
-if ($options['pass'] === NULL) {
+if ($options['pass'] === NULL && $options['noclean'] === FALSE) {
     v(2, "Removing temporary files.\n");
     system("rm -Rf " . escapeshellarg($temp_path));
 }