createrepo/__init__.py worker.py

 createrepo/__init__.py |    8 +++++++-
 worker.py              |   11 ++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

New commits:
commit fcb7fa19a488fd60c27b912fe2bbc16b614decb8
Author: Seth Vidal <[email protected]>
Date:   Wed Aug 31 12:47:18 2011 -0400

    add --pkglist option to worker.py - this allows us to pass
    the list of pkgs from a tempfile made by the main process.
    
    Apparently the shell doesn't like it when the worker is passed 30K pkgs
    as arguments :)
    
    this should resolve
    https://bugzilla.redhat.com/show_bug.cgi?id=734838

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index e3c18aa..0fbbff3 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -608,6 +608,12 @@ class MetaDataGenerator:
                 base_worker_cmdline.append('--verbose')
                 
             for worker_num in range(self.conf.workers):
+                pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num
+                print pkl
+                f = open(pkl, 'w') 
+                f.write('\n'.join(worker_chunks[worker_num]))
+                f.close()
+                
                 # make the worker directory
                 workercmdline = []
                 workercmdline.extend(base_worker_cmdline)
@@ -616,7 +622,7 @@ class MetaDataGenerator:
                     workercmdline.append('--tmpmdpath=%s' % thisdir)
                 else:
                     raise MDError, "Unable to create worker path: %s" % thisdir
-                workercmdline.extend(worker_chunks[worker_num])
+                workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num))
                 worker_cmd_dict[worker_num] = workercmdline
             
                 
diff --git a/worker.py b/worker.py
index ab78d90..23c87a3 100755
--- a/worker.py
+++ b/worker.py
@@ -5,6 +5,7 @@ import yum
 import createrepo
 import os
 import rpmUtils
+import re
 from optparse import OptionParser
 
 
@@ -23,6 +24,8 @@ def main(args):
     parser = OptionParser()
     parser.add_option('--tmpmdpath', default=None, 
                 help="path where the outputs should be dumped for this worker")
+    parser.add_option('--pkglist', default=None, 
+                help="file to read the pkglist from in lieu of all of them on the cli")
     parser.add_option("--pkgoptions", default=[], action='append',
                 help="pkgoptions in the format of key=value")
     parser.add_option("--quiet", default=False, action='store_true',
@@ -68,7 +71,13 @@ def main(args):
     fl = open(opts.tmpmdpath  + '/filelists.xml' , 'w')
     other = open(opts.tmpmdpath  + '/other.xml' , 'w')
     
-    
+    if opts.pkglist:
+        for line in open(opts.pkglist,'r').readlines():
+            line = line.strip()
+            if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
+                continue
+            pkgs.append(line)
+
     for pkgfile in pkgs:
         pkgpath = reldir + '/' + pkgfile
         if not os.path.exists(pkgpath):