Re: PATCH: add git to verify-cvs2svn.py ; fix too many warnings issue

James Blackburn <[email protected]> Wed, 25 Aug 2010 13:29:41 +0100
Newsgroups gmane.comp.version-control.subversion.cvs2svn.devel
Message-ID <[email protected]>
On 25 August 2010 09:13, Michael Haggerty <[email protected]> wrote:
> GIT_DIR=$PATH/.git git archive --format tar $BRANCH |
>    tar -C $dest_dir -x

Learn something new every day :)

Attached is a patch with both python tar and platform tar. I went for python
tar first, and it worked until I tried a larger repo.  I found that checking
out some tags would cause the whole verify process to hang.  Inserting
prints showed that git_cmd.wait() never returned, and the underlying git
proecss was <defunct>.  I'm guessing the pipe is full causing wait() to
deadlock.
I've left this code in, disabled by default, in case you can spot something
wrong.

Otherwise It verifies a small cvs conversion ok.

One other minor change:  I find that if I pipe or redirect output of
verify-cvs2svn.py it doesn't show up in a timely manner, so I've added a
stdout.flush() at various stages in verify_contents(...).

Cheers,
James

>
> It should also be faster than cloning the repository for each branch.
>
> The disadvantage is that it relies on the presence of "tar" in the
> command path.  To get around that limitation, one could use Python's
> "tarfile" module, using something like the last recipe on this page:
>
> http://docs.python.org/release/2.4/lib/tar-examples.html
>
>> It also fixes an issue whereby file mismatches are reproted file_size
>> / 8K times.
>
> Cool.  I just committed this part of your patch.
>
> Michael
>
>

------------------------------------------------------
http://cvs2svn.tigris.org/ds/viewMessage.do?dsForumId=1667&dsMessageId=2651184

To unsubscribe from this discussion, e-mail: [[email protected]].
verify-cvs2svn.patch (application/octet-stream, 4.7 KB)
### Eclipse Workspace Patch 1.0
#P cvs2svn
Index: contrib/verify-cvs2svn.py
===================================================================
--- contrib/verify-cvs2svn.py	(revision 5248)
+++ contrib/verify-cvs2svn.py	(working copy)
@@ -36,12 +36,14 @@
 import subprocess
 import shutil
 import re
+import tarfile
 
 
 # CVS and Subversion command line client commands
 CVS_CMD = 'cvs'
 SVN_CMD = 'svn'
 HG_CMD = 'hg'
+GIT_CMD = 'git'
 
 
 def pipe(cmd):
@@ -247,7 +249,82 @@
   name = 'git'
 
   def __init__(self, path):
-    raise NotImplementedError()
+    self.path = path
+    self.repo_cmd = [GIT_CMD, '--git-dir=' + self.path + '/.git', '--work-tree=' + self.path]
+
+    self._branches = None               # cache result of branches()
+    self._have_master = None           # so export_trunk() doesn't blow up
+
+  def __str__(self):
+    return os.path.basename(self.path)
+
+  def _export(self, dest_path, rev):
+    # clone the repository
+    cmd = [GIT_CMD, 'archive', '--remote=' + self.path, '--format=tar', rev]
+    git_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+    if False:
+      # Unfortunately for some git tags the below causes git_proc.wait() to hang
+      # The git archive process is in a <defunct> state and the verify-cvs2svn hangs for good.
+      tar = tarfile.open(mode="r|", fileobj=git_proc.stdout)
+      for tarinfo in tar:
+        tar.extract(tarinfo, dest_path)
+      tar.close()
+    else:
+      os.mkdir(dest_path)
+      tar_proc = subprocess.Popen(['tar', '-C', dest_path, '-x'], stdin=git_proc.stdout, stdout=subprocess.PIPE)
+      output = tar_proc.stdout.read()
+      status = tar_proc.wait()
+      if output or status:
+        raise RuntimeError('Git tar extraction of rev %s from repo %s to %s failed (%s)!' % rev, self.path, dest_path, output)
+
+    status = git_proc.wait()
+    if status:
+      raise RuntimeError('Git extract of rev %s from repo %s to %s failed!' % rev, self.path, dest_path)
+
+    if not os.path.exists(dest_path):
+      raise RuntimeError('Git clone of %s to %s failed!' % self.path, dest_path)
+
+  def export_trunk(self, dest_path):
+    self.branches()                     # ensure _have_default is set
+    if self._have_master:
+      self._export(dest_path, 'master')
+    else:
+      # same as CVS does when exporting empty trunk
+      os.mkdir(dest_path)
+
+  def export_tag(self, dest_path, tag):
+    self._export(dest_path, tag)
+
+  def export_branch(self, dest_path, branch):
+    self._export(dest_path, branch)
+
+  def tags(self):
+    cmd = self.repo_cmd + ['tag']
+    tags = self._split_output(cmd)
+    return tags
+
+  def branches(self):
+    if self._branches is None:
+      cmd = self.repo_cmd + ['branch']
+      branches = self._split_output(cmd)
+      # Remove the two chracters at the start of the branch name
+      for i in range(len(branches)):
+        branches[i] = branches[i][2:]
+      self._branches = branches
+      try:
+        branches.remove('master')
+        self._have_master = True
+      except ValueError:
+        self._have_master = False
+
+    return self._branches
+
+  def _split_output(self, cmd):
+    (output, status) = pipe(cmd)
+    if status:
+      cmd_failed(cmd, output, status)
+    return output.split("\n")[:-1]
 
 def transform_symbol(ctx, name):
   """Transform the symbol NAME using the renaming rules specified
@@ -429,6 +506,7 @@
 
   # Verify contents of trunk
   print 'Verifying trunk'
+  sys.stdout.flush()
   if not verify_contents_single(
         failures, cvsrepos, verifyrepos, 'trunk', None, ctx
         ):
@@ -437,6 +515,7 @@
   # Verify contents of all tags
   for tag in verifyrepos.tags():
     print 'Verifying tag', tag
+    sys.stdout.flush()
     if not verify_contents_single(
           failures, cvsrepos, verifyrepos, 'tag', tag, ctx
           ):
@@ -452,6 +531,7 @@
             failures, cvsrepos, verifyrepos, 'branch', branch, ctx
             ):
         locations.append('branch:' + branch)
+    sys.stdout.flush()
 
   assert bool(failures) == bool(locations), \
          "failures = %r\nlocations = %r" % (failures, locations)
@@ -464,6 +544,7 @@
       sys.stdout.write('  %s\n' % location)
   else:
     sys.stdout.write('PASS: %s == %s\n' % (cvsrepos, verifyrepos))
+  sys.stdout.flush()
 
 class OptionContext:
   pass
@@ -495,7 +576,7 @@
                     help='assume verify-repos is hg')
   parser.add_option('--git',
                     action='store_const', dest='repos_type', const='git',
-                    help='assume verify-repos is git (not implemented!)')
+                    help='assume verify-repos is git')
   parser.add_option('--suppress-keywords',
                     action='store_const', dest='keyword_opt', const='-kk',
                     help='suppress CVS keyword expansion '