Re: PATCH: add git to verify-cvs2svn.py ; fix too many warnings issue
James Blackburn <[email protected]> Wed, 25 Aug 2010 13:29:41 +0100
| Newsgroups | gmane.comp.version-control.subversion.cvs2svn.devel |
|---|---|
| Message-ID | <[email protected]> |
On 25 August 2010 09:13, Michael Haggerty <[email protected]> wrote: > GIT_DIR=$PATH/.git git archive --format tar $BRANCH | > tar -C $dest_dir -x Learn something new every day :) Attached is a patch with both python tar and platform tar. I went for python tar first, and it worked until I tried a larger repo. I found that checking out some tags would cause the whole verify process to hang. Inserting prints showed that git_cmd.wait() never returned, and the underlying git proecss was <defunct>. I'm guessing the pipe is full causing wait() to deadlock. I've left this code in, disabled by default, in case you can spot something wrong. Otherwise It verifies a small cvs conversion ok. One other minor change: I find that if I pipe or redirect output of verify-cvs2svn.py it doesn't show up in a timely manner, so I've added a stdout.flush() at various stages in verify_contents(...). Cheers, James > > It should also be faster than cloning the repository for each branch. > > The disadvantage is that it relies on the presence of "tar" in the > command path. To get around that limitation, one could use Python's > "tarfile" module, using something like the last recipe on this page: > > http://docs.python.org/release/2.4/lib/tar-examples.html > >> It also fixes an issue whereby file mismatches are reproted file_size >> / 8K times. > > Cool. I just committed this part of your patch. > > Michael > > ------------------------------------------------------ http://cvs2svn.tigris.org/ds/viewMessage.do?dsForumId=1667&dsMessageId=2651184 To unsubscribe from this discussion, e-mail: [[email protected]].
verify-cvs2svn.patch
(application/octet-stream, 4.7 KB)
### Eclipse Workspace Patch 1.0
#P cvs2svn
Index: contrib/verify-cvs2svn.py
===================================================================
--- contrib/verify-cvs2svn.py (revision 5248)
+++ contrib/verify-cvs2svn.py (working copy)
@@ -36,12 +36,14 @@
import subprocess
import shutil
import re
+import tarfile
# CVS and Subversion command line client commands
CVS_CMD = 'cvs'
SVN_CMD = 'svn'
HG_CMD = 'hg'
+GIT_CMD = 'git'
def pipe(cmd):
@@ -247,7 +249,82 @@
name = 'git'
def __init__(self, path):
- raise NotImplementedError()
+ self.path = path
+ self.repo_cmd = [GIT_CMD, '--git-dir=' + self.path + '/.git', '--work-tree=' + self.path]
+
+ self._branches = None # cache result of branches()
+ self._have_master = None # so export_trunk() doesn't blow up
+
+ def __str__(self):
+ return os.path.basename(self.path)
+
+ def _export(self, dest_path, rev):
+ # clone the repository
+ cmd = [GIT_CMD, 'archive', '--remote=' + self.path, '--format=tar', rev]
+ git_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+
+ if False:
+ # Unfortunately for some git tags the below causes git_proc.wait() to hang
+ # The git archive process is in a <defunct> state and the verify-cvs2svn hangs for good.
+ tar = tarfile.open(mode="r|", fileobj=git_proc.stdout)
+ for tarinfo in tar:
+ tar.extract(tarinfo, dest_path)
+ tar.close()
+ else:
+ os.mkdir(dest_path)
+ tar_proc = subprocess.Popen(['tar', '-C', dest_path, '-x'], stdin=git_proc.stdout, stdout=subprocess.PIPE)
+ output = tar_proc.stdout.read()
+ status = tar_proc.wait()
+ if output or status:
+ raise RuntimeError('Git tar extraction of rev %s from repo %s to %s failed (%s)!' % rev, self.path, dest_path, output)
+
+ status = git_proc.wait()
+ if status:
+ raise RuntimeError('Git extract of rev %s from repo %s to %s failed!' % rev, self.path, dest_path)
+
+ if not os.path.exists(dest_path):
+ raise RuntimeError('Git clone of %s to %s failed!' % self.path, dest_path)
+
+ def export_trunk(self, dest_path):
+ self.branches() # ensure _have_default is set
+ if self._have_master:
+ self._export(dest_path, 'master')
+ else:
+ # same as CVS does when exporting empty trunk
+ os.mkdir(dest_path)
+
+ def export_tag(self, dest_path, tag):
+ self._export(dest_path, tag)
+
+ def export_branch(self, dest_path, branch):
+ self._export(dest_path, branch)
+
+ def tags(self):
+ cmd = self.repo_cmd + ['tag']
+ tags = self._split_output(cmd)
+ return tags
+
+ def branches(self):
+ if self._branches is None:
+ cmd = self.repo_cmd + ['branch']
+ branches = self._split_output(cmd)
+ # Remove the two chracters at the start of the branch name
+ for i in range(len(branches)):
+ branches[i] = branches[i][2:]
+ self._branches = branches
+ try:
+ branches.remove('master')
+ self._have_master = True
+ except ValueError:
+ self._have_master = False
+
+ return self._branches
+
+ def _split_output(self, cmd):
+ (output, status) = pipe(cmd)
+ if status:
+ cmd_failed(cmd, output, status)
+ return output.split("\n")[:-1]
def transform_symbol(ctx, name):
"""Transform the symbol NAME using the renaming rules specified
@@ -429,6 +506,7 @@
# Verify contents of trunk
print 'Verifying trunk'
+ sys.stdout.flush()
if not verify_contents_single(
failures, cvsrepos, verifyrepos, 'trunk', None, ctx
):
@@ -437,6 +515,7 @@
# Verify contents of all tags
for tag in verifyrepos.tags():
print 'Verifying tag', tag
+ sys.stdout.flush()
if not verify_contents_single(
failures, cvsrepos, verifyrepos, 'tag', tag, ctx
):
@@ -452,6 +531,7 @@
failures, cvsrepos, verifyrepos, 'branch', branch, ctx
):
locations.append('branch:' + branch)
+ sys.stdout.flush()
assert bool(failures) == bool(locations), \
"failures = %r\nlocations = %r" % (failures, locations)
@@ -464,6 +544,7 @@
sys.stdout.write(' %s\n' % location)
else:
sys.stdout.write('PASS: %s == %s\n' % (cvsrepos, verifyrepos))
+ sys.stdout.flush()
class OptionContext:
pass
@@ -495,7 +576,7 @@
help='assume verify-repos is hg')
parser.add_option('--git',
action='store_const', dest='repos_type', const='git',
- help='assume verify-repos is git (not implemented!)')
+ help='assume verify-repos is git')
parser.add_option('--suppress-keywords',
action='store_const', dest='keyword_opt', const='-kk',
help='suppress CVS keyword expansion '