Trunk-to-branch script

#! /usr/bin/python

# (Be in -*- python -*- mode.)
#
# ====================================================================
# Copyright (C) 2010 Cabot Communications Ltd.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
# This software consists of voluntary contributions made by many
# individuals.  For exact contribution history, see the revision
# history and logs, available at http://cvs2svn.tigris.org/.
# ====================================================================

"""Usage: trunk_to_branch.py BRANCHNAME PATH...

Modify RCS files in PATH to:

1) Remove all existing branches, including tags that are on them.
  - Tags that are on trunk are kept
  - Vendor branches with a single "cvs import" are supported.
    The commit message and any tags will be moved from r1.1.1.1
    to r1.1.  (The contents of r1.1 and r1.1.1.1 is always the
    same anyway)
  - Vendor branches with more than one "cvs import" are not
    supported and will cause a fatal error.

2) Rewrite history to change the old trunk to a branch.  The new
   history will show that the file was never committed to trunk,
   only to the branch.  BRANCHNAME is the name of the new branch.
   I.e. if the old history of "whatever.c" is:

     1.1 log: "first commit", file contents: "foo"
     1.2 log: "change to bar", file contents: "bar"
     1.3 log: "got it right this time", file contents: "baz"

   then the new history will be:

   1.1 (file deleted), log: "file whatever.c was initially added
                             on branch BRANCHNAME".
     --- Branch 1.1.2 BRANCHNAME ---
     1.1.2.1 log: "first commit", file contents: "foo"
     1.1.2.2 log: "change to bar", file contents: "bar"
     1.1.2.3 log: "got it right this time", file contents: "baz"

Rationale:

  In CVS, we have directories called "config_for_project_FOO".  These
  contain a script that will check out all the other modules, typically
  from PROJECT-FOO-BR branch.  The config directory is on trunk.

  In SVN, we'd like to have these config directories on the same branch
  as all the other code.  (Partly because it's slightly trickier to do
  multi-branch checkouts in Subversion.  We'd have to use svn:externals
  or "svn switch", and both those options add complexity).

   So we're using this script to rewrite the history of the config
   directories, so cvs2svn puts them on the right branch.
"""

from __future__ import with_statement

import sys
import os

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import cvs2svn_rcsparse
from rcs_file_filter import WriteRCSFileSink
from cvs2svn_lib.common import is_trunk_revision
from cvs2svn_lib.rcs_stream import RCSStream

class RcsCorruptionError(Exception):
    pass

class RevData:
    """Data about a CVS revision.

    Member variables:

    REVISION is a string revision number, e.g. "1.1".
    TIMESTAMP is the timestamp in UNIX format
    AUTHOR is a string containing the author name.
    STATE is the state as a string, typically "Exp" or "dead".
    NEXT is the string revision number of the next revision along this
         branch (for branch revisions), or the previous revision on
         trunk (for trunk revisions). If there is no such revision, NEXT is
         None.
    BRANCHES is a list containing the string revision numbers of the first
             commit on each branch which branches from here.
    LOG is the log message, as a string.  None if it hasn't yet been read.
    TEXT is the contents of this revision as a string.  This is fulltext for
        the current head revision, and an RCS diff for any other revision.
        None if it hasn't yet been read.
    """
    def __init__(self, revision, timestamp, author, state, next):
        self.revision = revision
        self.timestamp = timestamp
        self.author = author
        self.state = state
        self.next = next
        self.log = None
        self.text = None
        self.branches = []

def fulltext_to_rcsdiff_vs_empty(new_full_text):
    """Does a RCS diff from an empty "old" version and a specified "new"
    version.  Returns the RCS diff."""
    if new_full_text == '':
        # Old and new are identical, so diff is empty
        return ''

    # Count number of lines, remembering that the last line might not end
    # with a newline.
    num_lines = new_full_text.count('\n')
    if not new_full_text.endswith('\n'):
        num_lines = num_lines + 1

    # RCS diff header saying "insert at the start of the file, the
    # following NUM_LINES lines".
    header = 'a0 %d\n' % (num_lines,)

    return header + new_full_text

# When doing this trunk-to-branch process, we don't try to do incremental
# processing.  Instead, we record all the data we need as the file is
# parsed, throwing away branches and handling trivial vendor branches
# as we go.  We then have a second step where we do the trunk-to-branch
# conversion, and a third step to write out the results.
#
# General design philosophy: check everything.  If there's anything
# unexpected in the RCS file, throw an RcsCorruptionError and die.  (This
# reduces the risk of silently doing the wrong thing and corrupting data).
class CaptureRcsData(cvs2svn_rcsparse.Sink):
    def __init__(self):
        self.head = None
        self.principal_branch = None
        self.symbols = []
        self.revs = {}
        self.revs_list = []
        self.expansion = None

        # RCS junk that just gets copied through
        self.comment = None
        self.description = None

    #########################################################################
    # Step 1: Sink methods, used to populate this class
    #########################################################################

    def set_head_revision(self, revision):
        self.head = revision

    def set_principal_branch(self, branch_name):
        # There's a live vendor branch.  We currently only support the normal
        # vendor branch number, which is 1.1.1.
        if branch_name != '1.1.1':
            raise RcsCorruptionError(
                'Nonstandard vendor branch %r' % (branch_name,)
                )
        self.principal_branch = branch_name

    def set_access(self, accessors):
        # This field doesn't need to be preserved.
        pass

    def define_tag(self, name, revision):
        # Tag or branch.  Branches, and any tags on a branch, will be thrown
        # away.  Tags on the 1.1.1.1 vendor branch revision will be moved to
        # the corresponding trunk revision, rev 1.1.
        if revision == '1.1.1.1':
            revision = '1.1'
        if is_trunk_revision(revision):
            self.symbols.append((name, revision))

    def set_locker(self, revision, locker):
        # This field doesn't need to be preserved.
        pass

    def set_locking(self, mode):
        # This field doesn't need to be preserved.
        pass

    def set_comment(self, comment):
        self.comment = comment

    def set_expansion(self, mode):
        self.expansion = mode

    def admin_completed(self):
        if self.head is None:
            raise RcsCorruptionError('No head revision')

    def define_revision(
            self, revision, timestamp, author, state, branches, next
            ):
        if revision == '1.1.1.1':
            # Initial commit on vendor branch, created by "cvs import".
            # Just check it looks normal.
            if next is not None:
                raise RcsCorruptionError(
                    'More than one "cvs import" is not supported'
                    )
            if '1.1' not in self.revs:
                raise RcsCorruptionError(
                    'Misordered revisions: 1.1.1.1 should be after 1.1'
                    )
            if (self.revs['1.1'].author != author
                 or self.revs['1.1'].timestamp != timestamp
                 or self.revs['1.1'].state != state):
                raise RcsCorruptionError(
                    'Revision 1.1.1.1 doesn\'t look like a normal import'
                    )
        elif is_trunk_revision(revision):
            # Trunk revision we need to keep.
            # Check they're appearing in reverse chronological order, and
            # just add it to the list.
            if revision in self.revs:
                raise RcsCorruptionError(
                    'Duplicate revision %r' % (revision,)
                    )
            if len(self.revs_list) == 0:
                expect_rev = self.head
            else:
                expect_rev = self.revs_list[-1].next
            if revision != expect_rev:
                raise RcsCorruptionError(
                    'Misordered revisions: got %r, expected %r'
                        % (revision, expect_rev)
                    )

            new_rev = RevData(revision, timestamp, author, state, next)
            self.revs[revision] = new_rev
            self.revs_list.append(new_rev)

    def tree_completed(self):
        # We should now have initial entries for each revision.

        if not self.revs_list:
            raise RcsCorruptionError('No revisions')

        if self.revs_list[-1].next is not None:
            raise RcsCorruptionError(
                'Revision %r refers to nonexistant next rev %r'
                    % (self.revs_list[-1].revision, self.revs_list[-1].next)
                )

        if len(self.revs_list) > 1 and self.principal_branch is not None:
            raise RcsCorruptionError(
                'Vendor branch still active, dispite later trunk commits'
                )
        self.principal_branch = None # We're removing the vendor branch.

    def set_description(self, description):
        self.description = description

    def set_revision_info(self, revision, log, text):
        if revision == '1.1.1.1':
            # Initial commit on vendor branch, created by "cvs import".
            if '1.1' not in self.revs:
                raise RcsCorruptionError('Missing revision 1.1')
            if self.revs['1.1'].log is None:
                raise RcsCorruptionError(
                    'Misordered revisions: 1.1.1.1 should be after 1.1'
                    )
            if self.revs['1.1'].log != 'Initial revision\n' or text != '':
                raise RcsCorruptionError('Doesn\'t look like a normal import')
            self.revs['1.1'].log = log
        elif is_trunk_revision(revision):
            # Trunk revision we need to keep.  Just record LOG and TEXT.
            if revision not in self.revs:
                raise RcsCorruptionError('Missing revision %r' % (revision,))
            this_rev = self.revs[revision]
            if this_rev.log is not None:
                raise RcsCorruptionError(
                    'Duplicate data for revision %r' % (revision,)
                    )
            this_rev.log = log
            this_rev.text = text

    def parse_completed(self):
        for rev in self.revs.itervalues():
            if rev.log is None:
                raise RcsCorruptionError(
                    'Missing data for revision %r' % (rev.revision,)
                    )
        del self.revs # Finished with this dict.  Will use REVS_LIST later.


    #########################################################################
    # Step 2: Processing
    #########################################################################

    def trunk_to_branch(self, new_branch_name, filename):
        """Convert the trunk to a branch.

        NEW_BRANCH_NAME is the name of the new branch.
        FILENAME is the filename of this RCS file, which is only needed for
                 a log message.
        """
        
        # self.revs_list now contains revs in REVERSE chronological order,
        # starting with a fulltext and then with diffs for remaining ones.
        # Reverse it.
        stream = RCSStream(self.revs_list[0].text)
        for index in range(1, len(self.revs_list)):
            self.revs_list[index - 1].text = stream.invert_diff(
                self.revs_list[index].text
                )
        self.revs_list[-1].text = stream.get_text()
        self.revs_list.reverse()
        # self.revs_list now contains revs in NORMAL chronological order,
        # starting with a fulltext and then with diffs for remaining ones.

        # Renumber revs onto 1.1.2 branch.  As we go, we regenerate the
        # 'next' pointers and generate a mapping from old to new revision
        # numbers.  The mapping will be used later to update tags.
        revs_map = {}
        for idx, rev in enumerate(self.revs_list):
            new_revision = '1.1.2.%d' % (idx + 1)
            revs_map[rev.revision] = new_revision
            rev.revision = new_revision
            if idx != len(self.revs_list) - 1:
                # not last entry
                rev.next = '1.1.2.%d' % (idx + 2)
            else:
                rev.next = None

        # Tweak rev 1.1.2.1 to contain a diff rather than a fulltext.
        # We're going to make rev 1.1 empty, i.e. the old file we're going
        # to apply this diff to is zero bytes long.
        self.revs_list[0].text = fulltext_to_rcsdiff_vs_empty(
            self.revs_list[0].text
            )

        # Insert a new dead revision 1.1 at the start of self.revs.
        rev = RevData('1.1', self.revs_list[0].timestamp,
                      self.revs_list[0].author, 'dead', None)
        rev.text = ''
        rev.branches = ['1.1.2.1']
        rev.log = 'file %s was initially added on branch %s.\n' % (
                    filename, new_branch_name)
        self.revs_list.insert(0, rev)
        self.head = '1.1'

        # Map revision numbers in self.symbols.  As we go, double-check that
        # the new branch name isn't already used.
        new_symbols = []
        for (name, revision) in self.symbols:
            if name == new_branch_name:
                raise RcsCorruptionError(
                    'Symbol %r already exists' % (new_branch_name,)
                    )
            new_symbols.append((name, revs_map[revision]))
        self.symbols = new_symbols

        # Name the new branch.
        self.symbols.append((new_branch_name, '1.1.0.2'))

    #########################################################################
    # Step 3: Output
    #########################################################################

    def write(self, dest):
        """Write out the data to the Sink DEST."""
        dest.set_head_revision(self.head)
        for (name, revision) in self.symbols:
            dest.define_tag(name, revision)
        dest.set_locking("strict")
        dest.set_comment(self.comment)
        dest.set_expansion(self.expansion)
        dest.admin_completed()
        for rev in self.revs_list:
            dest.define_revision(rev.revision, rev.timestamp, rev.author,
                                 rev.state, rev.branches, rev.next)
        dest.tree_completed()
        dest.set_description(self.description)
        for rev in self.revs_list:
            dest.set_revision_info(rev.revision, rev.log, rev.text)
        dest.parse_completed()


def process_file(filename, branchname):
    """Process a single file.

    Removes pre-existing branches and converts the trunk to a branch called
    BRANCHNAME.  The RCS file on disk is replaced with the edited version.
    FILENAME is the full path to the RCS file on disk, which is read and then
    replaced with the edited version."""

    # Step 1: Read file
    with open(filename, 'rb') as infp:
        data = CaptureRcsData()
        cvs2svn_rcsparse.parse(infp, data)

    # Step 2: Processing
    basename = os.path.basename(filename)
    if basename.endswith(',v'):
        basename = basename[:-2]
    data.trunk_to_branch(branchname, basename)

    # Step 3: Write out
    tmp_filename = filename + '.tmp'
    with open(tmp_filename, 'wb') as outfp:
        data.write(WriteRCSFileSink(outfp))
    os.rename(tmp_filename, filename)

def iter_files_in_dir(top_path):
    """Iterator that gives the full path to every file in a specified
    directory tree.  This recurses into subdirectories."""
    for (dirpath, dirnames, filenames) in os.walk(top_path):
        for name in filenames:
            yield os.path.join(dirpath, name)

def iter_rcs_files(list_of_starting_paths, verbose=False):
    """Iterator that gives the full path to every RCS file in a specified list
    of starting paths.  If one of the starting paths is a directory, then it
    will list all RCS files in that directory, including subdirectories.

    Optionally, if VERBOSE is True, will print out messages about ignored
    files."""
    for base_path in list_of_starting_paths:
        if os.path.isfile(base_path) and base_path.endswith(',v'):
            yield base_path
        elif os.path.isdir(base_path):
            for file_path in iter_files_in_dir(base_path):
                if file_path.endswith(',v'):
                    yield file_path
                elif verbose:
                    sys.stdout.write('File %s is being ignored.\n' % file_path)
        elif verbose:
            sys.stdout.write('PATH %s is being ignored.\n' % base_path)

def main():
    if len(sys.argv) < 3:
        sys.stderr.write('Usage: %s BRANCHNAME PATH...\n' % (sys.argv[0],))
        sys.exit(1)

    branchname = sys.argv[1]
    for path in iter_rcs_files(sys.argv[2:], verbose=True):
        sys.stdout.write('Processing %s...' % path)
        process_file(path, branchname)
        sys.stdout.write('done.\n')

if __name__ == '__main__':
    main()