Re: Contributed work statistic from git

#!/usr/bin/python


from HTMLParser import HTMLParser
from datetime import datetime, timedelta
from urllib2 import urlopen
from sys import stdout

dt_start = "2009-01-01" # time interval
dt_end = "2009-12-31"

cgit_base_url = "http://git.gnome.org/browse/" # keep the last slash there
modules_list = [] # list of modules to check; if empty then filled by base URI

cbugs = {} # author->count of commits mentioning 'bug'
ctotal = {} # author->count of all commits
total_commits_read = 0

class ModulesHTMLParser(HTMLParser):
	read_repo = False

	def handle_starttag(self, tag, attrs):
		global modules_list;
		if (tag == "td") :
			for name, value in attrs:
				if (name == "class" and value.lower() == "sublevel-repo") :
					self.read_repo = True
		if (self.read_repo and tag == "a") :
			for name, value in attrs:
				if (name == "href" and value.find("/browse/") == 0) :
					modules_list.append (value[8:-1])
					self.read_repo = False

class CommitsHTMLParser(HTMLParser):
	in_content = 0
	found_content = False
	col = -1
	val_when = ""
	val_text = ""
	val_author = ""
	read_commits = 0
	read_done = False

	def handle_starttag(self, tag, attrs):
		if (tag == "div" and attrs[0][0] == "class" and attrs[0][1] == "content") :
			self.in_content += 1
		if (self.in_content > 0 and tag == "tr") :
			self.col = -1
		if (self.in_content > 0 and tag == "td") :
			self.col += 1

	def handle_endtag(self, tag):
		global dt_start
		global dt_end
		global cbugs
		global ctotal

		if (self.in_content > 0 and tag == "tr") :
			if (self.val_when != "" and self.val_author != "" and self.val_text != "" ) :
				#print "'%s'" % self.val_when
				#print "   '%s'" % self.val_author
				#print "   '%s'" % self.val_text
				if (self.val_when >= dt_start and self.val_when <= dt_end) :
					self.read_commits += 1
					if 'bug' in self.val_text.lower():
						cbugs.setdefault(self.val_author, 0)
						cbugs[self.val_author] += 1
					ctotal.setdefault(self.val_author, 0)
					ctotal[self.val_author] += 1
					
				if (self.val_when < dt_start) :
					self.read_done = True
			self.val_when = ""
			self.val_author = ""
			self.val_text = ""
		if (self.in_content > 0 and tag == "div") :
			self.in_content -= 1

	def handle_data(self, data):
		if (self.in_content > 0):
			if (self.col == 0):
				if (data.lower().find ("days") > 0) :
					days = int(data[0:data.find(" ")])
					data = (datetime.now() - timedelta(days=days)).date().isoformat()
				if (data.lower().find ("hours") > 0) :
					hours = int(data[0:data.find(" ")])
					data = (datetime.now() - timedelta(hours=hours)).date().isoformat()
				if (data.lower().find ("min.") > 0) :
					minutes = int(data[0:data.find(" ")])
					data = (datetime.now() - timedelta(minutes=minutes)).date().isoformat()
				self.val_when = data
				self.found_content = True

			if (self.col == 1):
				self.val_text += data
			if (self.col == 2):
				self.val_author = data

def check_module(name, idx, tot):
	global total_commits_read

	print ("   Checking in '%s' " % name) + ("(%d" % idx) + ("/%d)" % tot) ,
	stdout.flush()

	offset = 0
	url = cgit_base_url + name + "/log/?ofs="
	commitsParser = CommitsHTMLParser()
	while not commitsParser.read_done :
		u = url + "%d" % offset
		offset += 50
		commitsParser.found_content = False
		f = urlopen(u)
		try:
			for line in f:
				commitsParser.feed (line)
		finally:
			f.close()
		if (not commitsParser.found_content) :
			break

	print " found %d commits" % commitsParser.read_commits
	total_commits_read += commitsParser.read_commits

def count_commits(items):
    commits_by_name = {}
    for name, count in items.items():
        commits_by_name.setdefault(name, 0)
        commits_by_name[name] += count

    for value, name in sorted([ i[::-1] for i in commits_by_name.items()], reverse=True)[:10]:
        print "%-40s %s" % (name, value)


print "Calculate git commit statistics between %s" % dt_start + " and %s" % dt_end

if (modules_list == []) :
	print " * Retrieving list of modules..."
	modulesParser = ModulesHTMLParser()
	f = urlopen (cgit_base_url)
	try:
		for line in f:
			modulesParser.feed (line)
	finally:
		f.close()

modules_count = 0
for m in modules_list:
	modules_count += 1

mod_idx = 1
for m in modules_list:
	check_module(m, mod_idx, modules_count)
	mod_idx += 1
print " * Done with all modules (total commits read: %d)" % total_commits_read
print ""
print 80 * "-"
print ""

print "Commits with bug reference:"
count_commits(cbugs)
print
print 80 * "-"
print
print "Total commits:"
count_commits(ctotal)