Repository visualization with Gource

As some of you already know, for the past 10 years I've been researching how to analyse, visualise and design collaborative processes and organizations. I started this research at Politecnico di Milano (MSc), then continued independently (openp2pdesign.org), and then started a doctorate at Aalto University (Doctor of Arts). The research is still going on, and the Fab Academy might be an interesting context for this as well.

When I attended the Fab Academy in 2012 (here the documentation, under Exercises > 20 BONUS), I created a video of its process with the Gource software:

At the moment, there are very few softwares for visualizing the process in a Git/Hg repository, and I will probably work on new visualization softwares soon, but it will take some time. Meanwhile, I used again Gource for visualizing the process in the repositories I worked with during this Fab Academy: Europe, and the Opendot + WeMake Fab Labs in Milan where I am the instructor. These videos were generated quite quickly, so they are not very refined at the moment, but they can give you an idea of what happened:

EU Repo

WeMake Repo

Opendot Repo

Instructions

On Linux, you can download and compile Gource. On Mac, I installed Gource with:

brew install gource

...and Avconv, for rendering the video, with:

brew install libav

You can then export a log file from the repo, in order to use that for rendering the video:

gource --output-custom-log file.log

The basic usage of Gource is without avatars. There are few scripts around that download avatars from gravatar.com, I modified them in two scripts: one that get avatars also from GitHub, and another that get avatars from a Mercurial repo:

	# -- coding: utf-8 --
	#
	# Based on:
	# https://code.google.com/p/gource/wiki/GravatarExample
	# https://gist.github.com/macagua/5c2f5e4e38df92aae7fe
	#
	# Usage with Gource: gource --user-image-dir .git/avatar/
	#
	# Get list of authors + email with hg log (todo)
	# hg log --template '{author}\n'
	#

	import os
	import requests
	import subprocess
	import hashlib


	def md5_hex(text):
	m = hashlib.md5()
	m.update(text.encode('ascii', errors='ignore'))
	return m.hexdigest()

	size = 90
	# Configure the path of the git project
	projectpath = "/Users/massimo/Documents/FabAcademy2015/europe"
	hgpath = os.path.join(projectpath, '.hg')
	output_dir = os.path.join(hgpath, 'avatar')

	# Create the folder for storing the images. It's in the .git folder, so it won't be tracked by git
	if not os.path.exists(output_dir):
	os.makedirs(output_dir)

	hglog = subprocess.check_output(["hg", "log", "--template","'{author}\n'"], cwd=projectpath)
	authors = set(hglog.decode('ascii', errors='ignore').splitlines())
	print(authors)
	print("")
	for author in authors:
	author = author.replace("''", "")
	if "<" in author:
	name, email = author.split('<')
	email = email.replace(">", "")
	print(name, email)
	if name[-1] == " ":
	name = name[:-1]
	else:
	name = author
	email = ""
	output_file = os.path.join(output_dir, name + '.png')
	if not os.path.exists(output_file):
	grav_url = "http://www.gravatar.com/avatar/" + md5_hex(email) + "?d=identicon&s=" + str(size)
	print(grav_url)
	print("")
	r = requests.get(grav_url)
	if r.ok:
	with open(output_file, 'wb') as img:
	img.write(r.content)

view raw GourceGravatarHG.py hosted with ❤ by GitHub

	# -- coding: utf-8 --
	#
	# Based on:
	# https://code.google.com/p/gource/wiki/GravatarExample
	# https://gist.github.com/macagua/5c2f5e4e38df92aae7fe
	#
	# Usage with Gource: gource --user-image-dir .git/avatar/
	#
	# Get list of authors + email with git log
	# git log --format='%aN\|%aE' \| sort -u
	#
	# Get list of authors + email with hg log (todo)
	# hg log --template 'author: {author}\n'
	#

	import requests
	import getpass
	import os
	import subprocess
	import hashlib
	from time import sleep
	import sys

	username = ""
	password = ""


	def md5_hex(text):
	m = hashlib.md5()
	m.update(text.encode('ascii', errors='ignore'))
	return m.hexdigest()


	def get_data(api_request):
	global username
	global password

	r = requests.get(api_request, auth=(username, password))
	data = r.json()

	if "message" in data.keys():
	# Countdown
	# http://stackoverflow.com/questions/3249524/print-in-one-line-dynamically-python
	for k in range(1, 60 * 15):
	remaining = 60 * 15 - k
	sys.stdout.write("\r%d seconds remaining " % remaining)
	sys.stdout.flush()
	sleep(1)
	sys.stdout.write("\n")
	# Another request
	r = requests.get(api_request, auth=(username, password))
	data = r.json()
	else:
	pass

	# Return data
	return data

	if __name__ == "__main__":
	global username
	global password

	# Clear screen
	os.system('cls' if os.name == 'nt' else 'clear')

	# Login to the GitHub API
	username = raw_input("Enter your GitHub username: ")
	password = getpass.getpass("Enter yor GitHub password: ")

	# Configure the path of the git project
	projectpath = "/Users/massimo/Documents/FabAcademy2015/FabAcademy2015-Opendot"
	gitpath = os.path.join(projectpath, '.git')
	output_dir = os.path.join(gitpath, 'avatar')

	# Create the folder for storing the images. It's in the .git folder, so it won't be tracked by git
	if not os.path.exists(output_dir):
	os.makedirs(output_dir)

	# Get the authors from the git log
	gitlog = subprocess.check_output(
	['git', 'log', '--pretty=format:%ae\|%an'], cwd=projectpath)
	authors = set(gitlog.decode('ascii', errors='ignore').splitlines())
	print ""
	print "USERS:"
	print(authors)

	# Check each author
	for author in authors:
	# Get e-mail and name from log
	email, name = author.split('\|')
	print ""
	print "Checking", name, email
	# Try to find the user on GitHub with the e-mail
	api_request = "https://api.github.com/search/users?utf8=%E2%9C%93&q=" + \
	email + "+in%3Aemail&type=Users"
	data = get_data(api_request)

	# Check if the user was found
	if "items" in data.keys():
	if len(data["items"]) == 1:
	url = data["items"][0]["avatar_url"]
	print "Avatar url:", url
	else:
	# Try to find the user on GitHub with the name
	api_request = "https://api.github.com/search/users?utf8=%E2%9C%93&q=" + \
	name + "+in%3Aname&type=Users"
	data = get_data(api_request)

	# Check if the user was found
	if "items" in data.keys():
	if len(data["items"]) == 1:
	url = data["items"][0]["avatar_url"]
	print "Avatar url:", url
	# Eventually try to find the user with Gravatar
	else:
	url = "http://www.gravatar.com/avatar/" + \
	md5_hex(email) + "?d=identicon&s=" + str(90)
	print "Avatar url:", url

	# Finally retrieve the image
	try:
	output_file = os.path.join(output_dir, name + '.png')
	if not os.path.exists(output_file):
	r = requests.get(url)
	if r.ok:
	with open(output_file, 'wb') as img:
	img.write(r.content)
	except:
	print "There was an error with", name, email

view raw GourceGravatarExample.py hosted with ❤ by GitHub

requests

view raw requeriments.txt hosted with ❤ by GitHub

There are many many options for controlling Gource, here's my setup for creating the video and converting at the same time:

gource 1280x720 --user-image-dir ../FabAcademy2015-WeMake/.git/avatar/ gource --seconds-per-day 1 --stop-at-end --highlight-users --multi-sampling --output-ppm-stream --output-framerate 30 --title "Fab Academy 2015 - WeMake (Milan)" --hide mouse,filenames,progress wemake.log -o - | avconv -y -r 30 -f image2pipe -vcodec ppm -i - -b 65536K wemake.mp4

The EU repo is too big: too many files and users, so I changed a bit the settings:

gource 1280x720 --user-image-dir ../europe/.hg/avatar/ gource --seconds-per-day 1 --stop-at-end --highlight-users --multi-sampling --output-ppm-stream --output-framerate 30 --title "Fab Academy 2015 - Europe" --hide dirnames,bloom,mouse,filenames,progress eu.log -o - | avconv -y -r 30 -f image2pipe -vcodec ppm -i - -b 65536K europe.mp4