#!/usr/bin/python

# Minas Gjoka

import urllib,re, sys, time, random, datetime, time, signal, urllib2, cookielib, httplib, gzip, glob, os, csv
import Gnuplot, Gnuplot.funcutils
from scipy import *

try:
	filename = sys.argv[1]
except:
	print 'Usage:', sys.argv[0], 'filename-withlistofprofiles'
	sys.exit(1)
	


#f_notinclude = open("/home/minas/workspace/adonomics/facebook-apps","r")
f_notinclude = open("/home/minas/facebook-apps","r")
excluded_apps = {}
comment_RegExp = re.compile('^#')
for line in f_notinclude:
	if not comment_RegExp.search(line.strip()):
		fields=line.split()
		excluded_apps[ int(fields[0]) ] = 1

entries = 0
validprofileRegExp = re.compile("<div class=\"container\" id=\"(\d+)\">")

appfinder1_RegExp = re.compile('<a class="app_icon" href="#a_(\d+)"')
appfinder2_RegExp = re.compile("/ajax/ct.php\?app_id=(\d+)")

fin = open(filename,"r")
list_files = fin.readlines()
fin.close()
print  "Number of files:" + str(len(list_files))
apps_uid = {}
users_appid = {}
profiles_skipped = 0
for line in list_files:
	item = line.strip()
	fields = os.path.basename(item).split("-")[1].split(".")
	uid = int(fields[0])

	if (os.path.splitext(item)[1] == ".gz"):
		try :
			file = gzip.open(item, 'rb',1)
		except:
			print 'Error  opening file %s .Skipping ' % (item) 
			continue
	else:
		file = open(item, "r")
        #print "Processing " + item
	
	htmlSource = file.read()
	
	n_apps =  len(validprofileRegExp.findall(htmlSource))
	if n_apps == 0:
		print "Invalid profile found"
		continue
	
	
	hash = {}
	iterator1 = appfinder1_RegExp.finditer(htmlSource)
	for match in iterator1:
		appid = int(match.group(1))
		hash[ appid ] = 1
	
	if len( hash.keys() )==0:
		#print "Profile %d skipped" % (uid)
		profiles_skipped = profiles_skipped + 1
		continue
	
	iterator2 = appfinder2_RegExp.finditer(htmlSource)
	
	for match in iterator2:
		appid = int(match.group(1))
		hash[ appid ] = 1
		
	for appid in hash.keys():
		if appid not in excluded_apps:
			list = apps_uid.get( uid, [])
			list.append(appid)
			apps_uid[ uid ] = list
			
			list = users_appid.get( appid, [])
			list.append(uid)
			users_appid[ appid ] = list
				
		
	entries = entries + 1
	if (entries % 500 == 0):
		print "(Skipped,TotalProcessed)=(%d,%d) profiles" % (profiles_skipped,entries)
	

fapps_uid = open("apps_uid","w")
for uid in apps_uid.keys():
	fapps_uid.write('%d ' % uid)

	list = apps_uid.get( uid, [])
	for appid in list:
		fapps_uid.write('%d ' % appid)
	fapps_uid.write('\n')
fapps_uid.close()

fusers_appid = open("users_appid","w")
for appid in users_appid.keys():
	fusers_appid.write('%d ' % appid)

	list = users_appid.get( appid, [])
	for uid in list:
		fusers_appid.write('%d ' %  uid)
	fusers_appid.write('\n')
fusers_appid.close()