#!/usr/bin/env python

#
# GetAvailable
#  Get available packages, recipes and tracked versions.
#
# (C) 2004 Andre Detsch. Released under the GNU GPL.
#

import os.path, sys, string, time, os, threading

from PythonUtils import *

goboTemp = getGoboVariable('goboTemp')
if not goboTemp:
	Log_Error("Unable to get $goboTemp value. Aborting.")
	sys.exit(1)

cacheDir = os.path.expanduser(goboTemp+'/Scripts-'+os.getenv('USER')+'/cache/')

def loadConfig():
	goboSettings = getGoboVariable('goboSettings')

	tr = {}
	for i in ['timeout']:
		try:
			tr[i] = getGoboVariable(i, 'Scripts/GetAvailable.conf')
		except:
			tr[i] = 0

	for i in ['defaultLocalPackagesPaths']:
		try:
			tr[i] = getGoboVariable(i, 'Scripts/GetAvailable.conf', True)
		except:
			tr[i] = []

	for listType in ['contribPackagesLists', 'officialPackagesLists', 'trackedVersionsLists']:
		try:
			tr[listType] = getGoboVariable(listType, 'Scripts/GetAvailable.conf', True)
			for j in range(len(tr[listType])):
				if '.' in tr[listType][j].split('/')[-1]:
					extension = '.' + tr[listType][j].split('/')[-1].split('.')[-1]
				else:
					extension = ''
				tr[listType][j] = (tr[listType][j], listType+str(j)+extension)

		except:
			tr[listType] = []

	tr['defaultLocalPackagesPaths'] = map(os.path.abspath, tr['defaultLocalPackagesPaths'])
	tr['defaultLocalPackagesPaths'] = map(os.path.expanduser, tr['defaultLocalPackagesPaths'])

	return tr

globalSettings = loadConfig()

downloadedCount = 0

def downloadFiles(files, force=False, hook=None, message=''):
	global downloadedCount
	downloadedCount = 0
	def doDownload(url, localfile, timeout=10):
		has_wget = not bash('wget --help &> /dev/null', 'v')
		if has_wget:
			has_cert_support = bash('wget --help | grep "no-check-certificate"')
			if has_cert_support:
				wget_cmd = 'wget --no-check-certificate'
			else:
				wget_cmd = 'wget'
			cmd = '%s --timeout=%d --tries=1 --quiet %s -O "%s" && touch "%s" || rm -f "%s"'%(wget_cmd, timeout, url, localfile, localfile, localfile)
			bash(cmd)
		else:
			try:
				urllib.urlretrieve(url, localfile)
			except:
				sys.stderr.write('Timeout\n')

		global downloadedCount
		downloadedCount = downloadedCount + 1
		if hook:
			hook(message, downloadedCount, len(files), False)


	threads = []
	if files and not os.access(cacheDir, os.W_OK):
		os.makedirs(cacheDir)

	# Clean up old and empty files
	currentTime = time.time()
	ageLimit = 60*60 # an hour
	for filename in os.listdir(cacheDir):
		st = os.stat(cacheDir+filename)
		if st.st_size == 0 or currentTime > (ageLimit + st.st_mtime):
			os.unlink(cacheDir+filename)

	try:
		timeout = int(globalSettings['timeout'])
	except:
		sys.stderr.write('Invalid timeout value "%s"\n'%globalSettings['timeout'])
		timeout = 10

	for url, filename in files:
		if force or not os.access(cacheDir+filename, os.R_OK): # or time.time() > (60*60 + os.stat(cacheDir+filename).st_mtime):
			import urllib
			t = threading.Thread(target=doDownload, args=(url, cacheDir+filename, timeout))
			t.setDaemon(True) # program exits on long timeout
			t.start()
			threads.append(t)
 		else:
			downloadedCount = downloadedCount + 1

	if threads and hook:
		hook(message, downloadedCount, len(files))

	begin = time.time()
	try:
		for thread in threads:
			loopStartDownloadedCount = downloadedCount
			while downloadedCount < len(files) and loopStartDownloadedCount == downloadedCount:
				waitTime = 0.05
				if hook:
					aborted = hook(message, downloadedCount, len(files), False)
					if aborted:
						return downloadedCount >= len(threads)

				#if (timeout + begin) > time.time():
				thread.join(waitTime)
				#else:
				#	sys.stderr.write('<Timeout>')
				#	if not hook:
				#		sys.stderr.write('\n')
				#	return downloadedCount >= len(threads)
	except KeyboardInterrupt:
		sys.stderr.write('<Interrupted>')
		if not hook:
			sys.stderr.write('\n')

	# returns false if a timeout has occured
	return downloadedCount >= len(threads)


def UpdateCache(types, force=True, hook=None):
	files = []

	if 'official_package' in types:
		for url, file in globalSettings['officialPackagesLists']:
			files.append((url, file))

	if 'contrib_package' in types:
		for url, file in globalSettings['contribPackagesLists']:
			files.append((url, file))

	if 'recipe' in types:
		compileRecipeDirs, getRecipeStores = getCompileOptions()
		i=0
		for getRecipeStore in getRecipeStores:
			files.append((getRecipeStore+'/MANIFEST.bz2', 'RecipeList'+str(i)+'.bz2'))
			i = i + 1

	if 'tracked' in types:
		for url, file in globalSettings['trackedVersionsLists']:
			files.append((url, file))

	if not files:
		return

	ret = downloadFiles(files, force, hook, 'Downloading updated remote lists')
	return ret

def addToDict(d, p, v, r, u):
	if not d.has_key(p):
		d[p] = {}
	if not d[p].has_key(v):
		d[p][v] = {}
	if not d[p][v].has_key(r):
		d[p][v][r] = []
	d[p][v][r].append(u)

def getUnpackedLocalPackages(directory, requireCurrent=False, d=None, default_revision='', current_only=False):
	import os, os.path

	if d is None:
		d = KeyInsensitiveDict();

	if not os.path.isdir(directory):
		return d

	programs = os.listdir(directory)

	for i in range(len(programs)):
		p = programs[i]
		if not os.path.isdir(directory+'/'+p):
			continue

		if current_only:
			if not os.path.isdir(directory+'/'+p+'/'+'Current'):
				continue

			try:
				v = os.readlink(directory+'/'+p+'/'+'Current').split('/')[-1]
			except:
				continue

			u = directory+'/'+p+'/'+v
			if os.access(u+'/'+'Resources/Revision', os.R_OK):
				f = open(u+'/'+'Resources/Revision')
				r = f.read().strip()
				f.close()
			else:
				r = default_revision
			addToDict(d, p,v,r,u)
			continue

		try:
			ds = os.listdir(directory+'/'+p)
		except:
			continue

		if requireCurrent and not 'Current' in ds:
			continue

		versions = [ v for v in ds if v not in ['Current', 'Settings', 'Variable', 'Headers'] ]
		versions = [ v for v in versions if os.path.isdir(directory+'/'+p+'/'+v) ]

		# Verify: what to do with an empty program entry?
		if not versions:
			continue

		for v in versions:
			# detsch, 12/11/2004: quick fix -> if not requireCurrent, assuming it is looking for recipes
			if not requireCurrent:
				v, r = Split_Version_Revision(v)
				u = directory+'/'+p+'/'+v
				if r:
					u += '-'+r
				else:
					r = default_revision
				if not os.access(u+'/'+'Recipe', os.R_OK):
					continue
			else:
				u = directory+'/'+p+'/'+v
				if os.access(u+'/'+'Resources/Revision', os.R_OK):
					f = open(u+'/'+'Resources/Revision')
					r = f.read().strip()
					f.close()
				else:
					r = default_revision
			addToDict(d, p,v,r,u)
	return d

def getPackedLocalPackages(directory, d = None):
	import os.path
	if directory and directory[-1] != '/':
		directory = directory + '/'

	if d == None:
		d = KeyInsensitiveDict()

	if not os.path.isdir(directory):
		return d

	files = os.listdir(directory)
	for file in files:
		l = file.split('--')
		if len(l) > 2:
			p = l[0]
			v,r = Split_Version_Revision(l[1])
			addToDict(d, p,v,r,directory+file)
	return d

def getVersionsFromList(file, d = None):
	if not os.access(file, os.R_OK):
		return d

	try:
		if file[-4:] == '.bz2':
			import bz2
			f = bz2.BZ2File(file)
		elif file[-3:] == '.gz':
			import gzip
			f = gzip.GzipFile(file)
		else:
			f = open(file)

		lines = f.readlines()
		f.close()

		if d == None:
			d = KeyInsensitiveDict()
	except Exception, e:
		Log_Terse("Cannot open %s - %s\nRemoving..."%(file,str(e)),"GetAvailable")
		os.remove(file)
		return d

	for line in lines:
		l = line.split()
		if len(l) >= 2:
			addToDict(d, l[0], l[1], '', str(l[0])+' '+str(l[1])+ ' ' + '<TRACKED>')
	return d

def getVersionsFromPackagesList(file, path_prefix = '', d = None):
	if not os.access(file, os.R_OK):
		return d

	if path_prefix and path_prefix[-1] != '/':
		path_prefix = path_prefix + '/'

	if d == None:
		d = KeyInsensitiveDict()

	try:
		if file[-4:] == '.bz2':
			import bz2
			f = bz2.BZ2File(file)
		elif file[-3:] == '.gz':
			import gzip
			f = gzip.GzipFile(file)
		else:
			f = open(file)

		lines = f.readlines()
		f.close()

		for line in lines:
			l = line.split('--')
			if len(l) > 2:
				p = l[0]
				v,r = Split_Version_Revision(l[1])
				addToDict(d, p,v,r, path_prefix+line.strip())
		return d
	except Exception, e:
		Log_Terse("Cannot open %s - %s\nRemoving...."%(file,str(e)),"GetAvailable")
		os.remove(file)
		return d


def GetAvailable(types, localdirs=None, forceupdate=False, hook=None, availables=None, accessWeb=True, doNotUpdate=False, goboPrograms=None):
	if 'recipe' in types:
		compileRecipeDirs, getRecipeStores = getCompileOptions()

	if availables == None:
		availables = {}

	if 'installed' in types:
		if goboPrograms:
			path= goboPrograms
		else:
			path = getGoboVariable('goboPrograms')
		if not availables.has_key('installed') or not availables['installed']:
			availables['installed'] = {}
		if not availables['installed'].has_key('programs') or not availables['installed']['programs']:
			availables['installed']['programs'] = KeyInsensitiveDict()
		getUnpackedLocalPackages(path, True, availables['installed']['programs'], default_revision= 'r1')

	if 'current' in types:
		if goboPrograms:
			path= goboPrograms
		else:
			path = getGoboVariable('goboPrograms')
		if not availables.has_key('current') or not availables['current']:
			availables['current'] = {}
		if not availables['current'].has_key('programs') or not availables['current']['programs']:
			availables['current']['programs'] = KeyInsensitiveDict()
		getUnpackedLocalPackages(path, True, availables['current']['programs'], default_revision= 'r1', current_only=True)

	if 'local_package' in types:
		if not availables.has_key('local_package') or not availables['local_package']:
			availables['local_package'] = {}

		if not availables['local_package'].has_key('programs') or not availables['local_package']['programs']:
			availables['local_package']['programs'] = KeyInsensitiveDict()

		if not localdirs:
			localdirs = globalSettings['defaultLocalPackagesPaths']

		import os
		for directory in localdirs:
			getPackedLocalPackages(os.path.expanduser(directory.strip()), availables['local_package']['programs'])
			getUnpackedLocalPackages(os.path.expanduser(directory.strip()), True, availables['local_package']['programs'])

	if accessWeb and (not doNotUpdate or forceupdate):
		UpdateCache(types, forceupdate, hook)

	if hook:
		hook('Loading lists', 0, 4)


	if 'official_package' in types:
		if not availables.has_key('official_package') or not availables['official_package']:
			availables['official_package'] = {}
		availables['official_package']['programs']= KeyInsensitiveDict()
		for url, file in globalSettings['officialPackagesLists']:
			availables['official_package']['programs'] = getVersionsFromPackagesList(cacheDir+file, string.join(url.split('/')[:-1], '/'), availables['official_package']['programs'])

	if hook:
		hook('Loading lists', 1, 4)

	if 'contrib_package' in types:
		if not availables.has_key('contrib_package') or not availables['contrib_package']:
			availables['contrib_package'] = {}

		availables['contrib_package']['programs']= KeyInsensitiveDict()
		for url, file in globalSettings['contribPackagesLists']:
			availables['contrib_package']['programs'] = getVersionsFromPackagesList(cacheDir+file, string.join(url.split('/')[:-1], '/'), availables['contrib_package']['programs'])

	if hook:
		hook('Loading lists', 2, 4)

	if 'recipe' in types:
		if not availables.has_key('recipe') or not availables['recipe']:
			availables['recipe'] = {}

		if not availables['recipe'].has_key('programs') or not availables['recipe']['programs']:
			availables['recipe']['programs'] = KeyInsensitiveDict()

		# searches local recipes first
		default_revision='r'+str(sys.maxint)
		for compileRecipeDir in compileRecipeDirs:
			availables['recipe']['programs'] = getPackedLocalPackages(compileRecipeDir, availables['recipe']['programs'])
			availables['recipe']['programs'] = getUnpackedLocalPackages(compileRecipeDir, False, availables['recipe']['programs'], default_revision=default_revision)
			default_revision = ''
		if accessWeb:
			i=0
			for getRecipeStore in getRecipeStores:
				availables['recipe']['programs'] = getVersionsFromPackagesList(cacheDir+'RecipeList'+str(i)+'.bz2', getRecipeStore, availables['recipe']['programs'])
				i = i + 1

	if hook:
		hook('Loading lists', 3, 4)

	if 'tracked' in types:
		if not availables.has_key('tracked') or not availables['tracked']:
			availables['tracked'] = {}
		availables['tracked']['programs'] = KeyInsensitiveDict()
		for url, file in globalSettings['trackedVersionsLists']:
			availables['tracked']['programs'] = getVersionsFromList(cacheDir+file, availables['tracked']['programs'])

	if hook:
		hook('Loading lists', 4, 4)

	return availables


#############################################################################
#  'main()'
#############################################################################

if __name__ == '__main__':
	import sys
	import getopt, os

	try:
		opts, args = getopt.getopt(sys.argv[1:], 't:Whp:', ['type=', 'types=', 'local-dirs=', 'full-list', 'force-update', 'no-web', 'gobo-programs=', 'help'])
	except getopt.GetoptError, detail:
		print sys.argv[0].split('/')[-1]+': '+str(detail)
		sys.exit(1)

	validTypes = ['local_package', 'official_package', 'recipe', 'contrib_package', 'tracked', 'installed', 'current', 'all']
	types      = ['local_package', 'official_package']
	typesChanged = False
	localdirs = None
	forceupdate = False
	noWeb = False
	goboPrograms = None

	for o, a in opts:
		if o in ['--types','--type', '-t']:
			typesChanged = True
			types = a.split(',')
			for i in range(len(types)):
				if len (types[i]) == 1:
					for t in validTypes:
						if types[i][0] == t[0]:
							types[i] = t
							break
			if 'all' in types:
				types = validTypes[:-1] # don't include 'all'
		if o == '--local-dirs':
			localdirs = a.split(',')
		elif o == '--force-update':
			forceupdate = True
		elif o in ['--gobo-programs', '-p']:
			goboPrograms = a
		elif o in ['--no-web', '-W']:
			noWeb = True
			if not typesChanged:
				types = ['local_package']
		elif o in ['--help', '-h']:
			print """
GetAvailable
 Get available packages, recipes and tracked versions.


Options:
 -t [t1,t2,...]
 --types=[t1,t2,...]  sets what kind of packages can be searched, in the
                      passed order. Valid types are:
                       local_package, official_package, contrib_package,
                       installed, recipe, tracked, all
                      using only the first character from any of the above
                      is also valid:
                       l, o, c, i, r, t, a
                      Default types are:
                       local_package, official_package
                      notice that when "recipe" type is used, Compile.conf is
                      read to set recipe-store locations and local recipes
                      locations.

 --local-dirs=[d1,..] where to look for local binary packages. By default,
                      uses the paths defined at GetAvailable.conf

 --force-update       downloads required packages list even if there is a
                      local copy (cached in """+cacheDir+""") newer than one hour.

 -W, --no-web         do not try to download anything and don't lists
                      remote recipes and packages (if not explicitly listed
                      in '--types='). Overrides '--force-update'

 -p
 --gobo-programs      Override default """+getGoboVariable("goboPrograms")+""" as path of installed packages

Examples of usage:
 GetAvailable --types=recipe

 """
			sys.exit(0)


	hook = consoleProgressHook

	availables = GetAvailable(types, localdirs, forceupdate, accessWeb=not noWeb, hook=hook, goboPrograms=goboPrograms)

	for t in availables.keys():
		for p in caseinsensitive_sort(availables[t]['programs'].keys()):
			for v in availables[t]['programs'][p].keys():
				print t,p,v,availables[t]['programs'][p][v]

	#print availables