more updates and ETP API version bump, since now we manage download URIs dynamically

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@175 cd1c1023-2f26-0410-ae45-c471fc1f0318

TODO

@@ -1,6 +1,7 @@
 TODO list (for developers only):
 - add a wrapper for qfile and qlist portage-utils tools because our internal repository is at /var/lib/entropy/portage
 - add proper SLOTS, FETCH_RESTRICTION support to enzyme, getBestAtom and so on... (maybe directly using emerge output?)
+- do not append download URLs directly into the .etp files
 
 Project Status:
 - entropy: not yet started, nothing to say then
@@ -8,4 +9,8 @@ Project Status:
 - reagent: first release up and working.
 - activator: not yet started, I need to prepare a ftp module that let us work without rsync and shit like that
 ============
-- client part: not yet defined apart from equilibrium name.
+- client part: not yet defined apart from equilibrium name.
+
+
+Features plan:
+- distcc support to enzyme

handlers/enzyme

@@ -44,8 +44,8 @@ def print_help():
     entropyTools.print_info(entropyTools.blue("Tools available: "))
     entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("world"))+entropyTools.yellow("\t\t to build all the possible new packages"))
     entropyTools.print_info(" \t\t"+entropyTools.red("--empty-tree")+"\t\t rebuild all, including updates")
-    entropyTools.print_info(" \t\t"+entropyTools.red("--deep")+"\t\t analyzes world dependencies deeply")
-    entropyTools.print_info(" \t\t"+entropyTools.red("--repackage-world")+"\t\t creates binaries of all the installed packages")
+    entropyTools.print_info(" \t\t"+entropyTools.red("--deep")+"\t\t\t analyzes world dependencies deeply")
+    entropyTools.print_info(" \t\t"+entropyTools.red("--repackage-installed")+"\t creates binaries of all the installed packages")
     entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("build"))+entropyTools.yellow("\t\t to build all the packages specified in <atom(s)>"))
     entropyTools.print_info(" \t\t"+entropyTools.red("--force-rebuild")+"\t\t force the building of the package, nevertheless")
     entropyTools.print_info(" \t\t"+entropyTools.red("--force-repackage")+"\t force the repackaging of all the possible package")

libraries/entropyConstants.py

@@ -59,8 +59,8 @@ etpData = {
 # the ARCHs that we support
 ETP_ARCHS = ["x86", "amd64"] # maybe ppc someday
 ETP_API_MAJOR = "1"
-ETP_API_MINOR = "2"
-ETP_API_SUBLEVEL = "2"
+ETP_API_MINOR = "3"
+ETP_API_SUBLEVEL = "0"
 ETP_API = ETP_API_MAJOR+"."+ETP_API_MINOR+"."+ETP_API_SUBLEVEL
 ETP_ARCH_CONST = "%ARCH%"
 ETP_REVISION_CONST = "%ETPREV%"
@@ -97,6 +97,7 @@ etpConst = {
     'enzymeconf': ETP_CONF_DIR+"/enzyme.conf", # enzyme.conf file
     'digestfile': "Manifest", # file that contains md5 hashes
     'extension': ".etp", # entropy files extension
+    'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path where we'll have to append the packages|uri part.
 }
 
 # Create paths
@@ -247,6 +248,7 @@ dbOR = "|or|"
 dbKEYWORDS = "KEYWORDS"
 dbCONTENTS = "CONTENTS"
 
+
 # Portage variables reference
 # vdbVARIABLE --> $VARIABLE
 vdbPORTDIR = "PORTDIR"

libraries/entropyTools.py

@@ -514,6 +514,27 @@ def synthetizeRoughDependencies(roughDependencies, useflags = None):
 
     return dependencies, conflicts
 
+# Collect installed packages
+def getInstalledPackages():
+    import os
+    appDbDir = getPortageAppDbPath()
+    dbDirs = os.listdir(appDbDir)
+    installedAtoms = []
+    for pkgsdir in dbDirs:
+	pkgdir = os.listdir(appDbDir+pkgsdir)
+	for pdir in pkgdir:
+	    pkgcat = pkgsdir.split("/")[len(pkgsdir.split("/"))-1]
+	    pkgatom = pkgcat+"/"+pdir
+	    if pkgatom.find("-MERGING-") == -1:
+	        installedAtoms.append(pkgatom)
+    return installedAtoms, len(installedAtoms)
+
+def getPortageAppDbPath():
+    rc = getPortageEnv("ROOT")+portage_const.VDB_PATH
+    if (not rc.endswith("/")):
+	return rc+"/"
+    return rc
+
 # get a list, returns a sorted list
 def alphaSorter(seq):
     def stripter(s, goodchrs):
@@ -558,7 +579,7 @@ def print_error(msg):
     print red(">>")+" "+msg
 
 def print_info(msg, back = False):
-    writechar("\r                                                                            \r")
+    writechar("\r                                                                                                \r")
     if (back):
 	writechar("\r"+green(">>")+" "+msg)
 	return

libraries/enzymeTools.py

@@ -439,8 +439,6 @@ def world(options):
 
     myopts = options[1:]
 
-    print "building world :P"
-
     enzymeRequestDeep = False
     enzymeRequestVerbose = False
     enzymeRequestRebuild = False
@@ -450,7 +448,7 @@ def world(options):
 	    enzymeRequestVerbose = True
 	elif ( i == "--empty-tree" ):
 	    enzymeRequestRebuild = True
-	elif ( i == "--repackage-world" ):
+	elif ( i == "--repackage-installed" ):
 	    enzymeRequestJustRepackageWorld = True
 	elif ( i == "--deep" ):
 	    enzymeRequestDeep = True
@@ -463,6 +461,30 @@ def world(options):
     etpConst['packagesstoredir'] = translateArch(etpConst['packagesstoredir'],getPortageEnv('CHOST'))
     etpConst['packagesbindir'] = translateArch(etpConst['packagesbindir'],getPortageEnv('CHOST'))
 
+    if (enzymeRequestJustRepackageWorld):
+	# create the list of installed packages
+	print_info(green(" * ")+red("Scanning system database..."),back = True)
+	installedPackages, pkgsnumber = getInstalledPackages()
+	print_info(green(" * ")+red("System database: ")+bold(str(pkgsnumber))+red(" installed packages"))
+	if pkgsnumber > 0:
+	    print_info(green(" * ")+red("Starting to build binaries..."))
+	else:
+	    print_error(red(" * ")+red("No detected packages??? Are you serious?"))
+	    sys.exit(301)
+	
+	localcount = 0
+	for pkg in installedPackages:
+	    localcount += 1
+	    print_info("   "+red("(")+green(str(localcount))+yellow("/")+blue(str(pkgsnumber))+red(")")+red(" Compressing... ")+bold(pkg),back = True)
+	    # FIXME: Check if the package is already available in the Store dir before building it again?
+	    rc = quickpkg(pkg,etpConst['packagesstoredir'])
+	    if (rc is None):
+		print_warning(red(" * ")+yellow(" quickpkg problem for ")+red(pkg))
+		# FIXME: remove sys.exit then...
+		sys.exit(302)
+	writechar("\n")
+	print_info(green(" * ")+red("All packages have been generates successfully"))
+        return 0
     print "ok... now?"
 
 def overlay(options):

libraries/reagentTools.py

@@ -118,6 +118,8 @@ def enzyme():
 	sys.exit(0)
 
     counter = 0
+    etpCreated = 0
+    etpNotCreated = 0
     for tbz2 in tbz2files:
 	counter += 1
 	tbz2name = tbz2.split("/")[len(tbz2.split("/"))-1]
@@ -125,10 +127,14 @@ def enzyme():
 	tbz2path = etpConst['packagesstoredir']+"/"+tbz2
 	rc = generator(tbz2path)
 	if (rc):
+	    etpCreated += 1
 	    os.system("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/ -f")
 	else:
+	    etpNotCreated += 1
 	    os.system("rm -rf "+tbz2path)
 
+    print_info(green(" * ")+red("Statistics: ")+blue("etp created: ")+bold(str(etpCreated))+yellow(" - ")+darkblue("etp discarded: ")+bold(str(etpNotCreated)))
+
 # This function extracts all the info from a .tbz2 file and returns them
 def extractPkgData(package):
 
@@ -193,13 +199,8 @@ def extractPkgData(package):
         etpData['homepage'] = ""
 
     print_info(yellow(" * ")+red("Getting package download URL..."),back = True)
-    # Fill url
-    for i in etpSources['packagesuri']:
-        etpData['download'] += translateArch(i+etpData['name']+"-"+etpData['version']+".tbz2",etpData['chost'])+" "
-    if (not etpData['download']):
-        print_error("no 'packages|<uri>' specified in "+etpConst['repositoriesconf'])
-	sys.exit(101)
-    etpData['download'] = removeSpaceAtTheEnd(etpData['download'])
+    # Fill download relative URI
+    etpData['download'] = translateArch(etpConst['binaryurirelativepath']+etpData['name']+"-"+etpData['version']+".tbz2")
 
     print_info(yellow(" * ")+red("Getting package category..."),back = True)
     # Fill category