now depends search works like a charm and it's really solid. added depends <-> atoms relation table, might break some installs even if I handled this case too

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@461 cd1c1023-2f26-0410-ae45-c471fc1f0318

client/equo

@@ -68,6 +68,7 @@ def print_help():
 
     print_info(" \t"+blue("database")+brown("\t handle installed packages database"))
     print_info(" \t\t"+darkgreen("generate")+red("\t\t generate installed packages database"))
+    print_info(" \t\t"+darkgreen("depends")+red("\t\t\t to regenerate/generate depends caching table"))
     print_info(" \t"+blue("cleanup")+brown("\t\t remove downloaded packages and clean temporary directories"))
 
 options = sys.argv[1:]
@@ -161,6 +162,7 @@ except:
 	print_error(darkred("Oh well, I cannot even write to /tmp. So, please copy the error and mail lxnay@sabayonlinux.org."))
 	sys.exit(1)
     print
+    print_error(darkred("!!! If the error is about missing tables in the database, just run 'equo database generate' again !!!"))
     print_error(blue("Ok, back here. Let me see if you are connected to the Internet. Yes, I am blue now, so?"))
     conntest = getOnlineContent("http://svn.sabayonlinux.org")
     if (conntest != False):

client/equoTools.py

@@ -1171,7 +1171,7 @@ def getRequiredPackages(foundAtoms, emptydeps = False):
    @output: removal tree dictionary, plus status code
 '''
 dictDeps = {}
-def generateRemovalTree(idpackages, output = False):
+def generateRemovalTree(idpackages, output = False, deep = False):
 
     clientDbconn = openClientDatabase()
     if (clientDbconn == -1):
@@ -1185,8 +1185,7 @@ def generateRemovalTree(idpackages, output = False):
     # Initialize
     for idpackage in idpackages:
 	atomIds.append(idpackage)
-	xatom = clientDbconn.retrieveAtom(idpackage)
-	xdepends = clientDbconn.searchDepends(dep_getkey(xatom))
+	xdepends = clientDbconn.searchDepends(idpackage)
 	dependencies.append(idpackage)
 	dictDeps[idpackage] = set(xdepends)
 	# even add my depends?
@@ -1196,8 +1195,7 @@ def generateRemovalTree(idpackages, output = False):
 	    # if not, remove from treeview too
 	    upstairsDepends.append(x)
 	    dependencies.append(x)
-	    xatom = clientDbconn.retrieveAtom(x)
-	    xdep = clientDbconn.searchDepends(dep_getkey(xatom))
+	    xdep = clientDbconn.searchDepends(x)
 	    dictDeps[x] = set(xdep)
 
     remainingDeps = dependencies[:]
@@ -1229,8 +1227,6 @@ def generateRemovalTree(idpackages, output = False):
 		#print atx
 	    #print "--------"
 	    
-	   
-	    
 	    for depend in depends:
 		if depend in treeview:
 		    #print "changed"
@@ -1267,7 +1263,7 @@ def generateRemovalTree(idpackages, output = False):
 		for x in xdeps:
 		    #print x
 		    xdep = atomMatchInRepository(x,clientDbconn)
-		    mydepends = clientDbconn.searchDepends(dep_getkey(x))
+		    mydepends = clientDbconn.searchDepends(xdep[0])
 		    dependencies.add(xdep[0])
 		    dictDeps[xdep[0]] = set(mydepends)
 		    
@@ -1368,7 +1364,7 @@ def compareVersions(listA,listB):
 '''
 def checkNeededDownload(filepath,checksum = None):
     # is the file available
-    if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath) and os.path.isfile(etpConst['entropyworkdir']+"/"+filepath+etpConst['packageshashfileext']):
+    if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath):
 	if checksum is None:
 	    return 0
 	else:
@@ -1678,6 +1674,10 @@ def removePackageFromDatabase(idpackage, clientDbconn = None):
 	clientDbconn = openClientDatabase()
 
     clientDbconn.removePackage(idpackage)
+    # also remove from dependstable
+    x = clientDbconn.removePackageFromDependsTable(idpackage)
+    if (x == 1): #`shit, needs regeneration
+	regenerateDependsTable(clientDbconn)
     
     if (closedb):
         clientDbconn.closeDB()
@@ -1891,10 +1891,19 @@ def database(options):
 	    # now add the package
 	    clientDbconn.addPackageToInstalledTable(x[0],x[1])
 
+	print_info(red("  Now generating depends caching table..."))
+	regenerateDependsTable(clientDbconn)
 	print_info(red("  Database reinitialized successfully."))
 
 	clientDbconn.closeDB()
 
+    elif (options[0] == "depends"):
+	print_info(red("  Regenerating depends caching table..."))
+	clientDbconn = openClientDatabase()
+	regenerateDependsTable(clientDbconn)
+	clientDbconn.closeDB()
+	print_info(red("  Depends caching table regenerated successfully."))
+
 
 def printPackageInfo(idpackage,dbconn, clientSearch = False, strictOutput = False, quiet = False):
     # now fetch essential info
@@ -2118,13 +2127,18 @@ def searchDepends(atoms, idreturn = False, verbose = False, quiet = False):
     clientDbconn = openClientDatabase()
     dataInfo = [] # when idreturn is True
     for atom in atoms:
-	result = clientDbconn.searchDepends(atom)
-	if (result):
+	result = atomMatchInRepository(atom,clientDbconn)
+	if (result[0] != -1):
+	    searchResults = clientDbconn.searchDepends(result[0])
+	    if searchResults == -2:
+		# I need to generate dependstable
+		regenerateDependsTable(clientDbconn)
+	        searchResults = clientDbconn.searchDepends(result[0])
 	    # print info
 	    if (not idreturn) and (not quiet):
 	        print_info(blue("     Keyword: ")+bold("\t"+atom))
-	        print_info(blue("     Found:   ")+bold("\t"+str(len(result)))+red(" entries"))
-	    for idpackage in result:
+	        print_info(blue("     Found:   ")+bold("\t"+str(len(searchResults)))+red(" entries"))
+	    for idpackage in searchResults:
 		if (idreturn):
 		    dataInfo.append(idpackage)
 		else:
@@ -2278,6 +2292,26 @@ def searchDescription(descriptions, idreturn = False, quiet = False):
     return 0
 
 
+'''
+   @description: recreate dependstable table in the client database, it's used for caching searchDepends requests
+   @input Nothing
+   @output: Nothing
+'''
+def regenerateDependsTable(dbconn,output = True):
+    dbconn.createDependsTable()
+    depends = dbconn.listAllDependencies()
+    count = 0
+    total = str(len(depends))
+    for depend in depends:
+	count += 1
+	atom = depend[1]
+	iddep = depend[0]
+	if output:
+	    print_info("  "+bold("(")+darkgreen(str(count))+"/"+blue(total)+bold(")")+red(" Resolving ")+bold(atom), back = True)
+	match = atomMatchInRepository(atom,dbconn)
+	if (match[0] != -1):
+	    dbconn.addDependRelationToDependsTable(iddep,match[0])
+
 '''
    @description: open the repository database and returns the pointer
    @input repositoryName: name of the client database
@@ -2556,13 +2590,14 @@ def installPackages(packages, ask = False, pretend = False, verbose = False, dep
     # running tasks
     totalqueue = str(len(runQueue))
     currentqueue = 0
+    clientDbconn = openClientDatabase()
+    
     for packageInfo in runQueue:
 	currentqueue += 1
 	idpackage = packageInfo[0]
 	repository = packageInfo[1]
 	# get package atom
 	dbconn = openRepositoryDatabase(repository)
-	clientDbconn = openClientDatabase()
 	pkgatom = dbconn.retrieveAtom(idpackage)
 	dbconn.closeDB()
 	#print actionQueue[pkgatom]
@@ -2591,7 +2626,14 @@ def installPackages(packages, ask = False, pretend = False, verbose = False, dep
 	    if (rc != 0):
 		clientDbconn.closeDB()
 		return -1,rc
-	clientDbconn.closeDB()
+
+    # regenerate depends table
+    print_info(red(" @@ ")+blue("Regenerating depends caching table..."), back = True)
+    regenerateDependsTable(clientDbconn, output = False)
+    print_info(red(" @@ ")+blue("Install Complete."))
+
+    clientDbconn.closeDB()
+
     return 0,0
 
 

libraries/databaseTools.py

@@ -669,7 +669,6 @@ def database(options):
 #####################################################################################
 
 # this class simply describes the current database status
-
 class databaseStatus:
 
     def __init__(self):
@@ -736,10 +735,10 @@ class etpDatabase:
 	self.readOnly = readOnly
 	self.noUpload = noUpload
 	self.packagesRemoved = False
+	self.packagesAdded = False
 	self.clientDatabase = clientDatabase
 	
 	# caching dictionaries
-	self.dependCache = {}
 	
 	if (self.clientDatabase):
 	    dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened by Entropy client, file: "+str(dbFile))
@@ -1199,6 +1198,7 @@ class etpDatabase:
 			)
 	    )
 
+	self.packagesAdded = True
 	self.commitChanges()
 	
 	return idpackage, revision, etpData, True
@@ -1351,7 +1351,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	prt = self.isProtectAvailable(protect)
 	if prt != -1:
-	    self.commitChanges()
 	    return prt
 	raise Exception, "I tried to insert a protect but then, fetching it returned -1. There's something broken."
 
@@ -1364,7 +1363,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	src = self.isSourceAvailable(source)
 	if src != -1:
-	    self.commitChanges()
 	    return src
 	raise Exception, "I tried to insert a source but then, fetching it returned -1. There's something broken."
 
@@ -1377,7 +1375,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	dep = self.isDependencyAvailable(dependency)
 	if dep != -1:
-	    self.commitChanges()
 	    return dep
 	raise Exception, "I tried to insert a dependency but then, fetching it returned -1. There's something broken."
 
@@ -1390,7 +1387,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	key = self.isKeywordAvailable(keyword)
 	if key != -1:
-	    self.commitChanges()
 	    return key
 	raise Exception, "I tried to insert a keyword but then, fetching it returned -1. There's something broken."
 
@@ -1403,7 +1399,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	use = self.isUseflagAvailable(useflag)
 	if use != -1:
-	    self.commitChanges()
 	    return use
 	raise Exception, "I tried to insert a useflag but then, fetching it returned -1. There's something broken."
 
@@ -1416,7 +1411,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	lic = self.isLicenseAvailable(license)
 	if lic != -1:
-	    self.commitChanges()
 	    return lic
 	raise Exception, "I tried to insert a license but then, fetching it returned -1. There's something broken."
 
@@ -1430,7 +1424,6 @@ class etpDatabase:
 	# get info about inserted value and return
 	idflag = self.areCompileFlagsAvailable(chost,cflags,cxxflags)
 	if idflag != -1:
-	    self.commitChanges()
 	    return idflag
 	raise Exception, "I tried to insert a flag tuple but then, fetching it returned -1. There's something broken."
 
@@ -2094,43 +2087,6 @@ class etpDatabase:
 	    result.append(row[0])
 	return result
 
-    def searchDepends(self, atomkey):
-	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDepends: called for "+atomkey)
-	
-	cached = self.dependCache.get(atomkey)
-	if cached:
-	    return cached
-
-	iddeps = []
-	self.cursor.execute('SELECT iddependency,dependency FROM dependenciesreference WHERE dependency LIKE "%'+atomkey+'%"') # FIXME: handle slotted packages, even if it's not a big issue atm
-	for row in self.cursor:
-	    iddeps.append(row)
-	try:
-	    atomkey = entropyTools.dep_getkey(atomkey)
-	    atomcat = atomkey.split("/")[0]
-	    atomname = atomkey.split("/")[1]
-	    if iddeps:
-		_iddeps = iddeps[:]
-	        for x in _iddeps:
-		    if x:
-			mykey = entropyTools.dep_getkey(x[1])
-			mycat = mykey.split("/")[0]
-			myname = mykey.split("/")[1]
-			if mykey.find("|or|") == -1 and mykey.find("|and|") == -1: # maybe find a better way
-			    if (mycat != atomcat) or (myname != atomname):
-			        iddeps.remove(x)
-	except:
-	    pass
-	result = []
-	for iddep in iddeps:
-	    #print iddep
-	    self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep[0])+'"')
-	    for row in self.cursor:
-	        result.append(row[0])
-	# caching
-	self.dependCache[atomkey] = result
-	return result
-
     def searchPackages(self, keyword, sensitive = False):
 	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackages: called for "+keyword)
 	result = []
@@ -2458,5 +2414,54 @@ class etpDatabase:
 
     def removePackageFromInstalledTable(self, idpackage):
 	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"removePackageFromInstalledTable: called for "+str(idpackage))
-	self.cursor.execute('DELETE FROM installedtable WHERE idpackage = '+str(idpackage))
-	
+	try:
+	    self.cursor.execute('DELETE FROM installedtable WHERE idpackage = '+str(idpackage))
+	except:
+	    self.createInstalledTable()
+
+    def removePackageFromDependsTable(self, idpackage):
+	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"removePackageFromDependsTable: called for "+str(idpackage))
+	try:
+	    self.cursor.execute('DELETE FROM dependstable WHERE idpackage = '+idpackage)
+	    return 0
+	except:
+	    return 1 # need reinit
+
+    def searchDepends(self, idpackage):
+	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDepends: called for idpackage "+str(idpackage))
+	iddeps = []
+	try:
+	    self.cursor.execute('SELECT iddependency FROM dependstable WHERE idpackage = "'+str(idpackage)+'"')
+	except:
+	    return -2 # table does not exist, please regenerate and re-run
+	for row in self.cursor:
+	    iddeps.append(row[0])
+	result = []
+	for iddep in iddeps:
+	    #print iddep
+	    self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep)+'"')
+	    for row in self.cursor:
+	        result.append(row[0])
+	return result
+
+    # temporary/compat functions
+    def createDependsTable(self):
+	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createDependsTable: called.")
+	self.cursor.execute('DROP TABLE IF EXISTS dependstable;')
+	self.cursor.execute('CREATE TABLE dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER );')
+
+    def createInstalledTable(self):
+	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createInstalledTable: called.")
+	self.cursor.execute('DROP TABLE IF EXISTS installedtable;')
+	self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER, repositoryname VARCHAR );')
+
+    def addDependRelationToDependsTable(self, iddependency, idpackage):
+	dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addDependRelationToDependsTable: called for iddependency "+str(iddependency)+" and idpackage "+str(idpackage))
+	# FIXME: should check for existance of the PK?
+	self.cursor.execute(
+		'INSERT into dependstable VALUES '
+		'(?,?)'
+		, (	iddependency,
+			idpackage,
+			)
+	)

libraries/entropyConstants.py

@@ -92,6 +92,7 @@ DROP TABLE IF EXISTS configprotect;
 DROP TABLE IF EXISTS configprotectmask;
 DROP TABLE IF EXISTS configprotectreference;
 DROP TABLE IF EXISTS installedtable;
+DROP TABLE IF EXISTS dependstable;
 """
 
 etpSQLInit = """

server/reagent

@@ -40,7 +40,7 @@ def print_help():
     print_info(" --help\t\tthis output")
     print_info(" --version\t\tprint version")
     print_info(" --nocolor\t\tdisable colorized output")
-    print_info(entropyTools.blue("Tools available: "))
+    print_info(blue("Tools available: "))
     print_info(" \t"+green("enzyme")+yellow("\t\t to analyze enzyme store directory"))
     print_info(" \t\t"+red("--force-bump")+"\t\t\t\t to force the revision bumping of the package entries")
     print_info(" \t\t"+red("--branch=[stable,unstable]")+"\t\t to force a specific branch")