From 0f3ef86bc1aa4de3d9f6f994754da76335ce05f9 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Fri, 24 Aug 2007 23:44:43 +0000 Subject: [PATCH] now depends search works like a charm and it's really solid. added depends <-> atoms relation table, might break some installs even if I handled this case too git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@461 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- client/equo | 2 + client/equoTools.py | 72 +++++++++++++++++++----- libraries/databaseTools.py | 101 ++++++++++++++++++---------------- libraries/entropyConstants.py | 1 + server/reagent | 2 +- 5 files changed, 114 insertions(+), 64 deletions(-) diff --git a/client/equo b/client/equo index 1b35a5b8f..d7c029fdc 100644 --- a/client/equo +++ b/client/equo @@ -68,6 +68,7 @@ def print_help(): print_info(" \t"+blue("database")+brown("\t handle installed packages database")) print_info(" \t\t"+darkgreen("generate")+red("\t\t generate installed packages database")) + print_info(" \t\t"+darkgreen("depends")+red("\t\t\t to regenerate/generate depends caching table")) print_info(" \t"+blue("cleanup")+brown("\t\t remove downloaded packages and clean temporary directories")) options = sys.argv[1:] @@ -161,6 +162,7 @@ except: print_error(darkred("Oh well, I cannot even write to /tmp. So, please copy the error and mail lxnay@sabayonlinux.org.")) sys.exit(1) print + print_error(darkred("!!! If the error is about missing tables in the database, just run 'equo database generate' again !!!")) print_error(blue("Ok, back here. Let me see if you are connected to the Internet. Yes, I am blue now, so?")) conntest = getOnlineContent("http://svn.sabayonlinux.org") if (conntest != False): diff --git a/client/equoTools.py b/client/equoTools.py index 83e50c4ac..8ba8eccb7 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -1171,7 +1171,7 @@ def getRequiredPackages(foundAtoms, emptydeps = False): @output: removal tree dictionary, plus status code ''' dictDeps = {} -def generateRemovalTree(idpackages, output = False): +def generateRemovalTree(idpackages, output = False, deep = False): clientDbconn = openClientDatabase() if (clientDbconn == -1): @@ -1185,8 +1185,7 @@ def generateRemovalTree(idpackages, output = False): # Initialize for idpackage in idpackages: atomIds.append(idpackage) - xatom = clientDbconn.retrieveAtom(idpackage) - xdepends = clientDbconn.searchDepends(dep_getkey(xatom)) + xdepends = clientDbconn.searchDepends(idpackage) dependencies.append(idpackage) dictDeps[idpackage] = set(xdepends) # even add my depends? @@ -1196,8 +1195,7 @@ def generateRemovalTree(idpackages, output = False): # if not, remove from treeview too upstairsDepends.append(x) dependencies.append(x) - xatom = clientDbconn.retrieveAtom(x) - xdep = clientDbconn.searchDepends(dep_getkey(xatom)) + xdep = clientDbconn.searchDepends(x) dictDeps[x] = set(xdep) remainingDeps = dependencies[:] @@ -1229,8 +1227,6 @@ def generateRemovalTree(idpackages, output = False): #print atx #print "--------" - - for depend in depends: if depend in treeview: #print "changed" @@ -1267,7 +1263,7 @@ def generateRemovalTree(idpackages, output = False): for x in xdeps: #print x xdep = atomMatchInRepository(x,clientDbconn) - mydepends = clientDbconn.searchDepends(dep_getkey(x)) + mydepends = clientDbconn.searchDepends(xdep[0]) dependencies.add(xdep[0]) dictDeps[xdep[0]] = set(mydepends) @@ -1368,7 +1364,7 @@ def compareVersions(listA,listB): ''' def checkNeededDownload(filepath,checksum = None): # is the file available - if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath) and os.path.isfile(etpConst['entropyworkdir']+"/"+filepath+etpConst['packageshashfileext']): + if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath): if checksum is None: return 0 else: @@ -1678,6 +1674,10 @@ def removePackageFromDatabase(idpackage, clientDbconn = None): clientDbconn = openClientDatabase() clientDbconn.removePackage(idpackage) + # also remove from dependstable + x = clientDbconn.removePackageFromDependsTable(idpackage) + if (x == 1): #`shit, needs regeneration + regenerateDependsTable(clientDbconn) if (closedb): clientDbconn.closeDB() @@ -1891,10 +1891,19 @@ def database(options): # now add the package clientDbconn.addPackageToInstalledTable(x[0],x[1]) + print_info(red(" Now generating depends caching table...")) + regenerateDependsTable(clientDbconn) print_info(red(" Database reinitialized successfully.")) clientDbconn.closeDB() + elif (options[0] == "depends"): + print_info(red(" Regenerating depends caching table...")) + clientDbconn = openClientDatabase() + regenerateDependsTable(clientDbconn) + clientDbconn.closeDB() + print_info(red(" Depends caching table regenerated successfully.")) + def printPackageInfo(idpackage,dbconn, clientSearch = False, strictOutput = False, quiet = False): # now fetch essential info @@ -2118,13 +2127,18 @@ def searchDepends(atoms, idreturn = False, verbose = False, quiet = False): clientDbconn = openClientDatabase() dataInfo = [] # when idreturn is True for atom in atoms: - result = clientDbconn.searchDepends(atom) - if (result): + result = atomMatchInRepository(atom,clientDbconn) + if (result[0] != -1): + searchResults = clientDbconn.searchDepends(result[0]) + if searchResults == -2: + # I need to generate dependstable + regenerateDependsTable(clientDbconn) + searchResults = clientDbconn.searchDepends(result[0]) # print info if (not idreturn) and (not quiet): print_info(blue(" Keyword: ")+bold("\t"+atom)) - print_info(blue(" Found: ")+bold("\t"+str(len(result)))+red(" entries")) - for idpackage in result: + print_info(blue(" Found: ")+bold("\t"+str(len(searchResults)))+red(" entries")) + for idpackage in searchResults: if (idreturn): dataInfo.append(idpackage) else: @@ -2278,6 +2292,26 @@ def searchDescription(descriptions, idreturn = False, quiet = False): return 0 +''' + @description: recreate dependstable table in the client database, it's used for caching searchDepends requests + @input Nothing + @output: Nothing +''' +def regenerateDependsTable(dbconn,output = True): + dbconn.createDependsTable() + depends = dbconn.listAllDependencies() + count = 0 + total = str(len(depends)) + for depend in depends: + count += 1 + atom = depend[1] + iddep = depend[0] + if output: + print_info(" "+bold("(")+darkgreen(str(count))+"/"+blue(total)+bold(")")+red(" Resolving ")+bold(atom), back = True) + match = atomMatchInRepository(atom,dbconn) + if (match[0] != -1): + dbconn.addDependRelationToDependsTable(iddep,match[0]) + ''' @description: open the repository database and returns the pointer @input repositoryName: name of the client database @@ -2556,13 +2590,14 @@ def installPackages(packages, ask = False, pretend = False, verbose = False, dep # running tasks totalqueue = str(len(runQueue)) currentqueue = 0 + clientDbconn = openClientDatabase() + for packageInfo in runQueue: currentqueue += 1 idpackage = packageInfo[0] repository = packageInfo[1] # get package atom dbconn = openRepositoryDatabase(repository) - clientDbconn = openClientDatabase() pkgatom = dbconn.retrieveAtom(idpackage) dbconn.closeDB() #print actionQueue[pkgatom] @@ -2591,7 +2626,14 @@ def installPackages(packages, ask = False, pretend = False, verbose = False, dep if (rc != 0): clientDbconn.closeDB() return -1,rc - clientDbconn.closeDB() + + # regenerate depends table + print_info(red(" @@ ")+blue("Regenerating depends caching table..."), back = True) + regenerateDependsTable(clientDbconn, output = False) + print_info(red(" @@ ")+blue("Install Complete.")) + + clientDbconn.closeDB() + return 0,0 diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 6994d01e5..b14f50fbe 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -669,7 +669,6 @@ def database(options): ##################################################################################### # this class simply describes the current database status - class databaseStatus: def __init__(self): @@ -736,10 +735,10 @@ class etpDatabase: self.readOnly = readOnly self.noUpload = noUpload self.packagesRemoved = False + self.packagesAdded = False self.clientDatabase = clientDatabase # caching dictionaries - self.dependCache = {} if (self.clientDatabase): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened by Entropy client, file: "+str(dbFile)) @@ -1199,6 +1198,7 @@ class etpDatabase: ) ) + self.packagesAdded = True self.commitChanges() return idpackage, revision, etpData, True @@ -1351,7 +1351,6 @@ class etpDatabase: # get info about inserted value and return prt = self.isProtectAvailable(protect) if prt != -1: - self.commitChanges() return prt raise Exception, "I tried to insert a protect but then, fetching it returned -1. There's something broken." @@ -1364,7 +1363,6 @@ class etpDatabase: # get info about inserted value and return src = self.isSourceAvailable(source) if src != -1: - self.commitChanges() return src raise Exception, "I tried to insert a source but then, fetching it returned -1. There's something broken." @@ -1377,7 +1375,6 @@ class etpDatabase: # get info about inserted value and return dep = self.isDependencyAvailable(dependency) if dep != -1: - self.commitChanges() return dep raise Exception, "I tried to insert a dependency but then, fetching it returned -1. There's something broken." @@ -1390,7 +1387,6 @@ class etpDatabase: # get info about inserted value and return key = self.isKeywordAvailable(keyword) if key != -1: - self.commitChanges() return key raise Exception, "I tried to insert a keyword but then, fetching it returned -1. There's something broken." @@ -1403,7 +1399,6 @@ class etpDatabase: # get info about inserted value and return use = self.isUseflagAvailable(useflag) if use != -1: - self.commitChanges() return use raise Exception, "I tried to insert a useflag but then, fetching it returned -1. There's something broken." @@ -1416,7 +1411,6 @@ class etpDatabase: # get info about inserted value and return lic = self.isLicenseAvailable(license) if lic != -1: - self.commitChanges() return lic raise Exception, "I tried to insert a license but then, fetching it returned -1. There's something broken." @@ -1430,7 +1424,6 @@ class etpDatabase: # get info about inserted value and return idflag = self.areCompileFlagsAvailable(chost,cflags,cxxflags) if idflag != -1: - self.commitChanges() return idflag raise Exception, "I tried to insert a flag tuple but then, fetching it returned -1. There's something broken." @@ -2094,43 +2087,6 @@ class etpDatabase: result.append(row[0]) return result - def searchDepends(self, atomkey): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDepends: called for "+atomkey) - - cached = self.dependCache.get(atomkey) - if cached: - return cached - - iddeps = [] - self.cursor.execute('SELECT iddependency,dependency FROM dependenciesreference WHERE dependency LIKE "%'+atomkey+'%"') # FIXME: handle slotted packages, even if it's not a big issue atm - for row in self.cursor: - iddeps.append(row) - try: - atomkey = entropyTools.dep_getkey(atomkey) - atomcat = atomkey.split("/")[0] - atomname = atomkey.split("/")[1] - if iddeps: - _iddeps = iddeps[:] - for x in _iddeps: - if x: - mykey = entropyTools.dep_getkey(x[1]) - mycat = mykey.split("/")[0] - myname = mykey.split("/")[1] - if mykey.find("|or|") == -1 and mykey.find("|and|") == -1: # maybe find a better way - if (mycat != atomcat) or (myname != atomname): - iddeps.remove(x) - except: - pass - result = [] - for iddep in iddeps: - #print iddep - self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep[0])+'"') - for row in self.cursor: - result.append(row[0]) - # caching - self.dependCache[atomkey] = result - return result - def searchPackages(self, keyword, sensitive = False): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackages: called for "+keyword) result = [] @@ -2458,5 +2414,54 @@ class etpDatabase: def removePackageFromInstalledTable(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"removePackageFromInstalledTable: called for "+str(idpackage)) - self.cursor.execute('DELETE FROM installedtable WHERE idpackage = '+str(idpackage)) - \ No newline at end of file + try: + self.cursor.execute('DELETE FROM installedtable WHERE idpackage = '+str(idpackage)) + except: + self.createInstalledTable() + + def removePackageFromDependsTable(self, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"removePackageFromDependsTable: called for "+str(idpackage)) + try: + self.cursor.execute('DELETE FROM dependstable WHERE idpackage = '+idpackage) + return 0 + except: + return 1 # need reinit + + def searchDepends(self, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDepends: called for idpackage "+str(idpackage)) + iddeps = [] + try: + self.cursor.execute('SELECT iddependency FROM dependstable WHERE idpackage = "'+str(idpackage)+'"') + except: + return -2 # table does not exist, please regenerate and re-run + for row in self.cursor: + iddeps.append(row[0]) + result = [] + for iddep in iddeps: + #print iddep + self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep)+'"') + for row in self.cursor: + result.append(row[0]) + return result + + # temporary/compat functions + def createDependsTable(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createDependsTable: called.") + self.cursor.execute('DROP TABLE IF EXISTS dependstable;') + self.cursor.execute('CREATE TABLE dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER );') + + def createInstalledTable(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createInstalledTable: called.") + self.cursor.execute('DROP TABLE IF EXISTS installedtable;') + self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER, repositoryname VARCHAR );') + + def addDependRelationToDependsTable(self, iddependency, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addDependRelationToDependsTable: called for iddependency "+str(iddependency)+" and idpackage "+str(idpackage)) + # FIXME: should check for existance of the PK? + self.cursor.execute( + 'INSERT into dependstable VALUES ' + '(?,?)' + , ( iddependency, + idpackage, + ) + ) \ No newline at end of file diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 8ac5cf2c9..2f7fc669a 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -92,6 +92,7 @@ DROP TABLE IF EXISTS configprotect; DROP TABLE IF EXISTS configprotectmask; DROP TABLE IF EXISTS configprotectreference; DROP TABLE IF EXISTS installedtable; +DROP TABLE IF EXISTS dependstable; """ etpSQLInit = """ diff --git a/server/reagent b/server/reagent index c7a54ecea..c163b09ca 100644 --- a/server/reagent +++ b/server/reagent @@ -40,7 +40,7 @@ def print_help(): print_info(" --help\t\tthis output") print_info(" --version\t\tprint version") print_info(" --nocolor\t\tdisable colorized output") - print_info(entropyTools.blue("Tools available: ")) + print_info(blue("Tools available: ")) print_info(" \t"+green("enzyme")+yellow("\t\t to analyze enzyme store directory")) print_info(" \t\t"+red("--force-bump")+"\t\t\t\t to force the revision bumping of the package entries") print_info(" \t\t"+red("--branch=[stable,unstable]")+"\t\t to force a specific branch")