From f3f709f0d97b2c8705345f43272d026bedf044f4 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Sun, 26 Aug 2007 20:45:30 +0000 Subject: [PATCH] some more work on dependencies and orphans function git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@463 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + client/equo | 1 + client/equoTools.py | 130 ++++++++-- libraries/databaseTools.py | 475 +++++++++++++++++++++++++++++++--- libraries/entropyConstants.py | 1 + server/reagent | 3 +- 6 files changed, 547 insertions(+), 64 deletions(-) diff --git a/TODO b/TODO index 3831fa75e..8fbce5ae8 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,7 @@ TODO list: - enzyme, reagent: test kernel dependent packages - enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails - enzyme, add a module that integrates revdep-rebuild +- database: complete caching infrastructure for all search*, list* and is/are* - test stabilization code CLIENT: - add conflicts dependencies support diff --git a/client/equo b/client/equo index d7c029fdc..91c54cc58 100644 --- a/client/equo +++ b/client/equo @@ -62,6 +62,7 @@ def print_help(): print_info(" \t\t"+darkgreen("depends")+red("\t\t\t search which packages depend on the provided atoms")) print_info(" \t\t"+darkgreen("files")+red("\t\t\t search files owned by the provided atoms")) print_info(" \t\t"+darkgreen("removal")+red("\t\t\t print the removal tree for specified atoms")) + print_info(" \t\t"+darkgreen("orphans")+red("\t\t\t search files that don't belong to any package")+bold(" [USE WITH CARE]")) print_info(" \t\t"+darkgreen("description")+red("\t\t search a package description")) print_info(" \t\t"+red("--verbose")+"\t\t show more details about what's going on") print_info(" \t\t"+red("--quiet")+"\t\t\t show less details (useful for scripting)") diff --git a/client/equoTools.py b/client/equoTools.py index 78c3357d4..081d97072 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -998,11 +998,11 @@ def filterSatisfiedDependencies(dependencies): # FIXME add force reinstall optio ''' @description: generates a dependency tree using unsatisfied dependencies @input package: atomInfo [idpackage,reponame] - @output: dependency tree dictionary, plus status code + @output: dependency tree dictionary, plus status code ''' -treecache = {} def generateDependencyTree(atomInfo, emptydeps = False): + treecache = {} unsatisfiedDeps = getDependencies(atomInfo) remainingDeps = unsatisfiedDeps[:] dependenciesNotFound = [] @@ -1171,16 +1171,25 @@ def getRequiredPackages(foundAtoms, emptydeps = False): @input package: idpackages list @output: depends tree dictionary, plus status code ''' -dependscache = {} -def generateDependsTree(idpackages): +def generateDependsTree(idpackages, dbconn = None): + + dependscache = {} + closedb = False + + # database istance is passed? + if dbconn == None: + closedb = True + clientDbconn = openClientDatabase() + else: + clientDbconn = dbconn - clientDbconn = openClientDatabase() dependsOk = False treeview = set(idpackages) treelevel = idpackages[:] tree = {} treedepth = 0 # I start from level 1 because level 0 is idpackages itself tree[treedepth] = idpackages[:] + monotree = set(idpackages[:]) # monodimensional tree # check if dependstable is sane before beginning rx = clientDbconn.searchDepends(idpackages[0]) @@ -1190,11 +1199,12 @@ def generateDependsTree(idpackages): while (not dependsOk): treedepth += 1 - tree[treedepth] = [] + tree[treedepth] = set([]) for idpackage in treelevel: passed = dependscache.get(idpackage,None) - if passed: + systempkg = clientDbconn.isSystemPackage(idpackage) + if passed or systempkg: try: while 1: treeview.remove(idpackage) except: @@ -1203,12 +1213,34 @@ def generateDependsTree(idpackages): # obtain its depends depends = clientDbconn.searchDepends(idpackage) + # filter already satisfied ones + depends = [x for x in depends if x not in list(monotree)] if (depends): # something depends on idpackage for x in depends: if x not in tree[treedepth]: - tree[treedepth].append(x) + tree[treedepth].add(x) + monotree.add(x) treeview.add(x) - + else: # if no depends found, grab its dependencies and check + + mydeps = set(clientDbconn.retrieveDependencies(idpackage)) + _mydeps = set([]) + for x in mydeps: + match = atomMatchInRepository(x,clientDbconn) + if match and match[1] == 0: + _mydeps.add(match[0]) + mydeps = _mydeps + # now filter them + mydeps = [x for x in mydeps if x not in list(monotree)] + for x in mydeps: + #print clientDbconn.retrieveAtom(x) + mydepends = clientDbconn.searchDepends(x) + mydepends = [y for y in mydepends if y not in list(monotree)] + if (not mydepends): + tree[treedepth].add(x) + monotree.add(x) + treeview.add(x) + dependscache[idpackage] = True try: while 1: treeview.remove(idpackage) @@ -1237,16 +1269,10 @@ def generateDependsTree(idpackages): except: pass x += 1 + del tree - - #l = len(newtree) - #for x in range(l)[::-1]: - #print "LEVEL: "+str(x) - #for y in newtree[x]: - #print clientDbconn.retrieveAtom(y) - - clientDbconn.closeDB() - + if closedb: + clientDbconn.closeDB() return newtree,0 # treeview is used to show deps while tree is used to run the dependency code. @@ -1632,7 +1658,7 @@ def removePackageFromDatabase(idpackage, clientDbconn = None): # also remove from dependstable x = clientDbconn.removePackageFromDependsTable(idpackage) if (x == 1): #`shit, needs regeneration - regenerateDependsTable(clientDbconn) + regenerateDependsTable(clientDbconn, output = False) if (closedb): clientDbconn.closeDB() @@ -1678,6 +1704,9 @@ def query(options): elif options[0] == "removal": rc = searchRemoval(myopts[1:], quiet = equoRequestQuiet) + elif options[0] == "orphans": + rc = searchOrphans(quiet = equoRequestQuiet) + elif options[0] == "description": rc = searchDescription(myopts[1:], quiet = equoRequestQuiet) @@ -2142,6 +2171,65 @@ def searchFiles(atoms, idreturn = False, quiet = False): return 0 +def searchOrphans(quiet = False): + + if (not quiet): + print_info(yellow(" @@ ")+darkgreen("Orphans Search...")) + + # start to list all files on the system: + dirs = etpConst['filesystemdirs'] + foundFiles = set([]) + for dir in dirs: + for currentdir,subdirs,files in os.walk(dir): + for filename in files: + file = currentdir+"/"+filename + mask = [x for x in etpConst['filesystemdirsmask'] if file.startswith(x)] + if (not mask): + if (not quiet): + print_info(red(" @@ ")+blue("Looking: ")+bold(file[:20]+"..."), back = True) + foundFiles.add(file) + totalfiles = len(foundFiles) + if (not quiet): + print_info(red(" @@ ")+blue("Analyzed directories: ")+string.join(etpConst['filesystemdirs']," ")) + print_info(red(" @@ ")+blue("Masked directories: ")+string.join(etpConst['filesystemdirsmask']," ")) + print_info(red(" @@ ")+blue("Number of files collected on the filesystem: ")+bold(str(totalfiles))) + print_info(red(" @@ ")+blue("Now looking into Installed Packages database...")) + + # list all idpackages + clientDbconn = openClientDatabase() + idpackages = clientDbconn.listAllIdpackages() + # create content list + length = str(len(idpackages)) + count = 0 + for idpackage in idpackages: + if (not quiet): + count += 1 + atom = clientDbconn.retrieveAtom(idpackage) + txt = "["+str(count)+"/"+length+"] " + print_info(red(" @@ ")+blue("Intersecting content of package: ")+txt+bold(atom), back = True) + content = clientDbconn.retrieveContent(idpackage) + _content = set([]) + for x in content: + if x.startswith("/usr/lib64"): + x = "/usr/lib"+x[len("/usr/lib64"):] + _content.add(x) + # remove from foundFiles + del content + foundFiles.difference_update(_content) + if (not quiet): + print_info(red(" @@ ")+blue("Intersection completed. Showing statistics: ")) + print_info(red(" @@ ")+blue("Number of total files: ")+bold(str(totalfiles))) + print_info(red(" @@ ")+blue("Number of matching files: ")+bold(str(totalfiles - len(foundFiles)))) + print_info(red(" @@ ")+blue("Number of orphaned files: ")+bold(str(len(foundFiles)))) + + print len(foundFiles) + f = open("/tmp/foundfiles.txt","w") + for x in foundFiles: + f.write(x+"\n") + f.flush() + f.close() + + def searchRemoval(atoms, idreturn = False, quiet = False): if (not idreturn) and (not quiet): @@ -2162,7 +2250,7 @@ def searchRemoval(atoms, idreturn = False, quiet = False): choosenRemovalQueue = [] if (not quiet): print_info(red(" @@ ")+blue("Calculating removal dependencies, please wait..."), back = True) - treeview = generateDependsTree(foundAtoms) + treeview = generateDependsTree(foundAtoms,clientDbconn) treelength = len(treeview[0]) if treelength > 1: treeview = treeview[0] @@ -2669,7 +2757,7 @@ def removePackages(packages, ask = False, pretend = False, verbose = False, deps if (lookForOrphanedPackages): choosenRemovalQueue = [] print_info(red(" @@ ")+blue("Calculating removal dependencies, please wait..."), back = True) - treeview = generateDependsTree(plainRemovalQueue) + treeview = generateDependsTree(plainRemovalQueue, clientDbconn) treelength = len(treeview[0]) if treelength > 1: treeview = treeview[0] diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index a21254873..815d2d7ac 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -444,13 +444,6 @@ def database(options): print_info(green(" * ")+red("All the selected packages have been marked as requested. Have fun.")) dbconn.closeDB() - elif (options[0] == "sanity-check"): - print_info(green(" * ")+red("Running sanity check on the database ... "), back = True) - dbconn = etpDatabase(readOnly = True) - dbconn.noopCycle() - dbconn.closeDB() - print_info(green(" * ")+red("Database sanity check passed.")) - elif (options[0] == "remove"): print_info(green(" * ")+red("Scanning packages that would be removed ..."), back = True) @@ -739,6 +732,7 @@ class etpDatabase: self.clientDatabase = clientDatabase # caching dictionaries + self.databaseCache = {} if (self.clientDatabase): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened by Entropy client, file: "+str(dbFile)) @@ -1619,123 +1613,281 @@ class etpDatabase: def retrieveAtom(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveAtom: retrieving Atom for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveAtom',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "atom" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') atom = '' for row in self.cursor: atom = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveAtom'] = atom return atom def retrieveBranch(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveBranch: retrieving Branch for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveBranch',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "branch" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - atom = '' + br = '' for row in self.cursor: - atom = row[0] + br = row[0] break - return atom + + ''' caching ''' + self.databaseCache[idpackage]['retrieveBranch'] = br + return br def retrieveDownloadURL(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDownloadURL: retrieving download URL for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveDownloadURL',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "download" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') download = '' for row in self.cursor: download = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveDownloadURL'] = download return download def retrieveDescription(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDescription: retrieving description for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveDescription',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "description" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') description = '' for row in self.cursor: description = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveDescription'] = description return description def retrieveHomepage(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveHomepage: retrieving Homepage for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveHomepage',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "homepage" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') home = '' for row in self.cursor: home = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveHomepage'] = home return home # in bytes def retrieveSize(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveSize: retrieving Size for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveSize',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "size" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') size = 'N/A' for row in self.cursor: size = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveSize'] = size return size def retrieveDigest(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDigest: retrieving Digest for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveDigest',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "digest" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') digest = '' for row in self.cursor: digest = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveDigest'] = digest return digest def retrieveName(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveName: retrieving Name for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveName',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "name" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') name = '' for row in self.cursor: name = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveName'] = name return name def retrieveVersion(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveVersion: retrieving Version for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveVersion',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "version" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') ver = '' for row in self.cursor: ver = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveVersion'] = ver return ver def retrieveRevision(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveRevision: retrieving Revision for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveRevision',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "revision" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') rev = '' for row in self.cursor: rev = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveRevision'] = rev return rev def retrieveDateCreation(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDateCreation: retrieving Creation Date for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveDateCreation',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "datecreation" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') date = 'N/A' for row in self.cursor: date = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveDateCreation'] = date return date def retrieveApi(self, idpackage): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"v: retrieving Database API for package ID "+str(idpackage)) + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveApi: retrieving Database API for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveApi',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "etpapi" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') api = -1 for row in self.cursor: api = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveApi'] = api return api def retrieveUseflags(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveUseflags: retrieving USE flags for package ID "+str(idpackage)) - self.cursor.execute('SELECT "flag" FROM useflags WHERE idpackage = "'+str(idpackage)+'"') - flags = [] - for row in self.cursor: - flags.append(row[0]) - return flags - def retrieveUseflags(self, idpackage): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveUseflags: retrieving USE flags for package ID "+str(idpackage)) + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveUseflags',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idflag" FROM useflags WHERE idpackage = "'+str(idpackage)+'"') idflgs = [] for row in self.cursor: @@ -1745,27 +1897,65 @@ class etpDatabase: self.cursor.execute('SELECT "flagname" FROM useflagsreference WHERE idflag = "'+str(idflg)+'"') for row in self.cursor: flags.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveUseflags'] = flags return flags def retrieveConflicts(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveConflicts: retrieving Conflicts for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveConflicts',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "conflict" FROM conflicts WHERE idpackage = "'+str(idpackage)+'"') confl = [] for row in self.cursor: confl.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveConflicts'] = confl return confl def retrieveProvide(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveProvide: retrieving Provide for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveProvide',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "atom" FROM provide WHERE idpackage = "'+str(idpackage)+'"') provide = [] for row in self.cursor: provide.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveProvide'] = provide return provide def retrieveDependencies(self, idpackage): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDependencies: retrieving Dependencies for package ID "+str(idpackage)) self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = "'+str(idpackage)+'"') + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveDependencies',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + iddeps = [] for row in self.cursor: iddeps.append(row[0]) @@ -1774,10 +1964,44 @@ class etpDatabase: self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = "'+str(iddep)+'"') for row in self.cursor: deps.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveDependencies'] = deps return deps + def retrieveIdDependencies(self, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveDependencies: retrieving Dependencies for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveIdDependencies',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + + self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = "'+str(idpackage)+'"') + iddeps = [] + for row in self.cursor: + iddeps.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveIdDependencies'] = iddeps + return iddeps + def retrieveBinKeywords(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveBinKeywords: retrieving Binary Keywords for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveBinKeywords',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idkeyword" FROM binkeywords WHERE idpackage = "'+str(idpackage)+'"') idkws = [] for row in self.cursor: @@ -1787,9 +2011,22 @@ class etpDatabase: self.cursor.execute('SELECT "keywordname" FROM keywordsreference WHERE idkeyword = "'+str(idkw)+'"') for row in self.cursor: kw.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveBinKeywords'] = kw return kw def retrieveKeywords(self, idpackage): + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveKeywords',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveKeywords: retrieving Keywords for package ID "+str(idpackage)) self.cursor.execute('SELECT "idkeyword" FROM keywords WHERE idpackage = "'+str(idpackage)+'"') idkws = [] @@ -1800,10 +2037,23 @@ class etpDatabase: self.cursor.execute('SELECT "keywordname" FROM keywordsreference WHERE idkeyword = "'+str(idkw)+'"') for row in self.cursor: kw.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveKeywords'] = kw return kw def retrieveProtect(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveProtect: retrieving CONFIG_PROTECT for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveProtect',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idprotect" FROM configprotect WHERE idpackage = "'+str(idpackage)+'"') idprotect = -1 for row in self.cursor: @@ -1816,10 +2066,23 @@ class etpDatabase: for row in self.cursor: protect = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveProtect'] = protect return protect def retrieveProtectMask(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveProtectMask: retrieving CONFIG_PROTECT_MASK for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveProtectMask',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idprotect" FROM configprotectmask WHERE idpackage = "'+str(idpackage)+'"') idprotect = -1 for row in self.cursor: @@ -1832,10 +2095,23 @@ class etpDatabase: for row in self.cursor: protect = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveProtectMask'] = protect return protect def retrieveSources(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveSources: retrieving Sources for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveSources',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT idsource FROM sources WHERE idpackage = "'+str(idpackage)+'"') idsources = [] for row in self.cursor: @@ -1845,44 +2121,109 @@ class etpDatabase: self.cursor.execute('SELECT source FROM sourcesreference WHERE idsource = "'+str(idsource)+'"') for row in self.cursor: sources.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveSources'] = sources return sources def retrieveContent(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveContent: retrieving Content for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveContent',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "file" FROM content WHERE idpackage = "'+str(idpackage)+'"') fl = [] for row in self.cursor: fl.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['retrieveContent'] = fl return fl def retrieveSlot(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveSlot: retrieving Slot for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveSlot',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "slot" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') ver = '' for row in self.cursor: ver = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveSlot'] = ver return ver def retrieveVersionTag(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveVersionTag: retrieving Version TAG for package ID "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveVersionTag',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "versiontag" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') ver = '' for row in self.cursor: ver = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveVersionTag'] = ver return ver def retrieveMirrorInfo(self, mirrorname): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveMirrorInfo: retrieving Mirror info for mirror name "+str(mirrorname)) + + ''' caching ''' + cached = self.databaseCache.get(mirrorname, None) + if cached: + rslt = self.databaseCache[mirrorname].get('retrieveMirrorInfo',None) + if rslt: + return rslt + else: + self.databaseCache[mirrorname] = {} + self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = "'+str(mirrorname)+'"') mirrorlist = [] for row in self.cursor: mirrorlist.append(row[0]) + + ''' caching ''' + self.databaseCache[mirrorname]['retrieveMirrorInfo'] = mirrorlist return mirrorlist def retrieveCategory(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveCategory: retrieving Category for package ID for "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveCategory',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idcategory" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') cat = '' for row in self.cursor: @@ -1894,10 +2235,23 @@ class etpDatabase: for row in self.cursor: cat = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveCategory'] = cat return cat def retrieveLicense(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveLicense: retrieving License for package ID for "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveLicense',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idlicense" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') lic = -1 for row in self.cursor: @@ -1909,10 +2263,23 @@ class etpDatabase: for row in self.cursor: licname = row[0] break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveLicense'] = licname return licname def retrieveCompileFlags(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveCompileFlags: retrieving CHOST,CFLAGS,CXXFLAGS for package ID for "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('retrieveCompileFlags',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + self.cursor.execute('SELECT "idflags" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') idflag = -1 for row in self.cursor: @@ -1924,6 +2291,9 @@ class etpDatabase: for row in self.cursor: flags = row break + + ''' caching ''' + self.databaseCache[idpackage]['retrieveCompileFlags'] = flags return flags # You must provide the full atom to this function @@ -2053,16 +2423,31 @@ class etpDatabase: def isSystemPackage(self,idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSystemPackage: called.") + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('isSystemPackage',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + result = -1 self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = "'+str(idpackage)+'"') for row in self.cursor: result = row[0] break + rslt = False if result != -1: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSystemPackage: package is in system.") - return True + rslt = True dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSystemPackage: package is NOT in system.") - return False + + ''' caching ''' + self.databaseCache[idpackage]['isSystemPackage'] = rslt + + return rslt def areCompileFlagsAvailable(self,chost,cflags,cxxflags): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"areCompileFlagsAvailable: called.") @@ -2100,6 +2485,16 @@ class etpDatabase: def searchProvide(self, keyword): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchProvide: called for "+keyword) + + ''' caching ''' + cached = self.databaseCache.get(keyword, None) + if cached: + rslt = self.databaseCache[keyword].get('searchProvide',None) + if rslt: + return rslt + else: + self.databaseCache[keyword] = {} + idpackage = [] self.cursor.execute('SELECT idpackage FROM provide WHERE atom = "'+keyword+'"') for row in self.cursor: @@ -2110,6 +2505,10 @@ class etpDatabase: for row in self.cursor: result = row break + + ''' caching ''' + self.databaseCache[keyword]['searchProvide'] = result + return result def searchProvideInBranch(self, keyword, branch): @@ -2325,25 +2724,6 @@ class etpDatabase: result.append(row) return result - # useful to quickly retrieve (and trash) all the data - # and look for problems. - def noopCycle(self): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"noopCycle: called. ") - self.cursor.execute('SELECT * FROM baseinfo') - self.cursor.execute('SELECT * FROM extrainfo') - self.cursor.execute('SELECT * FROM content') - self.cursor.execute('SELECT * FROM dependencies') - self.cursor.execute('SELECT * FROM conflicts') - self.cursor.execute('SELECT * FROM mirrorlinks') - self.cursor.execute('SELECT * FROM sources') - self.cursor.execute('SELECT * FROM useflags') - self.cursor.execute('SELECT * FROM keywords') - self.cursor.execute('SELECT * FROM binkeywords') - self.cursor.execute('SELECT * FROM categories') - self.cursor.execute('SELECT * FROM licenses') - self.cursor.execute('SELECT * FROM flags') - - def stabilizePackage(self,atom,stable = True): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable)) @@ -2429,6 +2809,16 @@ class etpDatabase: def searchDepends(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDepends: called for idpackage "+str(idpackage)) + + ''' caching ''' + cached = self.databaseCache.get(idpackage, None) + if cached: + rslt = self.databaseCache[idpackage].get('searchDepends',None) + if rslt: + return rslt + else: + self.databaseCache[idpackage] = {} + iddeps = [] try: self.cursor.execute('SELECT iddependency FROM dependstable WHERE idpackage = "'+str(idpackage)+'"') @@ -2442,6 +2832,10 @@ class etpDatabase: self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep)+'"') for row in self.cursor: result.append(row[0]) + + ''' caching ''' + self.databaseCache[idpackage]['searchDepends'] = result + return result # temporary/compat functions @@ -2457,7 +2851,6 @@ class etpDatabase: def addDependRelationToDependsTable(self, iddependency, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addDependRelationToDependsTable: called for iddependency "+str(iddependency)+" and idpackage "+str(idpackage)) - # FIXME: should check for existance of the PK? self.cursor.execute( 'INSERT into dependstable VALUES ' '(?,?)' diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 2f7fc669a..b1546b17c 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -356,6 +356,7 @@ etpConst = { 'branch': "unstable", # choosen branch 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) 'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem + 'filesystemdirsmask': ['/var/cache','/var/db','/var/empty','/var/lib/portage','/var/lib/entropy','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage'], 'officialrepositoryname': "sabayonlinux.org", # our official repository name 'packagedbdir': "/db", # directory of the database file in the .tbz2 package 'packagedbfile': "/data.db", # database file in the directory above diff --git a/server/reagent b/server/reagent index c163b09ca..8e6ec392a 100644 --- a/server/reagent +++ b/server/reagent @@ -55,8 +55,7 @@ def print_help(): print_info(" \t\t"+green("stabilize")+"\t\t\t\t Mark as stable a package, a list of packages, world") print_info(" \t\t"+green("unstabilize")+"\t\t\t\t Mark as unstable a package, a list of packages, world") print_info(" \t\t"+green("md5check")+"\t\t\t\t Check digest of a package, a list of packages, world") - #print_info(" \t\t"+green(bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package") - print_info(" \t\t"+green(bold("sanity-check"))+"\t\t\t\t Do a quick check on the database to assure data integrity") + #print_info(" \t\t"+green(bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package") FIXME can be done using sets print_info(" \t"+green(bold("deptest"))+yellow("\t\t Look for unsatisfied dependencies inside database")) print_info(" \t\t"+red("--quiet")+"\t\t\t\t\t just print the dependencies list") print_info(" \t"+green(bold("smartapps"))+yellow("\t Entropy application tool to create self-dependant applications [EXPERIMENTAL]"))