diff --git a/client/equo b/client/equo index c9d6e73e4..c52cd348c 100644 --- a/client/equo +++ b/client/equo @@ -224,7 +224,10 @@ def load_conf_cache(): if not etpUi['quiet']: print_info(red(" @@ ")+blue("Caching equo conf"), back = True) import text_configuration try: + oldquiet = etpUi['quiet'] + etpUi['quiet'] = True scandata = text_configuration.Equo.FileUpdates.scanfs(dcache = True) + etpUi['quiet'] = oldquiet except: if not etpUi['quiet']: print_info(red(" @@ ")+blue("Caching not run.")) return @@ -346,7 +349,7 @@ except IOError, e: if e.errno != 32: raise except KeyboardInterrupt: - reset_cache() + #reset_cache() sys.exit(0) #except Timeout: # pass diff --git a/client/text_security.py b/client/text_security.py index 25296de28..a6af74e95 100644 --- a/client/text_security.py +++ b/client/text_security.py @@ -46,9 +46,7 @@ def security(options): elif options[0] == "list": rc = list_advisories(only_affected = only_affected, only_unaffected = only_unaffected) elif options[0] == "install": - Equo.load_cache() rc = install_packages(fetch = fetch) - Equo.save_cache() elif options[0] == "info": rc = show_advisories_info(options[1:]) else: diff --git a/client/text_ui.py b/client/text_ui.py index b732246a1..ef8a80feb 100644 --- a/client/text_ui.py +++ b/client/text_ui.py @@ -84,32 +84,24 @@ def package(options): myopts = _myopts if (options[0] == "deptest"): - Equo.load_cache() rc, garbage = dependenciesTest() - Equo.save_cache() elif (options[0] == "libtest"): rc, garbage = librariesTest(listfiles = equoRequestListfiles) elif (options[0] == "install"): if (myopts) or (mytbz2paths) or (equoRequestResume): - Equo.load_cache() status, rc = installPackages(myopts, deps = equoRequestDeps, emptydeps = equoRequestEmptyDeps, onlyfetch = equoRequestOnlyFetch, deepdeps = equoRequestDeep, configFiles = equoRequestConfigFiles, tbz2 = mytbz2paths, resume = equoRequestResume, skipfirst = equoRequestSkipfirst, dochecksum = equoRequestChecksum) - Equo.save_cache() else: print_error(red(" Nothing to do.")) rc = 127 elif (options[0] == "world"): - Equo.load_cache() status, rc = worldUpdate(onlyfetch = equoRequestOnlyFetch, replay = (equoRequestReplay or equoRequestEmptyDeps), upgradeTo = equoRequestUpgradeTo, resume = equoRequestResume, skipfirst = equoRequestSkipfirst, human = True, dochecksum = equoRequestChecksum) - Equo.save_cache() elif (options[0] == "remove"): if myopts or equoRequestResume: - Equo.load_cache() status, rc = removePackages(myopts, deps = equoRequestDeps, deep = equoRequestDeep, configFiles = equoRequestConfigFiles, resume = equoRequestResume) - Equo.save_cache() else: print_error(red(" Nothing to do.")) rc = 127 @@ -160,7 +152,7 @@ def worldUpdate(onlyfetch = False, replay = False, upgradeTo = None, resume = Fa resume_cache['ask'] = etpUi['ask'] resume_cache['verbose'] = etpUi['verbose'] resume_cache['onlyfetch'] = onlyfetch - resume_cache['remove'] = remove + resume_cache['remove'] = remove.copy() Equo.dumpTools.dumpobj(etpCache['world'],resume_cache) else: # if resume, load cache if possible @@ -443,16 +435,16 @@ def installPackages(packages = [], atomsdata = [], deps = True, emptydeps = Fals action = -1 flags += "] " - repoinfo = red("[")+bold(packageInfo[1])+red("] ") + repoinfo = "["+brown(packageInfo[1])+"] " oldinfo = '' if action != 0: - oldinfo = " ["+blue(installedVer)+"/"+red(str(installedRev)) + oldinfo = " ["+blue(installedVer)+"|"+red(str(installedRev)) oldtag = "]" if installedTag: - oldtag = "/"+darkred(installedTag)+oldtag + oldtag = "|"+darkred(installedTag)+oldtag oldinfo += oldtag - print_info(darkred(" ##")+flags+repoinfo+enlightenatom(str(pkgatom))+"/"+str(pkgrev)+oldinfo) + print_info(darkred(" ##")+flags+repoinfo+blue(pkgatom)+"|"+red(str(pkgrev))+oldinfo) if (removalQueue): diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 6c0845408..fc92d7910 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -136,8 +136,9 @@ class etpDatabase(TextInterface): self.indexing = False self.dbFile = dbFile - # load db on disk cache? - self.loadDatabaseCache() + # no caching for non root and server connections + if (self.dbname == 'etpdb') or (etpConst['uid'] != 0): + self.xcache = False # create connection self.connection = dbapi2.connect(dbFile,timeout=300.0) @@ -461,11 +462,7 @@ class etpDatabase(TextInterface): clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest) # clear client cache - dbCacheStore[etpCache['dbMatch']+"client"] = {} - dbCacheStore[etpCache['dbSearch']+"client"] = {} - dumpTools.dumpobj(etpCache['dbMatch']+"client",{}) - dumpTools.dumpobj(etpCache['dbSearch']+"client",{}) - clientDbconn.clearInfoCache() + clientDbconn.clearCache() clientDbconn.closeDB() del clientDbconn @@ -534,11 +531,7 @@ class etpDatabase(TextInterface): self.runTreeUpdatesSlotmoveAction(command[1:]) # discard cache - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - dumpTools.dumpobj(etpCache['dbMatch']+self.dbname,{}) - dumpTools.dumpobj(etpCache['dbSearch']+self.dbname,{}) - self.clearInfoCache() + self.clearCache() # -- move action: @@ -699,67 +692,6 @@ class etpDatabase(TextInterface): header = darkred(" * ") ) - def loadDatabaseCache(self): - - if (self.xcache) and (self.dbname != 'etpdb') and (etpConst['uid'] == 0): - - ''' database query cache ''' - broken1 = False - dbinfo = dbCacheStore.get(etpCache['dbInfo']+self.dbname) - if dbinfo == None: - try: - dbCacheStore[etpCache['dbInfo']+self.dbname] = dumpTools.loadobj(etpCache['dbInfo']+self.dbname) - if dbCacheStore[etpCache['dbInfo']+self.dbname] == None: - broken1 = True - dbCacheStore[etpCache['dbInfo']+self.dbname] = {} - except: - broken1 = True - pass - - ''' database atom dependencies cache ''' - dbmatch = dbCacheStore.get(etpCache['dbMatch']+self.dbname) - broken2 = False - if dbmatch == None: - try: - dbCacheStore[etpCache['dbMatch']+self.dbname] = dumpTools.loadobj(etpCache['dbMatch']+self.dbname) - if dbCacheStore[etpCache['dbMatch']+self.dbname] == None: - broken2 = True - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - except: - broken2 = True - pass - - ''' database search cache ''' - dbmatch = dbCacheStore.get(etpCache['dbSearch']+self.dbname) - broken3 = False - if dbmatch == None: - try: - dbCacheStore[etpCache['dbSearch']+self.dbname] = dumpTools.loadobj(etpCache['dbSearch']+self.dbname) - if dbCacheStore[etpCache['dbSearch']+self.dbname] == None: - broken3 = True - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - except: - broken3 = True - pass - - if (broken1 or broken2 or broken3): - # discard both caches - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - dumpTools.dumpobj(etpCache['dbMatch']+self.dbname,{}) - dumpTools.dumpobj(etpCache['dbSearch']+self.dbname,{}) - self.clearInfoCache() - - else: - self.xcache = False # setting this to be safe - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - try: - self.clearInfoCache() - except: # if it's not possible to write cache - pass - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - - # this function manages the submitted package # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage @@ -1122,13 +1054,7 @@ class etpDatabase(TextInterface): ) ) - # clear caches - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - # dump to be sure - dumpTools.dumpobj(etpCache['dbMatch']+self.dbname,{}) - dumpTools.dumpobj(etpCache['dbSearch']+self.dbname,{}) - self.clearInfoCache() + self.clearCache() self.packagesAdded = True self.commitChanges() @@ -1294,12 +1220,7 @@ class etpDatabase(TextInterface): self.packagesRemoved = True # clear caches - dbCacheStore[etpCache['dbMatch']+self.dbname] = {} - dbCacheStore[etpCache['dbSearch']+self.dbname] = {} - # dump to be sure - dumpTools.dumpobj(etpCache['dbMatch']+self.dbname,{}) - dumpTools.dumpobj(etpCache['dbSearch']+self.dbname,{}) - self.clearInfoCache() + self.clearCache() self.commitChanges() @@ -1933,67 +1854,31 @@ class etpDatabase(TextInterface): def fetchone2set(self, item): return set(item) - def clearInfoCache(self): - # clear caches - dbCacheStore[etpCache['dbInfo']+self.dbname] = {} - # dump to be sure - dumpTools.dumpobj(etpCache['dbInfo']+self.dbname,{}) + def clearCache(self): + def do_clear(name): + dump_file = os.path.join(etpConst['dumpstoragedir'],name) + os.system("rm -rf %s*.dmp" % (dump_file,) ) + do_clear(etpCache['dbInfo']) + do_clear(etpCache['dbMatch']) - def fetchInfoCache(self,idpackage,function): + def fetchInfoCache(self, idpackage, function, extra_hash = 0): if (self.xcache): + c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash ) try: - cached = dbCacheStore[etpCache['dbInfo']+self.dbname].get(int(idpackage)) - except KeyError: # dict does not exist? - self.clearInfoCache() - return None - if cached != None: - rslt = cached.get(function) - if rslt != None: - if (type(rslt) is dict) or (type(rslt) is set): # needed ? - return rslt.copy() - elif (type(rslt) is list): - return rslt[:] - else: - return rslt - return None + cached = dumpTools.loadobj(etpCache['dbInfo']+c_hash) + if cached != None: + return cached + except EOFError: + pass - def storeInfoCache(self,idpackage,function,info_cache_data): + + def storeInfoCache(self, idpackage, function, info_cache_data, extra_hash = 0): if (self.xcache): + c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash ) try: - cache = dbCacheStore[etpCache['dbInfo']+self.dbname].get(int(idpackage)) - except KeyError: # something bad happened even here, reset cache - self.clearInfoCache() - cache = None - if cache == None: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} - if (type(info_cache_data) is set) or (type(info_cache_data) is dict): - dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)][function] = info_cache_data.copy() - elif (type(info_cache_data) is list): - dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)][function] = info_cache_data[:] - else: - dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)][function] = info_cache_data - - def fetchSearchCache(self,searchdata,function): - if (self.xcache): - cached = dbCacheStore[etpCache['dbSearch']+self.dbname].get(function) - if cached != None: - rslt = cached.get(searchdata) - if rslt != None: - if (type(rslt) is dict) or (type(rslt) is set): # needed ? - return rslt.copy() - elif (type(rslt) is list): - return rslt[:] - else: - return rslt - return None - - def storeSearchCache(self,searchdata,function,data): - if (self.xcache): - cache = dbCacheStore[etpCache['dbSearch']+self.dbname].get(function) - if cache == None: dbCacheStore[etpCache['dbSearch']+self.dbname][function] = {} - if (type(data) is set) or (type(data) is dict): - dbCacheStore[etpCache['dbSearch']+self.dbname][function][searchdata] = data.copy() - else: - dbCacheStore[etpCache['dbSearch']+self.dbname][function][searchdata] = data + dumpTools.dumpobj(etpCache['dbInfo']+c_hash,info_cache_data) + except IOError: + pass def retrieveRepositoryUpdatesDigest(self, repository): if not self.doesTableExist("treeupdates"): @@ -2305,12 +2190,16 @@ class etpDatabase(TextInterface): def retrieveNeeded(self, idpackage): + cache = self.fetchInfoCache(idpackage,'retrieveNeeded') + if cache != None: return cache + if not self.doesTableExist("needed"): self.createNeededTable() self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,)) needed = self.fetchall2set(self.cursor.fetchall()) + self.storeInfoCache(idpackage,'retrieveNeeded',needed) return needed def retrieveConflicts(self, idpackage): @@ -2442,6 +2331,10 @@ class etpDatabase(TextInterface): def retrieveContent(self, idpackage, extended = False, contentType = None): + c_hash = hash(extended)+hash(contentType) + cache = self.fetchInfoCache(idpackage,'retrieveContent', extra_hash = c_hash) + if cache != None: return cache + # like portage does self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") @@ -2465,18 +2358,19 @@ class etpDatabase(TextInterface): else: fl = self.fetchall2set(self.cursor.fetchall()) - return fl + self.storeInfoCache(idpackage,'retrieveContent',fl, extra_hash = c_hash) + return fl def retrieveSlot(self, idpackage): - cache = self.fetchInfoCache(idpackage,'retrieveSlot') - if cache != None: return cache + cache = self.fetchInfoCache(idpackage,'retrieveSlot') + if cache != None: return cache - self.cursor.execute('SELECT "slot" FROM baseinfo WHERE idpackage = (?)', (idpackage,)) - ver = self.cursor.fetchone()[0] + self.cursor.execute('SELECT "slot" FROM baseinfo WHERE idpackage = (?)', (idpackage,)) + ver = self.cursor.fetchone()[0] - self.storeInfoCache(idpackage,'retrieveSlot',ver) - return ver + self.storeInfoCache(idpackage,'retrieveSlot',ver) + return ver def retrieveVersionTag(self, idpackage): @@ -2881,10 +2775,6 @@ class etpDatabase(TextInterface): def searchPackagesByName(self, keyword, sensitive = False, branch = None): - if (self.xcache): - cached = self.fetchSearchCache((keyword,sensitive,branch),'searchPackagesByName') - if cached != None: return cached - if sensitive: searchkeywords = [keyword] else: @@ -2900,17 +2790,11 @@ class etpDatabase(TextInterface): self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) results = self.cursor.fetchall() - if (self.xcache): - self.storeSearchCache((keyword,sensitive,branch),'searchPackagesByName',results) return results def searchPackagesByCategory(self, keyword, like = False, branch = None): - if (self.xcache): - cached = self.fetchSearchCache((keyword,branch),'searchPackagesByCategory') - if cached != None: return cached - searchkeywords = [keyword] branchstring = '' if branch: @@ -2923,16 +2807,11 @@ class etpDatabase(TextInterface): self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) results = self.cursor.fetchall() - if (self.xcache): - self.storeSearchCache((keyword,branch),'searchPackagesByCategory',results) + return results def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None): - if (self.xcache): - cached = self.fetchSearchCache((name,category,sensitive,branch),'searchPackagesByNameAndCategory') - if cached != None: return cached - # get category id idcat = -1 self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) @@ -2961,16 +2840,11 @@ class etpDatabase(TextInterface): self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory = (?) '+branchstring, searchkeywords) results = self.cursor.fetchall() - if (self.xcache): - self.storeSearchCache((name,category,sensitive,branch),'searchPackagesByNameAndCategory',results) + return results def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False): - if (self.xcache): - cached = self.fetchSearchCache((key,version,branch,sensitive),'searchPackagesKeyVersion') - if cached != None: return cached - searchkeywords = [] if sensitive: searchkeywords.append(key) @@ -2990,8 +2864,7 @@ class etpDatabase(TextInterface): self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE LOWER(categories.category) || "/" || LOWER(baseinfo.name) = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) results = self.cursor.fetchall() - if (self.xcache): - self.storeSearchCache((key,version,branch,sensitive),'searchPackagesKeyVersion',results) + return results def listAllPackages(self): @@ -3034,15 +2907,8 @@ class etpDatabase(TextInterface): def listAllBranches(self): - if (self.xcache): - cached = self.fetchSearchCache((None,),'listAllBranches') - if cached != None: return cached - self.cursor.execute('SELECT branch FROM baseinfo') results = self.fetchall2set(self.cursor.fetchall()) - - if (self.xcache): - self.storeSearchCache((None,),'listAllBranches',results) return results def listIdPackagesInIdcategory(self,idcategory): @@ -3207,15 +3073,14 @@ class etpDatabase(TextInterface): def tablesChecksum(self): # NOTE: if you will add dependstable to the validation # please have a look at EquoInterface.__install_package_into_database world calculation cache stuff - import md5 self.cursor.execute('select * from baseinfo') - m = md5.new() + c_hash = 0 for row in self.cursor: - m.update(str(row)) + c_hash += hash(str(row)) self.cursor.execute('select * from dependenciesreference') for row in self.cursor: - m.update(str(row)) - return m.hexdigest() + c_hash += hash(str(row)) + return str(c_hash) ######################################################## @@ -3223,99 +3088,6 @@ class etpDatabase(TextInterface): ## Client Database API / but also used by server part # - def storeLibraryBreakageCache(self, broken_atoms, match, idpackage, deep_deps): - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return - repo_idpackage = match[0] - repo_name = match[1] - if self.dbname == "client" and not repo_name.endswith(".tbz2") and (etpConst['uid'] == 0): - if deep_deps: - deep = 1 - else: - deep = 0 - if not self.doesTableExist("library_breakages"): - self.createLibraryBreakagesTable() - repo_order_ck = hash(str(etpRepositoriesOrder) + \ - etpConst['systemroot'] + \ - str(etpRepositories[repo_name]['dbrevision'])) - mybroken = broken_atoms.copy() - if not mybroken: - mybroken |= set(["."]) - for atom in mybroken: - self.cursor.execute( - 'INSERT into library_breakages VALUES ' - '(NULL,?,?,?,?,?,?)' - , ( repo_idpackage, - repo_name, - idpackage, - deep, - repo_order_ck, - atom - ) - ) - self.commitChanges() - - def getLibraryBreakageCache(self, match, idpackage, deep_deps): - repo_idpackage = match[0] - repo_name = match[1] - if self.dbname == "client" and not repo_name.endswith(".tbz2"): - if deep_deps: - deep = 1 - else: - deep = 0 - if not self.doesTableExist("library_breakages"): - self.createLibraryBreakagesTable() - return None - repo_order_ck = hash(str(etpRepositoriesOrder) + \ - etpConst['systemroot'] + \ - str(etpRepositories[repo_name]['dbrevision'])) - # fetch data - self.cursor.execute(""" - SELECT atom FROM library_breakages WHERE - repoidpackage = (?) and - reponame = (?) and - idpackage = (?) and - deep = (?) and - repoorderck = (?) - """, - (repo_idpackage,repo_name,idpackage,deep,repo_order_ck) - ) - found = self.fetchall2set(self.cursor.fetchall()) - if found: - found = found - set(["."]) - return found - - def clearLibraryBreakageCache(self): - self.checkReadOnly() - self.cursor.execute("DROP INDEX IF EXISTS libraryindex;") - self.cursor.execute("DROP TABLE IF EXISTS library_breakages;") - self.createLibraryBreakagesTable() - self.commitChanges() - - def createLibraryBreakagesTable(self): - self.cursor.execute(""" - CREATE TABLE library_breakages ( - idbreak INTEGER PRIMARY KEY, - repoidpackage INTEGER, - reponame VARCHAR, - idpackage INTEGER, - deep INTEGER, - repoorderck INTEGER, - atom VARCHAR - ); - """) - self.cursor.execute(""" - CREATE INDEX libraryindex ON library_breakages ( - repoidpackage, - reponame, - idpackage, - repoorderck, - atom - ); - """) - def addPackageToInstalledTable(self, idpackage, repositoryName): self.checkReadOnly() self.cursor.execute( @@ -3648,43 +3420,35 @@ class etpDatabase(TextInterface): # def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter): - if (self.xcache): - try: - cache_tuple = (atom,matchSlot,matchTag,multiMatch,caseSensitive,matchBranches,packagesFilter) - cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get((cache_tuple)) - if cached: - return cached - else: - return None - except KeyError: # issues with dictionaries? - return None - else: - return None + if self.xcache: + c_hash = str( hash(atom) + \ + hash(matchSlot) + \ + hash(matchTag) + \ + hash(multiMatch) + \ + hash(caseSensitive) + \ + hash(tuple(matchBranches)) + \ + hash(packagesFilter) + \ + hash(self.dbname) + ) + cached = dumpTools.loadobj(etpCache['dbMatch']+c_hash) + if cached != None: + return cached def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter): - - data = { - 'result': result[:], - 'atom': atom, - 'caseSensitive': caseSensitive, - 'matchSlot': matchSlot, - 'multiMatch': multiMatch, - 'matchBranches': matchBranches, - 'matchTag': matchTag, - 'packagesFilter': packagesFilter, - 'dbname': self.dbname - } - task = entropyTools.parallelTask(self.__atomMatchStoreCache, data) - task.parallel_wait() - task.start() - del data - - def __atomMatchStoreCache(self, data): - try: - cache_tuple = (data['atom'],data['matchSlot'],data['matchTag'],data['multiMatch'],data['caseSensitive'],data['matchBranches'],data['packagesFilter']) - dbCacheStore[etpCache['dbMatch']+data['dbname']][cache_tuple] = data['result'][:] - except KeyError: # againnn, issues with dicts?? - pass + if self.xcache: + c_hash = str( hash(atom) + \ + hash(matchSlot) + \ + hash(matchTag) + \ + hash(multiMatch) + \ + hash(caseSensitive) + \ + hash(tuple(matchBranches)) + \ + hash(packagesFilter) + \ + hash(self.dbname) + ) + try: + dumpTools.dumpobj(etpCache['dbMatch']+c_hash,result) + except IOError: + pass # function that validate one atom by reading keywords settings @@ -3692,6 +3456,7 @@ class etpDatabase(TextInterface): def idpackageValidator(self,idpackage): reponame = self.dbname[5:] + cached = idpackageValidatorCache.get((idpackage,reponame)) if cached != None: return cached @@ -3734,6 +3499,7 @@ class etpDatabase(TextInterface): keyword_data = etpConst['packagemasking']['keywords']['repositories'][reponame].get(keyword) for atom in keyword_data: if atom == "*": # all packages in this repo with keyword "keyword" are ok + idpackageValidatorCache[(idpackage,reponame)] = idpackage return idpackage matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: diff --git a/libraries/entropy.py b/libraries/entropy.py index eec838090..c4169a2e5 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -95,6 +95,11 @@ class EquoInterface(TextInterface): # security interface self.Security = SecurityInterface(self) + if not self.xcache: + try: + self.purge_cache(False) + except: + pass def switchChroot(self, chroot = ""): # clean caches @@ -216,10 +221,10 @@ class EquoInterface(TextInterface): def purge_cache(self, showProgress = True): const_resetCache() if etpConst['uid'] == 0: - dumpdir = etpConst['dumpstoragedir'] - if not dumpdir.endswith("/"): dumpdir = dumpdir+"/" for key in etpCache: - cachefile = dumpdir+etpCache[key]+"*.dmp" + if key in ["library_breakage"]: # caches we can skip + continue + cachefile = os.path.join(etpConst['dumpstoragedir'],etpCache[key])+"*.dmp" if showProgress: self.updateProgress(darkred("Cleaning %s...") % (cachefile,), importance = 1, type = "warning", back = True) try: os.system("rm -f "+cachefile) @@ -270,7 +275,6 @@ class EquoInterface(TextInterface): depends.add(info[1]) self.updateProgress(darkgreen("Resolving metadata"), importance = 1, type = "warning") - atomMatchCache.clear() maxlen = len(names) cnt = 0 for name in names: @@ -314,84 +318,10 @@ class EquoInterface(TextInterface): pass self.updateProgress(darkred("Dependencies cache filled."), importance = 2, type = "warning") - self.save_cache() - def load_cache(self, showProgress = True): - - if (etpConst['uid'] != 0) or (not self.xcache): # don't load cache as user - return - - try: - if showProgress: self.updateProgress(blue("Loading On-Disk Cache..."), importance = 2, type = "info") - except: - pass - - # atomMatch - try: - mycache = self.dumpTools.loadobj(etpCache['atomMatch']) - if isinstance(mycache, dict): - atomMatchCache.clear() - atomMatchCache.update(mycache) - del mycache - except: - atomMatchCache.clear() - self.dumpTools.dumpobj(etpCache['atomMatch'],{}) - - # removal dependencies - try: - mycache = self.dumpTools.loadobj(etpCache['generateDependsTree']) - if isinstance(mycache, dict): - generateDependsTreeCache.clear() - generateDependsTreeCache.update(mycache) - del mycache - except: - generateDependsTreeCache.clear() - self.dumpTools.dumpobj(etpCache['generateDependsTree'],{}) - - # check_package_update cache - try: - mycache = self.dumpTools.loadobj(etpCache['check_package_update']) - if isinstance(mycache, dict): - check_package_update_cache.clear() - check_package_update_cache.update(mycache) - del mycache - except: - check_package_update_cache.clear() - self.dumpTools.dumpobj(etpCache['check_package_update'],{}) - - def save_cache(self): - - if (etpConst['uid'] != 0): # don't save cache as user - return - - self.dumpTools.dumpobj(etpCache['check_package_update'],check_package_update_cache) - - self.dumpTools.dumpobj(etpCache['atomMatch'],atomMatchCache) - if os.path.isfile(etpConst['dumpstoragedir']+"/"+etpCache['atomMatch']+".dmp"): - if os.stat(etpConst['dumpstoragedir']+"/"+etpCache['atomMatch']+".dmp")[6] > etpCacheSizes['atomMatch']: - # clean cache - self.dumpTools.dumpobj(etpCache['atomMatch'],{}) - self.dumpTools.dumpobj(etpCache['generateDependsTree'],generateDependsTreeCache) - if os.path.isfile(etpConst['dumpstoragedir']+"/"+etpCache['generateDependsTree']+".dmp"): - if os.stat(etpConst['dumpstoragedir']+"/"+etpCache['generateDependsTree']+".dmp")[6] > etpCacheSizes['generateDependsTree']: - # clean cache - self.dumpTools.dumpobj(etpCache['generateDependsTree'],{}) - for dbinfo in dbCacheStore: - self.dumpTools.dumpobj(dbinfo,dbCacheStore[dbinfo]) - # check size - if os.path.isfile(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp"): - if dbinfo.startswith(etpCache['dbMatch']): - if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbMatch']: - # clean cache - self.dumpTools.dumpobj(dbinfo,{}) - elif dbinfo.startswith(etpCache['dbInfo']): - if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbInfo']: - # clean cache - self.dumpTools.dumpobj(dbinfo,{}) - elif dbinfo.startswith(etpCache['dbSearch']): - if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbSearch']: - # clean cache - self.dumpTools.dumpobj(dbinfo,{}) + def clear_dump_cache(self, dump_name): + dump_file = os.path.join(etpConst['dumpstoragedir'],dump_name) + os.system("rm -rf %s*.dmp" % (dump_file,) ) ''' Cache stuff :: end @@ -627,7 +557,6 @@ class EquoInterface(TextInterface): self.clientDbconn.resetTreeupdatesDigests() # clean cache self.purge_cache(showProgress = False) - self.load_cache(showProgress = False) # reopen Client Database, this will make treeupdates to be re-read self.reopenClientDbconn() self.closeAllRepositoryDatabases() @@ -640,8 +569,11 @@ class EquoInterface(TextInterface): def check_package_update(self, atom): - if check_package_update_cache.has_key(atom): - return check_package_update_cache[atom] + if self.xcache: + c_hash = str(hash(atom)) + cached = self.dumpTools.loadobj(etpCache['check_package_update']+c_hash) + if cached != None: + return cached found = False match = self.clientDbconn.atomMatch(atom) @@ -657,8 +589,12 @@ class EquoInterface(TextInterface): matched = self.atomMatch(pkg_match) del match - # cache - check_package_update_cache[atom] = (found,matched) + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['check_package_update']+c_hash,(found,matched)) + except: + pass + return found, matched @@ -717,13 +653,17 @@ class EquoInterface(TextInterface): def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), packagesFilter = True): if self.xcache: - cached = atomMatchCache.get(atom) - if cached: - try: - if (cached['matchSlot'] == matchSlot) and (cached['matchBranches'] == matchBranches) and (cached['etpRepositories'] == etpRepositories) and (cached['caseSensitive'] == caseSensitive) and (cached['etpRepositoriesOrder'] == etpRepositoriesOrder) and (cached['packagesFilter'] == packagesFilter): - return cached['result'] - except KeyError: - pass + c_hash = str( hash(atom) + \ + hash(caseSensitive) + \ + hash(matchSlot) + \ + hash(tuple(matchBranches)) + \ + hash(packagesFilter) + \ + hash(tuple(etpRepositoriesOrder)) + \ + hash(tuple(etpRepositories)) + ) + cached = self.dumpTools.loadobj(etpCache['atomMatch']+c_hash) + if cached != None: + return cached repoResults = {} for repo in etpRepositoriesOrder: @@ -751,27 +691,21 @@ class EquoInterface(TextInterface): # nothing found if not repoResults: - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = -1,1 - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(-1,1)) + except IOError: + pass return -1,1 elif len(repoResults) == 1: # one result found repo = repoResults.keys()[0] - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[repo],repo - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repo],repo)) + except IOError: + pass return repoResults[repo],repo # FIXME: consider to rewrite the code below @@ -863,14 +797,11 @@ class EquoInterface(TextInterface): for repository in etpRepositoriesOrder: if repository in conflictingTags: # found it, WE ARE DOOONE! - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[repository],repository - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repository],repository)) + except IOError: + pass return repoResults[repository],repository else: # we are done!!! @@ -879,14 +810,11 @@ class EquoInterface(TextInterface): if str(conflictingTags[x]['revision']) == str(newerRevision): reponame = x break - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[reponame],reponame - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) + except IOError: + pass return repoResults[reponame],reponame else: # we're finally done @@ -895,14 +823,11 @@ class EquoInterface(TextInterface): if conflictingEntries[x]['versiontag'] == newerTag: reponame = x break - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[reponame],reponame - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) + except IOError: + pass return repoResults[reponame],reponame else: # we are fine, the newerVersion is not one of the duplicated ones @@ -911,14 +836,11 @@ class EquoInterface(TextInterface): if packageInformation[x]['version'] == newerVersion: reponame = x break - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[reponame],reponame - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) + except IOError: + pass return repoResults[reponame],reponame else: # yeah, we're done, just return the info @@ -930,34 +852,22 @@ class EquoInterface(TextInterface): if packageInformation[x]['version'] == newerVersion: reponame = x break - atomMatchCache[atom] = {} - atomMatchCache[atom]['result'] = repoResults[reponame],reponame - atomMatchCache[atom]['matchSlot'] = matchSlot - atomMatchCache[atom]['matchBranches'] = matchBranches - atomMatchCache[atom]['caseSensitive'] = caseSensitive - atomMatchCache[atom]['packagesFilter'] = packagesFilter - atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() - atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:] + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) + except IOError: + pass return repoResults[reponame],reponame def __repository_move_clear_cache(self, repoid): - # clean world_available cache - self.dumpTools.dumpobj(etpCache['world_available'], {}) - # clean world_update cache - self.dumpTools.dumpobj(etpCache['world_update'], {}) - # clean check_update_package_cache - check_package_update_cache.clear() - self.dumpTools.dumpobj(etpCache['check_package_update'],{}) - # clear atomMatchCache - atomMatchCache.clear() - self.dumpTools.dumpobj(etpCache['atomMatch'],{}) - dbCacheStore[etpCache['dbMatch']+etpConst['dbnamerepoprefix']+repoid] = {} - dbCacheStore[etpCache['dbSearch']+etpConst['dbnamerepoprefix']+repoid] = {} - dbCacheStore[etpCache['dbInfo']+etpConst['dbnamerepoprefix']+repoid] = {} - self.dumpTools.dumpobj(etpCache['dbMatch']+etpConst['dbnamerepoprefix']+repoid,{}) - self.dumpTools.dumpobj(etpCache['dbSearch']+etpConst['dbnamerepoprefix']+repoid,{}) - self.dumpTools.dumpobj(etpCache['dbInfo']+etpConst['dbnamerepoprefix']+repoid,{}) + self.clear_dump_cache(etpCache['world_available']) + self.clear_dump_cache(etpCache['world_update']) + self.clear_dump_cache(etpCache['check_package_update']) + self.clear_dump_cache(etpCache['filter_satisfied_deps']) + self.clear_dump_cache(etpCache['atomMatch']) + self.clear_dump_cache(etpCache['dbMatch']) + self.clear_dump_cache(etpCache['dbInfo']) def addRepository(self, repodata): @@ -1076,6 +986,15 @@ class EquoInterface(TextInterface): ''' def filterSatisfiedDependencies(self, dependencies, deep_deps = False): + if self.xcache: + c_data = list(dependencies) + c_data.sort() + c_hash = str(hash(tuple(c_data))+hash(deep_deps)) + del c_data + cached = self.dumpTools.loadobj(etpCache['filter_satisfied_deps']+c_hash) + if cached != None: + return cached + unsatisfiedDeps = set() satisfiedDeps = set() @@ -1084,14 +1003,6 @@ class EquoInterface(TextInterface): depsatisfied = set() depunsatisfied = set() - ''' caching ''' - cached = filterSatisfiedDependenciesCache.get(dependency) - if cached: - if (cached['deep_deps'] == deep_deps): - unsatisfiedDeps.update(cached['depunsatisfied']) - satisfiedDeps.update(cached['depsatisfied']) - continue - ### conflict if dependency[0] == "!": testdep = dependency[1:] @@ -1125,7 +1036,6 @@ class EquoInterface(TextInterface): if (deep_deps): vcmp = self.entropyTools.entropyCompareVersions((repo_pkgver,repo_pkgtag,repo_pkgrev),(installedVer,installedTag,installedRev)) if vcmp != 0: - filterSatisfiedDependenciesCmpResults[dependency] = vcmp depunsatisfied.add(dependency) else: # check if needed is the same? @@ -1134,7 +1044,6 @@ class EquoInterface(TextInterface): depsatisfied.add(dependency) else: # not the same version installed - filterSatisfiedDependenciesCmpResults[dependency] = 10 depunsatisfied.add(dependency) if depsatisfied: @@ -1148,11 +1057,11 @@ class EquoInterface(TextInterface): unsatisfiedDeps.update(depunsatisfied) satisfiedDeps.update(depsatisfied) - ''' caching ''' - filterSatisfiedDependenciesCache[dependency] = {} - filterSatisfiedDependenciesCache[dependency]['depunsatisfied'] = depunsatisfied - filterSatisfiedDependenciesCache[dependency]['depsatisfied'] = depsatisfied - filterSatisfiedDependenciesCache[dependency]['deep_deps'] = deep_deps + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['filter_satisfied_deps']+c_hash,(unsatisfiedDeps,satisfiedDeps)) + except IOError: + pass return unsatisfiedDeps, satisfiedDeps @@ -1254,7 +1163,7 @@ class EquoInterface(TextInterface): matchcache.add(match) deptree.add((mydep[0],match)) # add match - # library breakage check + # extra library breakages check clientmatch = self.clientDbconn.atomMatch(key, matchSlot = matchslot) if clientmatch[0] != -1: broken_atoms = self.__lookup_library_breakages(match, clientmatch, deep_deps = deep_deps) @@ -1295,16 +1204,15 @@ class EquoInterface(TextInterface): return newdeptree,0 # note: newtree[0] contains possible conflicts - def clear_library_breakages_cache(self): - self.clientDbconn.clearLibraryBreakageCache() - def __lookup_library_breakages(self, match, clientmatch, deep_deps = False): # there is no need to update this cache when "match" will be installed, because at that point # clientmatch[0] will differ. - cached = self.clientDbconn.getLibraryBreakageCache(match, clientmatch[0], deep_deps) - if cached != None: - return cached + if self.xcache: + c_hash = str(hash(tuple(match))+hash(clientmatch[0])+hash(deep_deps)) + cached = self.dumpTools.loadobj(etpCache['library_breakage']+c_hash) + if cached != None: + return cached matchdb = self.openRepositoryDatabase(match[1]) reponeeded = matchdb.retrieveNeeded(match[0]) @@ -1319,11 +1227,11 @@ class EquoInterface(TextInterface): clientcontent = self.clientDbconn.retrieveContent(clientmatch[0]) clientcontent = set([x for x in clientcontent if (x.find(".so") != -1)]) clientcontent = set([x for x in clientcontent if (self.clientDbconn.isNeededAvailable(os.path.basename(x)) > 0)]) - contentdiff = clientcontent - repocontent - del repocontent, clientcontent + clientcontent -= repocontent + del repocontent search_libs = set() linker_paths = self.entropyTools.collectLinkerPaths() - for cfile in contentdiff: + for cfile in clientcontent: cpath = os.path.dirname(cfile) if cpath in linker_paths: # there's a breakage @@ -1331,11 +1239,11 @@ class EquoInterface(TextInterface): # search cfile search_libs.add(cfile) #search_libs.update(neededdiff) - del contentdiff + del clientcontent search_matches = set() for x in search_libs: y = self.clientDbconn.searchNeeded(x) - search_matches.update(y) + search_matches |= y del search_libs found_search_atoms = set() for x in search_matches: @@ -1345,22 +1253,37 @@ class EquoInterface(TextInterface): found_search_atoms.add("%s:%s" % (search_key,str(search_slot),)) if found_search_atoms: search_unsat, xxx = self.filterSatisfiedDependencies(found_search_atoms, deep_deps = deep_deps) - broken_atoms.update(search_unsat) + broken_atoms |= search_unsat - self.clientDbconn.storeLibraryBreakageCache(broken_atoms, match, clientmatch[0], deep_deps) + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['library_breakage']+c_hash,broken_atoms) + except IOError: + pass return broken_atoms def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False): + if self.xcache: + c_data = list(matched_atoms) + c_data.sort() + c_hash = str(hash(tuple(c_data))+hash(empty_deps)+hash(deep_deps)) + del c_data + cached = self.dumpTools.loadobj(etpCache['dep_tree']+c_hash) + if cached != None: + return cached + deptree = {} deptree[0] = set() - if not etpUi['quiet']: atomlen = len(matched_atoms); count = 0 + atomlen = len(matched_atoms); count = 0 matchfilter = matchContainer() for atomInfo in matched_atoms: - if not etpUi['quiet']: count += 1; self.updateProgress(":: Sorting dependencies "+str(round((float(count)/atomlen)*100,1))+"% ::", importance = 0, type = "info", back = True) + count += 1 + if (count%10 == 0) or (count == atomlen) or (count == 1): + self.updateProgress(":: Sorting dependencies "+str(round((float(count)/atomlen)*100,1))+"% ::", importance = 0, type = "info", back = True) # check if atomInfo is in matchfilter @@ -1388,6 +1311,11 @@ class EquoInterface(TextInterface): matchfilter.clear() del matchfilter + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['dep_tree']+c_hash,(deptree,0)) + except IOError: + pass return deptree,0 ''' @@ -1398,11 +1326,14 @@ class EquoInterface(TextInterface): ''' def generate_depends_tree(self, idpackages, deep = False): - ''' caching ''' - cached = generateDependsTreeCache.get(tuple(idpackages)) - if cached: - if (cached['deep'] == deep): - return cached['result'] + if self.xcache: + c_data = list(idpackages) + c_data.sort() + c_hash = str( hash(tuple(c_data)) + hash(deep) ) + del c_data + cached = self.dumpTools.loadobj(etpCache['depends_tree']+c_hash) + if cached != None: + return cached dependscache = set() dependsOk = False @@ -1492,10 +1423,11 @@ class EquoInterface(TextInterface): del tree - ''' caching ''' - generateDependsTreeCache[tuple(idpackages)] = {} - generateDependsTreeCache[tuple(idpackages)]['result'] = newtree,0 - generateDependsTreeCache[tuple(idpackages)]['deep'] = deep + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['depends_tree']+c_hash,(newtree,0)) + except IOError: + pass return newtree,0 # treeview is used to show deps while tree is used to run the dependency code. def list_repo_categories(self): @@ -1534,20 +1466,14 @@ class EquoInterface(TextInterface): if self.xcache: repo_digest = self.all_repositories_checksum() client_digest = self.clientDbconn.tablesChecksum() - disk_cache = self.dumpTools.loadobj(etpCache['world_available']) - try: - if disk_cache != None: - if disk_cache['repo_digest'] == repo_digest and \ - disk_cache['branch'] == branch and \ - disk_cache['client_digest'] == client_digest and \ - disk_cache['etpRepositories_keys'] == etpRepositories.keys() and \ - disk_cache['etpRepositoriesOrder'] == etpRepositoriesOrder: - return disk_cache['available'] - except: - try: - self.dumpTools.dumpobj(etpCache['world_available'], {}) - except IOError: - pass + c_hash = str(hash(repo_digest) + \ + hash(branch) + \ + hash(client_digest) + \ + hash(tuple(etpRepositories)) + \ + hash(tuple(etpRepositoriesOrder))) + disk_cache = self.dumpTools.loadobj(etpCache['world_available']+c_hash) + if disk_cache != None: + return disk_cache available = set() self.setTotalCycles(len(etpRepositories)) @@ -1576,59 +1502,44 @@ class EquoInterface(TextInterface): if self.xcache: try: - mycache = {} - mycache['repo_digest'] = repo_digest - mycache['client_digest'] = client_digest - mycache['available'] = available.copy() - mycache['branch'] = branch - mycache['etpRepositories_keys'] = etpRepositories.keys()[:] - mycache['etpRepositoriesOrder'] = etpRepositoriesOrder[:] - # save cache - self.dumpTools.dumpobj(etpCache['world_available'], mycache) - mycache.clear() - del mycache - except: - self.dumpTools.dumpobj(etpCache['world_available'], {}) - + disk_cache = self.dumpTools.dumpobj(etpCache['world_available']+c_hash,available) + except IOError: + pass return available def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None): if self.xcache: if db_digest == None: db_digest = self.clientDbconn.tablesChecksum() - disk_cache = self.dumpTools.loadobj(etpCache['world_update']) - try: - if disk_cache != None: - if disk_cache['db_digest'] == db_digest and \ - disk_cache['empty_deps'] == empty_deps and \ - disk_cache['etpRepositories_keys'] == etpRepositories.keys() and \ - disk_cache['etpRepositoriesOrder'] == etpRepositoriesOrder and \ - disk_cache['branch'] == branch: - return disk_cache['update'],disk_cache['remove'],disk_cache['fine'] - except: - try: - self.dumpTools.dumpobj(etpCache['world_update'], {}) - except IOError: - pass + c_hash = str( hash(db_digest) + \ + hash(empty_deps) + \ + hash(tuple(etpRepositories)) + \ + hash(tuple(etpRepositoriesOrder)) + \ + hash(branch) + ) + disk_cache = self.dumpTools.loadobj(etpCache['world_update']+c_hash) + if disk_cache != None: + return disk_cache def calculate_world_updates(self, empty_deps = False, branch = etpConst['branch']): - update = set() - remove = set() - fine = set() - db_digest = self.clientDbconn.tablesChecksum() cached = self.get_world_update_cache(empty_deps = empty_deps, branch = branch, db_digest = db_digest) if cached != None: return cached + update = set() + remove = set() + fine = set() + # get all the installed packages idpackages = self.clientDbconn.listAllIdpackages() maxlen = len(idpackages) count = 0 for idpackage in idpackages: count += 1 - self.updateProgress("Calculating world dependencies", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::") + if (count%10 == 0) or (count == maxlen) or (count == 1): + self.updateProgress("Calculating world dependencies", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::") tainted = False myscopedata = self.clientDbconn.getScopeData(idpackage) #atom = myscopedata[0] @@ -1693,23 +1604,17 @@ class EquoInterface(TextInterface): del idpackages - if self.xcache and (etpConst['uid'] == 0): + if self.xcache: + c_hash = str( hash(db_digest) + \ + hash(empty_deps) + \ + hash(tuple(etpRepositories)) + \ + hash(tuple(etpRepositoriesOrder)) + \ + hash(branch) + ) try: - mycache = {} - mycache['db_digest'] = db_digest - mycache['update'] = update.copy() - mycache['remove'] = remove.copy() - mycache['fine'] = fine.copy() - mycache['empty_deps'] = empty_deps - mycache['branch'] = branch - mycache['etpRepositories_keys'] = etpRepositories.keys()[:] - mycache['etpRepositoriesOrder'] = etpRepositoriesOrder[:] - # save cache - self.dumpTools.dumpobj(etpCache['world_update'], mycache) - mycache.clear() - del mycache - except: - self.dumpTools.dumpobj(etpCache['world_update'], {}) + self.dumpTools.dumpobj(etpCache['world_update']+c_hash, (update, remove, fine)) + except IOError: + pass return update, remove, fine def is_match_masked(self, match): @@ -2254,7 +2159,7 @@ class PackageInterface: os.mkdir(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']) # now unpack for real xpakPath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakfilename'] - + if (self.infoDict['smartpackage']): # we need to get the .xpak from database xdbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository']) @@ -2293,12 +2198,7 @@ class PackageInterface: def __remove_package(self): # clear on-disk cache - generateDependsTreeCache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],generateDependsTreeCache) - check_package_update_cache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['check_package_update'],{}) - # remove security advisories cache - self.Entropy.dumpTools.dumpobj(etpCache['advisories'],{}) + self.__clear_cache() self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: "+str(self.infoDict['removeatom'])) @@ -2313,10 +2213,8 @@ class PackageInterface: self.__remove_package_from_database() # Handle gentoo database - if (etpConst['gentoo-compat']): # FIXME: remove dep_striptag asap - gentooAtom = self.Entropy.entropyTools.dep_striptag( - self.Entropy.entropyTools.remove_tag(self.infoDict['removeatom']) - ) + if (etpConst['gentoo-compat']): + gentooAtom = self.Entropy.entropyTools.remove_tag(self.infoDict['removeatom']) self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing from Portage: "+str(gentooAtom)) self.__remove_package_from_gentoo_database(gentooAtom) del gentooAtom @@ -2501,33 +2399,20 @@ class PackageInterface: ''' def __remove_package_from_database(self): self.error_on_not_prepared() - - find = False - disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_update']) - if disk_cache != None: - try: - slot = self.Entropy.clientDbconn.retrieveSlot(self.infoDict['removeidpackage']) - key = self.Entropy.entropyTools.dep_getkey(self.infoDict['removeatom']) - matched = self.Entropy.atomMatch(key, matchSlot = slot) - if matched[0] != -1: - dbconn = self.Entropy.openRepositoryDatabase(matched[1]) - atom = dbconn.retrieveAtom(matched[0]) - cached_item = (atom,matched) - if cached_item in disk_cache['update']: - find = True - except: - self.Entropy.dumpTools.dumpobj(etpCache['world_update'], {}) - self.Entropy.clientDbconn.removePackage(self.infoDict['removeidpackage']) - - if find: - disk_cache['update'].remove(cached_item) - db_digest = self.Entropy.clientDbconn.tablesChecksum() - disk_cache['db_digest'] = db_digest - self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache) - return 0 + def __clear_cache(self): + self.Entropy.clear_dump_cache(etpCache['advisories']) + self.Entropy.clear_dump_cache(etpCache['filter_satisfied_deps']) + self.Entropy.clear_dump_cache(etpCache['depends_tree']) + self.Entropy.clear_dump_cache(etpCache['check_package_update']) + self.Entropy.clear_dump_cache(etpCache['dep_tree']) + self.Entropy.clear_dump_cache(etpCache['world_available']) + self.Entropy.clear_dump_cache(etpCache['world_update']) + self.Entropy.clear_dump_cache(etpCache['dbMatch']) + self.Entropy.clear_dump_cache(etpCache['dbInfo']) + ''' @description: install unpacked files, update database and also update gentoo db if requested @output: 0 = all fine, >0 = error! @@ -2535,12 +2420,7 @@ class PackageInterface: def __install_package(self): # clear on-disk cache - generateDependsTreeCache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],{}) - check_package_update_cache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['check_package_update'],{}) - # clear advisories cache - self.Entropy.dumpTools.dumpobj(etpCache['advisories'],{}) + self.__clear_cache() self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing package: "+str(self.infoDict['atom'])) @@ -2704,40 +2584,6 @@ class PackageInterface: ctime = self.Entropy.entropyTools.getCurrentUnixTime() self.Entropy.clientDbconn.setDateCreation(idpk, str(ctime)) - # update world calculation cache - disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_update']) - if disk_cache != None: - try: - atom = self.Entropy.clientDbconn.retrieveAtom(idpk) - slot = self.Entropy.clientDbconn.retrieveSlot(idpk) - cached_item = (atom,(self.infoDict['idpackage'],self.infoDict['repository'])) - if cached_item in disk_cache['update']: - disk_cache['update'].remove(cached_item) - db_digest = self.Entropy.clientDbconn.tablesChecksum() - disk_cache['db_digest'] = db_digest - self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache) - else: - # add it, because can happen that a user firstly removes the package, then - # decides to install a specific version which is not the latest - key = self.Entropy.entropyTools.dep_getkey(atom) - match = self.Entropy.atomMatch(key, matchSlot = slot) - if (match[0] != -1) and (match != cached_item[1]): - ldbconn = self.Entropy.openRepositoryDatabase(match[1]) - latest_atom = ldbconn.retrieveAtom(match[0]) - disk_cache['update'].add(latest_atom,match) - # if you came from databaseTools.validateDatabase, this is the reason: - # db_digest is the checksum of baseinfo and extrainfo, if you add dependstable - # then you'll have to move this at the bottom of this function because, if you - # read below, dependstable will be updated, so checksum won't never match, - # the same for the condition above - db_digest = self.Entropy.clientDbconn.tablesChecksum() - disk_cache['db_digest'] = db_digest - self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache) - except Exception,e: - # invalidate cache - self.Entropy.dumpTools.dumpobj(etpCache['world_update'],{}) - print "DEBUG: please REPORT %s: %s" % (str(Exception),str(e),) - # add idpk to the installedtable self.Entropy.clientDbconn.removePackageFromInstalledTable(idpk) self.Entropy.clientDbconn.addPackageToInstalledTable(idpk,self.infoDict['repository']) @@ -3536,7 +3382,7 @@ class FileUpdatesInterface: # store data try: self.Entropy.dumpTools.dumpobj(etpCache['configfiles'],scandata) - except: + except IOError: pass return scandata @@ -3590,7 +3436,7 @@ class FileUpdatesInterface: scandata[index] = mydata.copy() try: self.Entropy.dumpTools.dumpobj(etpCache['configfiles'],scandata) - except: + except IOError: pass def remove_from_cache(self, sd, key): @@ -3800,7 +3646,7 @@ class RepoInterface: def clear_repository_cache(self, repo): self.__validate_repository_id(repo) - self.Entropy.dumpTools.dumpobj(etpCache['dbInfo']+repo,{}) + self.Entropy.clear_dump_cache(etpCache['dbInfo']) # this function can be reimplemented def download_item(self, item, repo, cmethod = None): @@ -3816,25 +3662,6 @@ class RepoInterface: return False return True - def close_transactions(self): - - if not self.dbupdated: - return - - # safely clean ram caches - atomMatchCache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['atomMatch'],{}) - generateDependsTreeCache.clear() - self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],{}) - for dbinfo in dbCacheStore: - dbCacheStore[dbinfo].clear() - self.Entropy.dumpTools.dumpobj(dbinfo,{}) - - # clean resume caches - self.Entropy.dumpTools.dumpobj(etpCache['install'],{}) - self.Entropy.dumpTools.dumpobj(etpCache['world'],{}) - self.Entropy.dumpTools.dumpobj(etpCache['remove'],[]) - def sync(self): # close them @@ -4008,21 +3835,19 @@ class RepoInterface: ) dbconn = self.Entropy.openRepositoryDatabase(repo) dbconn.createAllIndexes() - self.Entropy.clientDbconn.createAllIndexes() + try: # client db can be absent + self.Entropy.clientDbconn.createAllIndexes() + except: + pass self.Entropy.cycleDone() - self.close_transactions() - # keep them closed self.Entropy.closeAllRepositoryDatabases() # clean caches if self.dbupdated: - try: # if no client db exists, we can barely ignore this - self.Entropy.clear_library_breakages_cache() - except: - pass + self.Entropy.clear_dump_cache(etpCache['library_breakage']) self.Entropy.generate_cache(depcache = True, configcache = False) # update Security Advisories self.Entropy.Security.fetch_advisories() @@ -6399,7 +6224,7 @@ class SecurityInterface: def clear(self, xcache = False): self.adv_metadata = None if xcache: - self.Entropy.dumpTools.dumpobj(etpCache['advisories'], {}) + self.Entropy.clear_dump_cache(etpCache['advisories']) def __get_advisories_cache(self): @@ -6407,31 +6232,25 @@ class SecurityInterface: return self.adv_metadata if self.Entropy.xcache: - adv_cache = self.Entropy.dumpTools.loadobj(etpCache['advisories']) dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir']) - try: - if adv_cache['systemroot'] == etpConst['systemroot'] and \ - adv_cache['branch'] == etpConst['branch'] and \ - adv_cache['data'] and \ - adv_cache['checksum'] == dir_checksum: - self.adv_metadata = adv_cache['data'].copy() - return adv_cache['data'] - except: - try: - self.Entropy.dumpTools.dumpobj(etpCache['advisories'], {}) - except IOError: - pass + c_hash = str( hash(etpConst['branch']) + \ + hash(dir_checksum) + \ + hash(etpConst['systemroot']) + ) + adv_metadata = self.Entropy.dumpTools.loadobj(etpCache['advisories']+c_hash) + if adv_metadata != None: + self.adv_metadata = adv_metadata.copy() + return self.adv_metadata def __set_advisories_cache(self, adv_metadata): if self.Entropy.xcache: dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir']) - adv_cache = {} - adv_cache['branch'] = etpConst['branch'] - adv_cache['data'] = adv_metadata - adv_cache['checksum'] = dir_checksum - adv_cache['systemroot'] = etpConst['systemroot'] + c_hash = str( hash(etpConst['branch']) + \ + hash(dir_checksum) + \ + hash(etpConst['systemroot']) + ) try: - adv_cache = self.Entropy.dumpTools.dumpobj(etpCache['advisories'],adv_cache) + self.Entropy.dumpTools.dumpobj(etpCache['advisories']+c_hash,adv_metadata) except IOError: pass diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index a50b0f711..6c3e8eb81 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -304,16 +304,6 @@ CREATE TABLE binkeywords ( idkeyword INTEGER ); -CREATE TABLE library_breakages ( - idbreak INTEGER PRIMARY KEY, - repoidpackage INTEGER, - reponame VARCHAR, - idpackage INTEGER, - deep INTEGER, - repoorderck INTEGER, - atom VARCHAR -); - """ # ETP_ARCH_CONST setup @@ -355,26 +345,19 @@ ETP_LOGPRI_ERROR = "[ ERROR ]" etpCache = { 'configfiles': 'scanfs', # used to store information about files that should be merged using "equo conf merge" 'dbMatch': 'cache_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (dep solving) - 'dbSearch': 'search_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (searches) 'dbInfo': 'info_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (info retrival) - 'atomMatch': 'atomMatchCache', # used to store info about repository dependencies solving - 'generateDependsTree': 'generateDependsTreeCache', # used to store info about removal dependencies + 'atomMatch': 'atom_match_', # used to store info about repository dependencies solving 'install': 'resume_install', # resume cache (install) 'remove': 'resume_remove', # resume cache (remove) 'world': 'resume_world', # resume cache (world) - 'world_update': 'world_cache', - 'world_available': 'available_cache', - 'check_package_update': 'package_update', - 'advisories': 'advisories_cache' -} - -# byte sizes of disk caches -etpCacheSizes = { - 'dbMatch': 3000000, # bytes - 'dbInfo': 6000000, # bytes - 'dbSearch': 2000000, # bytes - 'atomMatch': 3000000, # bytes - 'generateDependsTree': 3000000, # bytes + 'world_update': 'world_cache_', + 'world_available': 'available_cache_', + 'check_package_update': 'package_update_', + 'advisories': 'advisories_cache_', + 'dep_tree': 'dep_tree_', + 'depends_tree': 'depends_tree_', + 'filter_satisfied_deps': 'filter_satisfied_deps_', + 'library_breakage': 'library_breakage_' } # ahahaha @@ -401,60 +384,20 @@ etpRSSMessages = { etpHandlers = {} # CACHING dictionaries -dbCacheStore = {} -atomMatchCache = {} -atomClientMatchCache = {} -generateDependsTreeCache = {} idpackageValidatorCache = {} -filterSatisfiedDependenciesCache = {} -filterSatisfiedDependenciesCmpResults = {} -check_package_update_cache = {} -ververifyCache = {} -isjustnameCache = {} -catpkgsplitCache = {} -dep_striptagCache = {} -dep_getkeyCache = {} -dep_getcpvCache = {} -dep_getslotCache = {} -dep_gettagCache = {} -removePackageOperatorsCache = {} -compareVersionsCache = {} -getNewerVersionCache = {} -getEntropyNewerVersionCache = {} linkerPaths = set() # repository atoms updates digest cache repositoryUpdatesDigestCache_db = {} repositoryUpdatesDigestCache_disk = {} - fetch_repository_if_not_available_cache = {} repo_error_messages_cache = set() ### Application disk cache def const_resetCache(): - for item in dbCacheStore: - dbCacheStore[item].clear() - atomMatchCache.clear() - atomClientMatchCache.clear() - generateDependsTreeCache.clear() idpackageValidatorCache.clear() - filterSatisfiedDependenciesCache.clear() - filterSatisfiedDependenciesCmpResults.clear() - ververifyCache.clear() - isjustnameCache.clear() - catpkgsplitCache.clear() - dep_striptagCache.clear() - dep_getkeyCache.clear() - dep_getcpvCache.clear() - dep_getslotCache.clear() - dep_gettagCache.clear() - removePackageOperatorsCache.clear() - compareVersionsCache.clear() - getNewerVersionCache.clear() - getEntropyNewerVersionCache.clear() linkerPaths.clear() repositoryUpdatesDigestCache_db.clear() repositoryUpdatesDigestCache_disk.clear() - check_package_update_cache.clear() fetch_repository_if_not_available_cache.clear() repo_error_messages_cache.clear() diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 5522ada4d..662d8ee81 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -545,11 +545,6 @@ def isjustpkgname(mypkg): def ververify(myverx, silent=1): - cached = ververifyCache.get(myverx) - if cached != None: - return cached - - ververifyCache[myverx] = 1 myver = myverx[:] if myver.endswith("*"): myver = myver[:len(myver)-1] @@ -558,7 +553,6 @@ def ververify(myverx, silent=1): else: if not silent: print "!!! syntax error in version: %s" % myver - ververifyCache[myverx] = 0 return 0 @@ -582,15 +576,9 @@ def isjustname(mypkg): 2) 1 if it is """ - cached = isjustnameCache.get(mypkg) - if cached != None: - return cached - - isjustnameCache[mypkg] = 1 myparts = mypkg.split('-') for x in myparts: if ververify(x): - isjustnameCache[mypkg] = 0 return 0 return 1 @@ -635,10 +623,6 @@ def catpkgsplit(mydata,silent=1): an InvalidData Exception will be thrown """ - cached = catpkgsplitCache.get(mydata) - if cached != None: - return cached - # Categories may contain a-zA-z0-9+_- but cannot start with - mysplit=mydata.split("/") p_split=None @@ -649,10 +633,8 @@ def catpkgsplit(mydata,silent=1): retval=[mysplit[0]] p_split=pkgsplit(mysplit[1],silent=silent) if not p_split: - catpkgsplitCache[mydata] = None return None retval.extend(p_split) - catpkgsplitCache[mydata] = retval return retval def pkgsplit(mypkg,silent=1): @@ -698,23 +680,6 @@ def pkgsplit(mypkg,silent=1): else: return None -# FIXME: deprecated, use remove_tag - will be removed soonly -def dep_striptag(mydepx): - - cached = dep_striptagCache.get(mydepx) - if cached != None: - return cached - - mydep = mydepx[:] - if not (isjustname(mydep)): - if mydep.split("-")[len(mydep.split("-"))-1].startswith("t"): # tag -> remove - tag = mydep.split("-")[len(mydep.split("-"))-1] - mydep = mydep[:len(mydep)-len(tag)-1] - - dep_striptagCache[mydepx] = mydep - return mydep - - def dep_getkey(mydepx): """ Return the category/package-name of a depstring. @@ -728,26 +693,18 @@ def dep_getkey(mydepx): @rtype: String @return: The package category/package-version """ - - cached = dep_getkeyCache.get(mydepx) - if cached != None: - return cached - + mydep = mydepx[:] - mydep = dep_striptag(mydep) mydep = remove_tag(mydep) - + mydep = dep_getcpv(mydep) if mydep and isspecific(mydep): - mysplit = catpkgsplit(mydep) - if not mysplit: - dep_getkeyCache[mydepx] = mydep - return mydep - dep_getkeyCache[mydepx] = mysplit[0] + "/" + mysplit[1] - return mysplit[0] + "/" + mysplit[1] + mysplit = catpkgsplit(mydep) + if not mysplit: + return mydep + return mysplit[0] + "/" + mysplit[1] else: - dep_getkeyCache[mydepx] = mydep - return mydep + return mydep def dep_getcpv(mydep): @@ -764,10 +721,6 @@ def dep_getcpv(mydep): @return: The depstring with the operator removed """ - cached = dep_getcpvCache.get(mydep) - if cached != None: - return cached - if mydep and mydep[0] == "*": mydep = mydep[1:] if mydep and mydep[-1] == "*": @@ -782,7 +735,6 @@ def dep_getcpv(mydep): if colon != -1: mydep = mydep[:colon] - dep_getcpvCache[mydep] = mydep return mydep @@ -800,18 +752,12 @@ def dep_getslot(dep): @return: The slot """ - cached = dep_getslotCache.get(dep) - if cached != None: - return cached - colon = dep.rfind(":") if colon != -1: mydep = dep[colon+1:] rslt = remove_tag(mydep) - dep_getslotCache[dep] = rslt return rslt - dep_getslotCache[dep] = None return None def remove_slot(mydep): @@ -852,26 +798,16 @@ def dep_gettag(dep): """ - cached = dep_gettagCache.get(dep) - if cached != None: - return cached - colon = dep.rfind("#") if colon != -1: mydep = dep[colon+1:] rslt = remove_slot(mydep) - dep_gettagCache[dep] = rslt return rslt - dep_gettagCache[dep] = None return None def removePackageOperators(atom): - cached = removePackageOperatorsCache.get(atom) - if cached != None: - return cached - original = atom if atom[0] == ">" or atom[0] == "<": atom = atom[1:] @@ -880,7 +816,6 @@ def removePackageOperators(atom): if atom[0] == "~": atom = atom[1:] - removePackageOperatorsCache[original] = atom return atom # Version compare function taken from portage_versions.py @@ -892,12 +827,7 @@ suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1} endversion_keys = ["pre", "p", "alpha", "beta", "rc"] def compareVersions(ver1, ver2): - cached = compareVersionsCache.get(tuple([ver1,ver2])) - if cached != None: - return cached - if ver1 == ver2: - compareVersionsCache[tuple([ver1,ver2])] = 0 return 0 #mykey=ver1+":"+ver2 match1 = ver_regexp.match(ver1) @@ -939,13 +869,10 @@ def compareVersions(ver1, ver2): for i in range(0, max(len(list1), len(list2))): if len(list1) <= i: - compareVersionsCache[tuple([ver1,ver2])] = -1 return -1 elif len(list2) <= i: - compareVersionsCache[tuple([ver1,ver2])] = 1 return 1 elif list1[i] != list2[i]: - compareVersionsCache[tuple([ver1,ver2])] = list1[i] - list2[i] return list1[i] - list2[i] # main version is equal, so now compare the _suffix part @@ -962,7 +889,6 @@ def compareVersions(ver1, ver2): else: s2 = suffix_regexp.match(list2[i]).groups() if s1[0] != s2[0]: - compareVersionsCache[tuple([ver1,ver2])] = suffix_value[s1[0]] - suffix_value[s2[0]] return suffix_value[s1[0]] - suffix_value[s2[0]] if s1[1] != s2[1]: # it's possible that the s(1|2)[1] == '' @@ -971,7 +897,6 @@ def compareVersions(ver1, ver2): except ValueError: r1 = 0 try: r2 = int(s2[1]) except ValueError: r2 = 0 - compareVersionsCache[tuple([ver1,ver2])] = r1 - r2 return r1 - r2 # the suffix part is equal to, so finally check the revision @@ -983,7 +908,6 @@ def compareVersions(ver1, ver2): r2 = int(match2.group(10)) else: r2 = 0 - compareVersionsCache[tuple([ver1,ver2])] = r1 - r2 return r1 - r2 ''' @@ -1020,17 +944,12 @@ def entropyCompareVersions(listA,listB): @description: reorder a version list @input versionlist: a list @output: the ordered list - XXX DEPRECATED ''' def getNewerVersion(versions): if len(versions) == 1: return versions - cached = getNewerVersionCache.get(tuple(versions)) - if cached != None: - return cached - versionlist = versions[:] rc = False @@ -1052,7 +971,6 @@ def getNewerVersion(versions): if (not change): rc = True - getNewerVersionCache[tuple(versions)] = versionlist return versionlist @@ -1065,10 +983,6 @@ def getEntropyNewerVersion(versions): if len(versions) == 1: return versions - cached = getEntropyNewerVersionCache.get(tuple(versions)) - if cached != None: - return cached - myversions = versions[:] # ease the work @@ -1091,7 +1005,6 @@ def getEntropyNewerVersion(versions): if (not change): rc = True - getEntropyNewerVersionCache[tuple(versions)] = myversions return myversions ''' @@ -2221,9 +2134,10 @@ def collectLinkerPaths(): f = open(etpConst['systemroot']+"/etc/ld.so.conf","r") paths = f.readlines() for path in paths: - if path.strip(): + path = path.strip() + if path: if path[0] == "/": - ldpaths.add(os.path.normpath(path.strip())) + ldpaths.add(os.path.normpath(path)) f.close() except: pass diff --git a/spritz/src/entropyapi.py b/spritz/src/entropyapi.py index ee7a323d9..96de479c9 100644 --- a/spritz/src/entropyapi.py +++ b/spritz/src/entropyapi.py @@ -200,4 +200,3 @@ class GuiUrlFetcher(urlFetcher): EquoConnection = Equo() -EquoConnection.load_cache() \ No newline at end of file diff --git a/spritz/src/spritz.py b/spritz/src/spritz.py index d06630303..35ba7f5fe 100644 --- a/spritz/src/spritz.py +++ b/spritz/src/spritz.py @@ -73,7 +73,6 @@ class SpritzController(Controller): def quit(self, widget=None, event=None ): - EquoConnection.save_cache() ''' Main destroy Handler ''' gtkEventThread.doQuit() if self.isWorking: @@ -860,7 +859,6 @@ class SpritzApplication(SpritzController,SpritzGUI): # regenerate packages information self.etpbase.clearPackages() self.setupSpritz() - self.Equo.save_cache() self.endWorking() #self.progress.hide() if quit: