From 84cbf8b40962470da193238ccfab10c19b01e4b2 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Mon, 11 Feb 2008 17:47:35 +0000 Subject: [PATCH] improved caching validation git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1183 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 2 +- libraries/databaseTools.py | 299 ++++++++++++++++-------------- libraries/entropy.py | 340 +++++++++++++++++++--------------- libraries/entropyConstants.py | 8 +- 4 files changed, 349 insertions(+), 300 deletions(-) diff --git a/TODO b/TODO index a394fc52f..452e8821d 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,6 @@ TODO list: + - cache update instead of trashing each time, look for caches that can be kept if bound to specific checksums - split RDEPEND and PDEPEND - - improve dependencies order resolution - migrate server code to ServerInterface [] write a tool that helps keeping packages updated (also supporting injected ones) [] complete reagent spm interface diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index fc92d7910..697973b13 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -53,7 +53,7 @@ def openClientDatabase(xcache = True, generate = False, indexing = True): if (not generate) and (not os.path.isfile(etpConst['etpdatabaseclientfilepath'])): raise exceptionTools.SystemDatabaseError("SystemDatabaseError: system database not found. Either does not exist or corrupted.") else: - conn = etpDatabase(readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = 'client', xcache = xcache, indexing = indexing) + conn = etpDatabase(readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = etpConst['clientdbid'], xcache = xcache, indexing = indexing) # validate database if not generate: conn.validateDatabase() @@ -122,7 +122,7 @@ def backupClientDatabase(): class etpDatabase(TextInterface): - def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False, xcache = False, dbname = 'etpdb', indexing = True): + def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], indexing = True): self.readOnly = readOnly self.noUpload = noUpload @@ -137,7 +137,7 @@ class etpDatabase(TextInterface): self.dbFile = dbFile # no caching for non root and server connections - if (self.dbname == 'etpdb') or (etpConst['uid'] != 0): + if (self.dbname == etpConst['serverdbid']) or (etpConst['uid'] != 0): self.xcache = False # create connection @@ -847,7 +847,7 @@ class etpDatabase(TextInterface): ) ) except: - if self.dbname == "client": # force only for client database + if self.dbname == etpConst['clientdbid']: # force only for client database if not self.doesTableExist("counters"): self.createCountersTable() else: @@ -859,7 +859,7 @@ class etpDatabase(TextInterface): idpackage, ) ) - elif self.dbname == "etpdb": + elif self.dbname == etpConst['serverdbid']: raise # on disk size @@ -1179,12 +1179,12 @@ class etpDatabase(TextInterface): # counter self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage) except: - if self.dbname == "client": + if self.dbname == etpConst['clientdbid']: if not self.doesTableExist("counters"): self.createCountersTable() else: raise - elif self.dbname == "etpdb": + elif self.dbname == etpConst['serverdbid']: raise try: # on disk sizes @@ -1455,7 +1455,7 @@ class etpDatabase(TextInterface): try: self.cursor.execute('UPDATE counters SET counter = (?) WHERE idpackage = (?)', (counter,idpackage,)) except: - if self.dbname == "client": + if self.dbname == etpConst['clientdbid']: if not self.doesTableExist("counters"): self.createCountersTable() else: @@ -1858,14 +1858,15 @@ class etpDatabase(TextInterface): def do_clear(name): dump_file = os.path.join(etpConst['dumpstoragedir'],name) os.system("rm -rf %s*.dmp" % (dump_file,) ) - do_clear(etpCache['dbInfo']) - do_clear(etpCache['dbMatch']) + do_clear(etpCache['dbInfo']+self.dbname) + do_clear(etpCache['dbMatch']+self.dbname) + do_clear(etpCache['dbSearch']+self.dbname) def fetchInfoCache(self, idpackage, function, extra_hash = 0): if (self.xcache): - c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash ) + c_hash = str( hash(int(idpackage)) + hash(function) + extra_hash ) try: - cached = dumpTools.loadobj(etpCache['dbInfo']+c_hash) + cached = dumpTools.loadobj(etpCache['dbInfo']+self.dbname+c_hash) if cached != None: return cached except EOFError: @@ -1874,9 +1875,26 @@ class etpDatabase(TextInterface): def storeInfoCache(self, idpackage, function, info_cache_data, extra_hash = 0): if (self.xcache): - c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash ) + c_hash = str( hash(int(idpackage)) + hash(function) + extra_hash ) try: - dumpTools.dumpobj(etpCache['dbInfo']+c_hash,info_cache_data) + dumpTools.dumpobj(etpCache['dbInfo']+self.dbname+c_hash,info_cache_data) + except IOError: + pass + + def fetchSearchCache(self, cache_hash): + if self.xcache: + try: + cached = dumpTools.loadobj(etpCache['dbSearch']+self.dbname+cache_hash) + if cached != None: + return cached + except EOFError: + pass + + + def storeSearchCache(self, cache_hash, cache_data): + if self.xcache: + try: + dumpTools.dumpobj(etpCache['dbSearch']+self.dbname+cache_hash,cache_data) except IOError: pass @@ -2023,7 +2041,7 @@ class etpDatabase(TextInterface): if mycounter: counter = mycounter[0] except: - if self.dbname == "client": + if self.dbname == etpConst['clientdbid']: if not self.doesTableExist("counters"): self.createCountersTable() else: @@ -2371,18 +2389,18 @@ class etpDatabase(TextInterface): self.storeInfoCache(idpackage,'retrieveSlot',ver) return ver - + def retrieveVersionTag(self, idpackage): - cache = self.fetchInfoCache(idpackage,'retrieveVersionTag') - if cache != None: return cache + cache = self.fetchInfoCache(idpackage,'retrieveVersionTag') + if cache != None: return cache - self.cursor.execute('SELECT "versiontag" FROM baseinfo WHERE idpackage = (?)', (idpackage,)) - ver = self.cursor.fetchone()[0] + self.cursor.execute('SELECT "versiontag" FROM baseinfo WHERE idpackage = (?)', (idpackage,)) + ver = self.cursor.fetchone()[0] + + self.storeInfoCache(idpackage,'retrieveVersionTag',ver) + return ver - self.storeInfoCache(idpackage,'retrieveVersionTag',ver) - return ver - def retrieveMirrorInfo(self, mirrorname): self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) @@ -2718,27 +2736,34 @@ class etpDatabase(TextInterface): def searchProvide(self, keyword, slot = None, tag = None, branch = None): - self.cursor.execute('SELECT idpackage FROM provide WHERE atom = (?)', (keyword,)) - idpackage = self.cursor.fetchone() - if not idpackage: - return () + c_hash = str( hash("searchProvide") + hash(keyword) + hash(slot) + hash(branch) + hash(tag) ) + cache = self.fetchSearchCache(c_hash) + if cache != None: return cache - slotstring = '' + self.cursor.execute('SELECT idpackage FROM provide WHERE atom = (?)', (keyword,)) + idpackage = self.cursor.fetchone() + if not idpackage: + return () + + slotstring = '' searchkeywords = [idpackage[0]] - if slot: + if slot: searchkeywords.append(slot) - slotstring = ' and slot = (?)' - tagstring = '' - if tag: + slotstring = ' and slot = (?)' + tagstring = '' + if tag: searchkeywords.append(tag) - tagstring = ' and versiontag = (?)' - branchstring = '' - if branch: + tagstring = ' and versiontag = (?)' + branchstring = '' + if branch: searchkeywords.append(branch) - branchstring = ' and branch = (?)' + branchstring = ' and branch = (?)' - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = (?)'+slotstring+tagstring+branchstring, searchkeywords) - return self.cursor.fetchall() + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = (?)'+slotstring+tagstring+branchstring, searchkeywords) + + results = self.cursor.fetchall() + self.storeSearchCache(c_hash,results) + return results def searchPackagesByDescription(self, keyword): self.cursor.execute('SELECT idpackage FROM extrainfo WHERE LOWER(description) LIKE (?)', ("%"+keyword.lower()+"%",)) @@ -2774,23 +2799,29 @@ class etpDatabase(TextInterface): return result def searchPackagesByName(self, keyword, sensitive = False, branch = None): - + + c_hash = str( hash("searchPackagesByName") + hash(keyword) + hash(sensitive) + hash(branch) ) + cache = self.fetchSearchCache(c_hash) + if cache != None: return cache + if sensitive: searchkeywords = [keyword] else: searchkeywords = [keyword.lower()] - branchstring = '' - if branch: + branchstring = '' + if branch: searchkeywords.append(branch) - branchstring = ' and branch = (?)' - - if (sensitive): - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords) - else: - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) - - results = self.cursor.fetchall() - return results + branchstring = ' and branch = (?)' + + if sensitive: + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords) + else: + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) + + results = self.cursor.fetchall() + + self.storeSearchCache(c_hash,results) + return results def searchPackagesByCategory(self, keyword, like = False, branch = None): @@ -2811,37 +2842,41 @@ class etpDatabase(TextInterface): return results def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None): - - # get category id - idcat = -1 - self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) - idcat = self.cursor.fetchone() - if not idcat: - return () - else: - idcat = idcat[0] + + c_hash = str( hash("searchPackagesByNameAndCategory") + hash(name) + hash(category) + hash(sensitive) + hash(branch) ) + cache = self.fetchSearchCache(c_hash) + if cache != None: return cache + + # get category id + idcat = -1 + self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) + idcat = self.cursor.fetchone() + if not idcat: + return () + else: + idcat = idcat[0] searchkeywords = [] if sensitive: searchkeywords.append(name) else: searchkeywords.append(name.lower()) - + searchkeywords.append(idcat) - - branchstring = '' - if branch: + + branchstring = '' + if branch: searchkeywords.append(branch) - branchstring = ' and branch = (?)' + branchstring = ' and branch = (?)' - if (sensitive): - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?) AND idcategory = (?) '+branchstring, searchkeywords) - else: - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory = (?) '+branchstring, searchkeywords) - - results = self.cursor.fetchall() + if (sensitive): + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?) AND idcategory = (?) '+branchstring, searchkeywords) + else: + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory = (?) '+branchstring, searchkeywords) - return results + results = self.cursor.fetchall() + self.storeSearchCache(c_hash,results) + return results def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False): @@ -2907,8 +2942,13 @@ class etpDatabase(TextInterface): def listAllBranches(self): + cache = self.fetchInfoCache(0,'listAllBranches') + if cache != None: return cache + self.cursor.execute('SELECT branch FROM baseinfo') results = self.fetchall2set(self.cursor.fetchall()) + + self.storeInfoCache(0,'listAllBranches',results) return results def listIdPackagesInIdcategory(self,idcategory): @@ -3201,57 +3241,33 @@ class etpDatabase(TextInterface): self.createUseflagsIndex() def createNeededIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return + if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( idneeded,library )') self.commitChanges() def createUseflagsIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return + if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( idflag,flagname )') self.commitChanges() def createContentIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return + if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex ON content ( file )') self.commitChanges() def createBaseinfoIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return - self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, slot, branch, revision )') + if self.dbname != etpConst['serverdbid'] and self.indexing: + self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, versiontag, slot, branch, revision )') self.commitChanges() def createDependenciesIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return + if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesindex ON dependencies ( idpackage, iddependency )') self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesreferenceindex ON dependenciesreference ( iddependency, dependency )') self.commitChanges() def createExtrainfoIndex(self): - if self.dbname != "etpdb" and self.indexing: - try: - self.checkReadOnly() - except exceptionTools.OperationNotPermitted: - return + if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( idpackage, description, homepage, download, digest, datecreation, size )') self.commitChanges() @@ -3381,7 +3397,7 @@ class etpDatabase(TextInterface): idpackage, ) ) - if etpConst['uid'] == 0 and self.dbname == "etpdb": # force commit even if readonly, this will allow to automagically fix dependstable server side + if etpConst['uid'] == 0 and self.dbname == etpConst['serverdbid']: # force commit even if readonly, this will allow to automagically fix dependstable server side self.connection.commit() # we don't care much about syncing the database since it's quite trivial ''' @@ -3427,10 +3443,9 @@ class etpDatabase(TextInterface): hash(multiMatch) + \ hash(caseSensitive) + \ hash(tuple(matchBranches)) + \ - hash(packagesFilter) + \ - hash(self.dbname) + hash(packagesFilter) ) - cached = dumpTools.loadobj(etpCache['dbMatch']+c_hash) + cached = dumpTools.loadobj(etpCache['dbMatch']+self.dbname+c_hash) if cached != None: return cached @@ -3442,11 +3457,10 @@ class etpDatabase(TextInterface): hash(multiMatch) + \ hash(caseSensitive) + \ hash(tuple(matchBranches)) + \ - hash(packagesFilter) + \ - hash(self.dbname) + hash(packagesFilter) ) try: - dumpTools.dumpobj(etpCache['dbMatch']+c_hash,result) + dumpTools.dumpobj(etpCache['dbMatch']+self.dbname+c_hash,result) except IOError: pass @@ -3536,11 +3550,11 @@ class etpDatabase(TextInterface): if not self.dbname.startswith(etpConst['dbnamerepoprefix']): return results - newresults = [] + newresults = set() for item in results: rc = self.idpackageValidator(item[1]) if rc != -1: - newresults.append(item) + newresults.add(item) return newresults def __filterSlot(self, idpackage, slot): @@ -3573,7 +3587,7 @@ class etpDatabase(TextInterface): newlist.add(data) - return list(newlist) + return newlist @@ -3616,7 +3630,7 @@ class etpDatabase(TextInterface): justname = entropyTools.isjustname(strippedAtom) pkgversion = '' - if (not justname): + if not justname: # get version data = entropyTools.catpkgsplit(strippedAtom) @@ -3636,14 +3650,14 @@ class etpDatabase(TextInterface): if (matchBranches): myBranchIndex = tuple(matchBranches) # force to tuple for security else: - if (self.dbname == 'client'): + if (self.dbname == etpConst['clientdbid']): # collect all available branches myBranchIndex = tuple(self.listAllBranches()) else: myBranchIndex = (etpConst['branch'],) # IDs found in the database that match our search - foundIDs = [] + foundIDs = set() for idx in myBranchIndex: results = self.searchPackagesByName(pkgname, sensitive = caseSensitive, branch = idx) @@ -3698,7 +3712,7 @@ class etpDatabase(TextInterface): continue # search into another branch # if we get here, we have found the needed IDs - foundIDs = results + foundIDs |= set(results) break else: @@ -3711,31 +3725,23 @@ class etpDatabase(TextInterface): if mypkgcat != "null": foundCat = self.retrieveCategory(results[0][1]) if mypkgcat == foundCat: - foundIDs.append(results[0]) + foundIDs.add(results[0]) else: continue else: - foundIDs.append(results[0]) + foundIDs.add(results[0]) break ### FILTERING ### FILTERING ### FILTERING - if packagesFilter: # keyword filtering - foundIDs = self.packagesFilter(foundIDs) - - # filter broken entries - if self.dbname == "client": - for x in range(len(foundIDs)): - try: - self.retrieveAtom(foundIDs[x][1]) - except TypeError: - del foundIDs[x] - # filter slot and tag foundIDs = self.__filterSlotTag(foundIDs, matchSlot, matchTag) + if packagesFilter: # keyword filtering + foundIDs = self.packagesFilter(foundIDs) + ### END FILTERING ### END FILTERING ### END FILTERING @@ -3749,7 +3755,7 @@ class etpDatabase(TextInterface): ### FILLING dbpkginfo ### FILLING dbpkginfo - dbpkginfo = [] + dbpkginfo = set() # now we have to handle direction if (direction) or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")): @@ -3776,15 +3782,15 @@ class etpDatabase(TextInterface): myver = entropyTools.remove_revision(dbver) if myver == pkgversion: # found - dbpkginfo.append([idpackage,dbver]) + dbpkginfo.add((idpackage,dbver)) else: # media-libs/test-1.2* support if pkgversion[-1] == "*": if dbver.startswith(pkgversion[:-1]): - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) # do versions match? elif pkgversion == dbver: - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) elif (direction.find(">") != -1) or (direction.find("<") != -1): @@ -3801,23 +3807,23 @@ class etpDatabase(TextInterface): if direction == ">": # the --deep mode should really act on this if (pkgcmp < 0): # found - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) elif direction == "<": if (pkgcmp > 0): # found - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) elif direction == ">=": # the --deep mode should really act on this if (pkgcmp <= 0): # found - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) elif direction == "<=": if (pkgcmp >= 0): # found - dbpkginfo.append((idpackage,dbver)) + dbpkginfo.add((idpackage,dbver)) else: # just the key - dbpkginfo = [((x[1]),self.retrieveVersion(x[1])) for x in foundIDs] + dbpkginfo = set([((x[1]),self.retrieveVersion(x[1])) for x in foundIDs]) ### END FILLING dbpkginfo ### END FILLING dbpkginfo @@ -3833,11 +3839,18 @@ class etpDatabase(TextInterface): return x,0 if len(dbpkginfo) == 1: - self.atomMatchStoreCache((dbpkginfo[0][0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) - return dbpkginfo[0][0],0 + x = dbpkginfo.pop() + self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) + return x[0],0 - versions = [(x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) for x in dbpkginfo] - versionlist = entropyTools.getEntropyNewerVersion(versions) - x = dbpkginfo[versions.index(versionlist[0])][0] + dbpkginfo = list(dbpkginfo) + pkgdata = {} + versions = set() + for x in dbpkginfo: + info_tuple = (x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) + versions.add(info_tuple) + pkgdata[info_tuple] = x[0] + newer = entropyTools.getEntropyNewerVersion(list(versions))[0] + x = pkgdata[newer] self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return x,0 diff --git a/libraries/entropy.py b/libraries/entropy.py index c4169a2e5..05389d5e7 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -222,14 +222,12 @@ class EquoInterface(TextInterface): const_resetCache() if etpConst['uid'] == 0: for key in etpCache: - if key in ["library_breakage"]: # caches we can skip - continue - cachefile = os.path.join(etpConst['dumpstoragedir'],etpCache[key])+"*.dmp" - if showProgress: self.updateProgress(darkred("Cleaning %s...") % (cachefile,), importance = 1, type = "warning", back = True) - try: - os.system("rm -f "+cachefile) - except: - pass + cachefiles = [x for x in os.listdir(etpConst['dumpstoragedir']) if x.startswith(etpCache[key]) and x.endswith(".dmp")] + for item in cachefiles: + item = os.path.join(etpConst['dumpstoragedir'],item) + if showProgress: self.updateProgress(darkred("Cleaning %s...") % (item,), importance = 1, type = "warning", back = True) + if os.path.isfile(item): + os.remove(item) if showProgress: self.updateProgress(darkgreen("Cache is now empty."), importance = 2, type = "info") @@ -314,6 +312,7 @@ class EquoInterface(TextInterface): update, remove, fine = self.calculate_world_updates() del fine, remove self.retrieveInstallQueue(update, False, False) + self.calculate_available_packages() except: pass @@ -321,7 +320,11 @@ class EquoInterface(TextInterface): def clear_dump_cache(self, dump_name): dump_file = os.path.join(etpConst['dumpstoragedir'],dump_name) - os.system("rm -rf %s*.dmp" % (dump_file,) ) + files = [x for x in os.listdir(etpConst['dumpstoragedir']) if x.startswith(dump_name) and x.endswith(".dmp")] + for item in files: + item = os.path.join(etpConst['dumpstoragedir'],item) + if os.path.isfile(item): + os.remove(item) ''' Cache stuff :: end @@ -567,6 +570,7 @@ class EquoInterface(TextInterface): found, match = self.check_package_update("app-admin/equo") return found + # better to use key:slot def check_package_update(self, atom): if self.xcache: @@ -643,30 +647,42 @@ class EquoInterface(TextInterface): fetch_repository_if_not_available_cache[reponame] = rc return rc - ''' - @description: matches the package that user chose, using dbconnection.atomMatch searching in all available repositories. - @input atom: user choosen package name - @output: the matched selection, list: [package id,repository name] | if nothing found, returns: ( -1,1 ) - @ exit errors: - -1 => repository cannot be fetched online - ''' - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), packagesFilter = True): + def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), packagesFilter = True, multiMatch = False, multiRepo = False): + + valid_repos = [] + for repoid in etpRepositoriesOrder: + # open database + try: + self.openRepositoryDatabase(repoid) + valid_repos.append(repoid) + except exceptionTools.RepositoryError: + continue # repo not available if self.xcache: + + if multiMatch: + m_hash = 2 + else: + m_hash = -2 + if multiRepo: + r_hash = 3 + else: + r_hash = -3 c_hash = str( hash(atom) + \ hash(caseSensitive) + \ - hash(matchSlot) + \ + hash(str(matchSlot)) + \ hash(tuple(matchBranches)) + \ hash(packagesFilter) + \ - hash(tuple(etpRepositoriesOrder)) + \ - hash(tuple(etpRepositories)) + hash(tuple(valid_repos)) + \ + hash(tuple(etpRepositories)) + \ + m_hash + r_hash ) cached = self.dumpTools.loadobj(etpCache['atomMatch']+c_hash) if cached != None: return cached repoResults = {} - for repo in etpRepositoriesOrder: + for repo in valid_repos: # check if repo exists if not repo.endswith(".tbz2"): @@ -674,41 +690,32 @@ class EquoInterface(TextInterface): if fetch != 0: continue # cannot fetch repo, excluding - # open database - try: - dbconn = self.openRepositoryDatabase(repo) - except exceptionTools.RepositoryError, e: - continue # repo not available - # search + dbconn = self.openRepositoryDatabase(repo) query = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter) if query[1] == 0: # package found, add to our dictionary repoResults[repo] = query[0] - # handle repoResults - packageInformation = {} + dbpkginfo = (-1,1) + + packageInformation = {} # nothing found if not repoResults: - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(-1,1)) - except IOError: - pass - return -1,1 + dbpkginfo = (-1,1) + + elif multiRepo: + data = set() + for repoid in repoResults: + data.add((repoResults[repoid],repoid)) + dbpkginfo = (data,0) elif len(repoResults) == 1: # one result found repo = repoResults.keys()[0] - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repo],repo)) - except IOError: - pass - return repoResults[repo],repo + dbpkginfo = (repoResults[repo],repo) - # FIXME: consider to rewrite the code below elif len(repoResults) > 1: # we have to decide which version should be taken @@ -722,39 +729,44 @@ class EquoInterface(TextInterface): if not x.endswith(".tbz2"): del repoResults[x] + versions = [] # get package information for all the entries for repo in repoResults: - - # open database dbconn = self.openRepositoryDatabase(repo) - # search packageInformation[repo] = {} - packageInformation[repo]['version'] = dbconn.retrieveVersion(repoResults[repo]) + version = dbconn.retrieveVersion(repoResults[repo]) + packageInformation[repo]['version'] = version + versions.append(version) packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(repoResults[repo]) packageInformation[repo]['revision'] = dbconn.retrieveRevision(repoResults[repo]) - versions = [] - repoNames = [] - # compare versions - for repo in packageInformation: - repoNames.append(repo) - versions.append(packageInformation[repo]['version']) + # if no duplicates are found, we're done + if len(versions) == len(set(versions)): - # found duplicates, this mean that we have to look at the revision and then, at the version tag - # if all this shait fails, get the uppest repository - # if no duplicates, we're done - if (len(versions) > len(set(versions))): - # there are duplicated results, fetch them - # get the newerVersion - newerVersion = self.entropyTools.getNewerVersion(versions) - newerVersion = newerVersion[0] - # is newerVersion, the duplicated one? + newerVersion = self.entropyTools.getNewerVersion(versions)[0] + for reponame in packageInformation: + if packageInformation[reponame]['version'] == newerVersion: + break + dbpkginfo = (repoResults[reponame],reponame) + + else: + + newerVersion = self.entropyTools.getNewerVersion(versions)[0] duplicatedEntries = self.entropyTools.extractDuplicatedEntries(versions) needFiltering = False if newerVersion in duplicatedEntries: needFiltering = True - if (needFiltering): + if not needFiltering: + + # we are fine, the newerVersion is not one of the duplicated ones + for reponame in packageInformation: + if packageInformation[reponame]['version'] == newerVersion: + break + dbpkginfo = (repoResults[reponame],reponame) + + else: + # we have to decide which one is good # we have newerVersion conflictingEntries = {} @@ -766,8 +778,7 @@ class EquoInterface(TextInterface): # at this point compare tags tags = [conflictingEntries[x]['versiontag'] for x in conflictingEntries] - newerTag = self.entropyTools.getNewerVersionTag(tags) - newerTag = newerTag[0] + newerTag = self.entropyTools.getNewerVersionTag(tags)[0] # is the chosen tag duplicated? duplicatedTags = self.entropyTools.extractDuplicatedEntries(tags) @@ -775,7 +786,16 @@ class EquoInterface(TextInterface): if newerTag in duplicatedTags: needFiltering = True - if (needFiltering): + if not needFiltering: + + # we're finally done + for reponame in conflictingEntries: + if conflictingEntries[reponame]['versiontag'] == newerTag: + break + dbpkginfo = (repoResults[reponame],reponame) + + else: + # yes, it is. we need to compare revisions conflictingTags = {} for repo in conflictingEntries: @@ -783,81 +803,52 @@ class EquoInterface(TextInterface): conflictingTags[repo] = {} conflictingTags[repo]['revision'] = conflictingEntries[repo]['revision'] - revisions = [] - for repo in conflictingTags: - revisions.append(str(conflictingTags[repo]['revision'])) + revisions = [conflictingTags[x]['revision'] for x in conflictingTags] newerRevision = max(revisions) duplicatedRevisions = self.entropyTools.extractDuplicatedEntries(revisions) needFiltering = False if newerRevision in duplicatedRevisions: needFiltering = True - if (needFiltering): - # ok, we must get the repository with the biggest priority - for repository in etpRepositoriesOrder: - if repository in conflictingTags: - # found it, WE ARE DOOONE! - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repository],repository)) - except IOError: - pass - return repoResults[repository],repository - else: - # we are done!!! - reponame = '' - for x in conflictingTags: - if str(conflictingTags[x]['revision']) == str(newerRevision): - reponame = x + if not needFiltering: + + for reponame in conflictingTags: + if conflictingTags[reponame]['revision'] == newerRevision: break - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) - except IOError: - pass - return repoResults[reponame],reponame - else: - # we're finally done - reponame = '' - for x in conflictingEntries: - if conflictingEntries[x]['versiontag'] == newerTag: - reponame = x - break - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) - except IOError: - pass - return repoResults[reponame],reponame + dbpkginfo = (repoResults[reponame],reponame) + + else: + + # ok, we must get the repository with the biggest priority + for reponame in valid_repos: + if reponame in conflictingTags: + break + dbpkginfo = (repoResults[reponame],reponame) + + # multimatch support + if multiMatch: + + if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure + if multiRepo: + data = set() + for match in dbpkginfo[0]: + dbconn = self.openRepositoryDatabase(match[1]) + matches = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, multiMatch = True) + for repoidpackage in matches[0]: + data.add((repoidpackage,match[1])) + dbpkginfo = (data,0) else: - # we are fine, the newerVersion is not one of the duplicated ones - reponame = '' - for x in packageInformation: - if packageInformation[x]['version'] == newerVersion: - reponame = x - break - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) - except IOError: - pass - return repoResults[reponame],reponame - else: - # yeah, we're done, just return the info - newerVersion = self.entropyTools.getNewerVersion(versions) - # get the repository name - newerVersion = newerVersion[0] - reponame = '' - for x in packageInformation: - if packageInformation[x]['version'] == newerVersion: - reponame = x - break - if self.xcache: - try: - self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame)) - except IOError: - pass - return repoResults[reponame],reponame + dbconn = self.openRepositoryDatabase(dbpkginfo[1]) + matches = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, multiMatch = True) + dbpkginfo = (set([(x,dbpkginfo[1]) for x in matches[0]]),0) + + if self.xcache: + try: + self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,dbpkginfo) + except IOError: + pass + + return dbpkginfo def __repository_move_clear_cache(self, repoid): @@ -866,8 +857,9 @@ class EquoInterface(TextInterface): self.clear_dump_cache(etpCache['check_package_update']) self.clear_dump_cache(etpCache['filter_satisfied_deps']) self.clear_dump_cache(etpCache['atomMatch']) - self.clear_dump_cache(etpCache['dbMatch']) - self.clear_dump_cache(etpCache['dbInfo']) + self.clear_dump_cache(etpCache['dbMatch']+repoid) + self.clear_dump_cache(etpCache['dbInfo']+repoid) + self.clear_dump_cache(etpCache['dbSearch']+repoid) def addRepository(self, repodata): @@ -1460,24 +1452,31 @@ class EquoInterface(TextInterface): sum_hashes += dbconn.tablesChecksum() return sum_hashes + def get_available_packges_chash(self, branch): + repo_digest = self.all_repositories_checksum() + # client digest not needed, cache is kept updated + c_hash = str(hash(repo_digest) + \ + hash(branch) + \ + hash(tuple(etpRepositories)) + \ + hash(tuple(etpRepositoriesOrder))) + return c_hash + # this function searches all the not installed packages available in the repositories - def calculate_available_packages(self, branch = etpConst['branch']): + def calculate_available_packages(self): if self.xcache: - repo_digest = self.all_repositories_checksum() - client_digest = self.clientDbconn.tablesChecksum() - c_hash = str(hash(repo_digest) + \ - hash(branch) + \ - hash(client_digest) + \ - hash(tuple(etpRepositories)) + \ - hash(tuple(etpRepositoriesOrder))) - disk_cache = self.dumpTools.loadobj(etpCache['world_available']+c_hash) + c_hash = self.get_available_packges_chash(etpConst['branch']) + disk_cache = self.dumpTools.loadobj(etpCache['world_available']) if disk_cache != None: - return disk_cache + try: + if disk_cache['chash'] == c_hash: + return disk_cache['available'] + except KeyError: + pass available = set() - self.setTotalCycles(len(etpRepositories)) - for repo in etpRepositories: + self.setTotalCycles(len(etpRepositoriesOrder)) + for repo in etpRepositoriesOrder: try: dbconn = self.openRepositoryDatabase(repo) except exceptionTools.RepositoryError: @@ -1488,7 +1487,7 @@ class EquoInterface(TextInterface): maxlen = len(idpackages) for idpackage in idpackages: count += 1 - self.updateProgress("Calculating updates for %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = "::") + self.updateProgress("Calculating updates for %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::") # ignore masked packages idpackage = dbconn.idpackageValidator(idpackage) if idpackage == -1: @@ -1502,7 +1501,10 @@ class EquoInterface(TextInterface): if self.xcache: try: - disk_cache = self.dumpTools.dumpobj(etpCache['world_available']+c_hash,available) + data = {} + data['chash'] = c_hash + data['available'] = available + self.dumpTools.dumpobj(etpCache['world_available'],data) except IOError: pass return available @@ -1539,7 +1541,7 @@ class EquoInterface(TextInterface): for idpackage in idpackages: count += 1 if (count%10 == 0) or (count == maxlen) or (count == 1): - self.updateProgress("Calculating world dependencies", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::") + self.updateProgress("Calculating world packages", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::") tainted = False myscopedata = self.clientDbconn.getScopeData(idpackage) #atom = myscopedata[0] @@ -1583,7 +1585,6 @@ class EquoInterface(TextInterface): matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchSlot = myscopedata[4], matchBranches = (branch,)) if matchresults[0] != -1: mdbconn = self.openRepositoryDatabase(matchresults[1]) - #matchatom = mdbconn.retrieveAtom(matchresults[0]) update.add(matchresults) else: # don't take action if it's just masked @@ -2408,10 +2409,38 @@ class PackageInterface: self.Entropy.clear_dump_cache(etpCache['depends_tree']) self.Entropy.clear_dump_cache(etpCache['check_package_update']) self.Entropy.clear_dump_cache(etpCache['dep_tree']) - self.Entropy.clear_dump_cache(etpCache['world_available']) self.Entropy.clear_dump_cache(etpCache['world_update']) - self.Entropy.clear_dump_cache(etpCache['dbMatch']) - self.Entropy.clear_dump_cache(etpCache['dbInfo']) + self.Entropy.clear_dump_cache(etpCache['dbInfo']+etpConst['clientdbid']) + self.Entropy.clear_dump_cache(etpCache['dbMatch']+etpConst['clientdbid']) + self.Entropy.clear_dump_cache(etpCache['dbSearch']+etpConst['clientdbid']) + + # update world available cache + if self.Entropy.xcache and (self.action in ("remove","install")): + c_hash = self.Entropy.get_available_packges_chash(etpConst['branch']) + disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_available']) + if disk_cache != None: + try: + if disk_cache['chash'] == c_hash: + + # remove and old install + if self.infoDict['removeidpackage'] != -1: + key = self.Entropy.entropyTools.dep_getkey(self.infoDict['removeatom']) + slot = self.infoDict['slot'] + matches = self.Entropy.atomMatch(key, matchSlot = slot, multiRepo = True, multiMatch = True) + if matches[1] == 0: + disk_cache['available'].update(matches[0]) + + # install, doing here because matches[0] could contain self.matched_atoms + if self.matched_atom in disk_cache['available']: + disk_cache['available'].remove(self.matched_atom) + + self.Entropy.dumpTools.dumpobj(etpCache['world_available'],disk_cache) + + except KeyError: + try: + self.Entropy.dumpTools.dumpobj(etpCache['world_available'],{}) + except IOError: + pass ''' @description: install unpacked files, update database and also update gentoo db if requested @@ -3130,6 +3159,7 @@ class PackageInterface: self.infoDict.clear() self.infoDict['triggers'] = {} self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage) + self.infoDict['slot'] = self.Entropy.clientDbconn.retrieveSlot(idpackage) self.infoDict['removeidpackage'] = idpackage self.infoDict['diffremoval'] = False removeConfig = False @@ -3646,7 +3676,9 @@ class RepoInterface: def clear_repository_cache(self, repo): self.__validate_repository_id(repo) - self.Entropy.clear_dump_cache(etpCache['dbInfo']) + self.Entropy.clear_dump_cache(etpCache['dbInfo']+repo) + self.Entropy.clear_dump_cache(etpCache['dbMatch']+repo) + self.Entropy.clear_dump_cache(etpCache['dbSearch']+repo) # this function can be reimplemented def download_item(self, item, repo, cmethod = None): diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 6c3e8eb81..285693837 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -344,8 +344,9 @@ ETP_LOGPRI_ERROR = "[ ERROR ]" # disk caching dictionary etpCache = { 'configfiles': 'scanfs', # used to store information about files that should be merged using "equo conf merge" - 'dbMatch': 'cache_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (dep solving) - 'dbInfo': 'info_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (info retrival) + 'dbMatch': 'cache_', # db atom match cache + 'dbInfo': 'info_', # db data retrieval cache + 'dbSearch': 'search_', # db search cache 'atomMatch': 'atom_match_', # used to store info about repository dependencies solving 'install': 'resume_install', # resume cache (install) 'remove': 'resume_remove', # resume cache (remove) @@ -621,6 +622,9 @@ def initConfig_entropyConstants(rootdir): 'x11-drivers/ati-drivers': ['x11-drivers/ati-drivers'], }, + 'clientdbid': "client", + 'serverdbid': "etpdb", + } etpConst.update(myConst) del myConst