improved caching validation

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1183 cd1c1023-2f26-0410-ae45-c471fc1f0318

TODO

@@ -1,6 +1,6 @@
 TODO list:
+  - cache update instead of trashing each time, look for caches that can be kept if bound to specific checksums
   - split RDEPEND and PDEPEND
-  - improve dependencies order resolution
   - migrate server code to ServerInterface
      [] write a tool that helps keeping packages updated (also supporting injected ones)
      [] complete reagent spm interface

libraries/databaseTools.py

@@ -53,7 +53,7 @@ def openClientDatabase(xcache = True, generate = False, indexing = True):
     if (not generate) and (not os.path.isfile(etpConst['etpdatabaseclientfilepath'])):
         raise exceptionTools.SystemDatabaseError("SystemDatabaseError: system database not found. Either does not exist or corrupted.")
     else:
-        conn = etpDatabase(readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = 'client', xcache = xcache, indexing = indexing)
+        conn = etpDatabase(readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = etpConst['clientdbid'], xcache = xcache, indexing = indexing)
         # validate database
         if not generate:
             conn.validateDatabase()
@@ -122,7 +122,7 @@ def backupClientDatabase():
 
 class etpDatabase(TextInterface):
 
-    def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False, xcache = False, dbname = 'etpdb', indexing = True):
+    def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], indexing = True):
 
         self.readOnly = readOnly
         self.noUpload = noUpload
@@ -137,7 +137,7 @@ class etpDatabase(TextInterface):
         self.dbFile = dbFile
 
         # no caching for non root and server connections
-        if (self.dbname == 'etpdb') or (etpConst['uid'] != 0):
+        if (self.dbname == etpConst['serverdbid']) or (etpConst['uid'] != 0):
             self.xcache = False
 
         # create connection
@@ -847,7 +847,7 @@ class etpDatabase(TextInterface):
                     )
                 )
             except:
-                if self.dbname == "client": # force only for client database
+                if self.dbname == etpConst['clientdbid']: # force only for client database
                     if not self.doesTableExist("counters"):
                         self.createCountersTable()
                     else:
@@ -859,7 +859,7 @@ class etpDatabase(TextInterface):
                         idpackage,
                         )
                     )
-                elif self.dbname == "etpdb":
+                elif self.dbname == etpConst['serverdbid']:
                     raise
 
         # on disk size
@@ -1179,12 +1179,12 @@ class etpDatabase(TextInterface):
             # counter
             self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage)
         except:
-            if self.dbname == "client":
+            if self.dbname == etpConst['clientdbid']:
                 if not self.doesTableExist("counters"):
                     self.createCountersTable()
                 else:
                     raise
-            elif self.dbname == "etpdb":
+            elif self.dbname == etpConst['serverdbid']:
                 raise
         try:
             # on disk sizes
@@ -1455,7 +1455,7 @@ class etpDatabase(TextInterface):
         try:
             self.cursor.execute('UPDATE counters SET counter = (?) WHERE idpackage = (?)', (counter,idpackage,))
         except:
-            if self.dbname == "client":
+            if self.dbname == etpConst['clientdbid']:
                 if not self.doesTableExist("counters"):
                     self.createCountersTable()
                 else:
@@ -1858,14 +1858,15 @@ class etpDatabase(TextInterface):
         def do_clear(name):
             dump_file = os.path.join(etpConst['dumpstoragedir'],name)
             os.system("rm -rf %s*.dmp" % (dump_file,) )
-        do_clear(etpCache['dbInfo'])
-        do_clear(etpCache['dbMatch'])
+        do_clear(etpCache['dbInfo']+self.dbname)
+        do_clear(etpCache['dbMatch']+self.dbname)
+        do_clear(etpCache['dbSearch']+self.dbname)
 
     def fetchInfoCache(self, idpackage, function, extra_hash = 0):
         if (self.xcache):
-            c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash )
+            c_hash = str( hash(int(idpackage)) + hash(function) + extra_hash )
             try:
-                cached = dumpTools.loadobj(etpCache['dbInfo']+c_hash)
+                cached = dumpTools.loadobj(etpCache['dbInfo']+self.dbname+c_hash)
                 if cached != None:
                     return cached
             except EOFError:
@@ -1874,9 +1875,26 @@ class etpDatabase(TextInterface):
 
     def storeInfoCache(self, idpackage, function, info_cache_data, extra_hash = 0):
         if (self.xcache):
-            c_hash = str( hash(int(idpackage)) + hash(function) + hash(self.dbname) + extra_hash )
+            c_hash = str( hash(int(idpackage)) + hash(function) + extra_hash )
             try:
-                dumpTools.dumpobj(etpCache['dbInfo']+c_hash,info_cache_data)
+                dumpTools.dumpobj(etpCache['dbInfo']+self.dbname+c_hash,info_cache_data)
+            except IOError:
+                pass
+
+    def fetchSearchCache(self, cache_hash):
+        if self.xcache:
+            try:
+                cached = dumpTools.loadobj(etpCache['dbSearch']+self.dbname+cache_hash)
+                if cached != None:
+                    return cached
+            except EOFError:
+                pass
+
+
+    def storeSearchCache(self, cache_hash, cache_data):
+        if self.xcache:
+            try:
+                dumpTools.dumpobj(etpCache['dbSearch']+self.dbname+cache_hash,cache_data)
             except IOError:
                 pass
 
@@ -2023,7 +2041,7 @@ class etpDatabase(TextInterface):
             if mycounter:
                 counter = mycounter[0]
         except:
-            if self.dbname == "client":
+            if self.dbname == etpConst['clientdbid']:
                 if not self.doesTableExist("counters"):
                     self.createCountersTable()
                 else:
@@ -2371,18 +2389,18 @@ class etpDatabase(TextInterface):
 
         self.storeInfoCache(idpackage,'retrieveSlot',ver)
         return ver
-    
+
     def retrieveVersionTag(self, idpackage):
 
-	cache = self.fetchInfoCache(idpackage,'retrieveVersionTag')
-	if cache != None: return cache
+        cache = self.fetchInfoCache(idpackage,'retrieveVersionTag')
+        if cache != None: return cache
 
-	self.cursor.execute('SELECT "versiontag" FROM baseinfo WHERE idpackage = (?)', (idpackage,))
-	ver = self.cursor.fetchone()[0]
+        self.cursor.execute('SELECT "versiontag" FROM baseinfo WHERE idpackage = (?)', (idpackage,))
+        ver = self.cursor.fetchone()[0]
+
+        self.storeInfoCache(idpackage,'retrieveVersionTag',ver)
+        return ver
 
-	self.storeInfoCache(idpackage,'retrieveVersionTag',ver)
-	return ver
-    
     def retrieveMirrorInfo(self, mirrorname):
 
 	self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,))
@@ -2718,27 +2736,34 @@ class etpDatabase(TextInterface):
 
     def searchProvide(self, keyword, slot = None, tag = None, branch = None):
 
-	self.cursor.execute('SELECT idpackage FROM provide WHERE atom = (?)', (keyword,))
-	idpackage = self.cursor.fetchone()
-	if not idpackage:
-	    return ()
+        c_hash = str( hash("searchProvide") + hash(keyword) + hash(slot) + hash(branch) + hash(tag) )
+        cache = self.fetchSearchCache(c_hash)
+        if cache != None: return cache
 
-	slotstring = ''
+        self.cursor.execute('SELECT idpackage FROM provide WHERE atom = (?)', (keyword,))
+        idpackage = self.cursor.fetchone()
+        if not idpackage:
+            return ()
+
+        slotstring = ''
         searchkeywords = [idpackage[0]]
-	if slot:
+        if slot:
             searchkeywords.append(slot)
-	    slotstring = ' and slot = (?)'
-	tagstring = ''
-	if tag:
+            slotstring = ' and slot = (?)'
+        tagstring = ''
+        if tag:
             searchkeywords.append(tag)
-	    tagstring = ' and versiontag = (?)'
-	branchstring = ''
-	if branch:
+            tagstring = ' and versiontag = (?)'
+        branchstring = ''
+        if branch:
             searchkeywords.append(branch)
-	    branchstring = ' and branch = (?)'
+            branchstring = ' and branch = (?)'
 
-	self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = (?)'+slotstring+tagstring+branchstring, searchkeywords)
-	return self.cursor.fetchall()
+        self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = (?)'+slotstring+tagstring+branchstring, searchkeywords)
+
+        results = self.cursor.fetchall()
+        self.storeSearchCache(c_hash,results)
+        return results
 
     def searchPackagesByDescription(self, keyword):
 	self.cursor.execute('SELECT idpackage FROM extrainfo WHERE LOWER(description) LIKE (?)', ("%"+keyword.lower()+"%",))
@@ -2774,23 +2799,29 @@ class etpDatabase(TextInterface):
 	return result
 
     def searchPackagesByName(self, keyword, sensitive = False, branch = None):
-	
+
+        c_hash = str( hash("searchPackagesByName") + hash(keyword) + hash(sensitive) + hash(branch) )
+        cache = self.fetchSearchCache(c_hash)
+        if cache != None: return cache
+
         if sensitive:
             searchkeywords = [keyword]
         else:
             searchkeywords = [keyword.lower()]
-	branchstring = ''
-	if branch:
+        branchstring = ''
+        if branch:
             searchkeywords.append(branch)
-	    branchstring = ' and branch = (?)'
-	
-	if (sensitive):
-	    self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords)
-	else:
-	    self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords)
-	
-	results = self.cursor.fetchall()
-	return results
+            branchstring = ' and branch = (?)'
+
+        if sensitive:
+            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords)
+        else:
+            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords)
+
+        results = self.cursor.fetchall()
+
+        self.storeSearchCache(c_hash,results)
+        return results
 
 
     def searchPackagesByCategory(self, keyword, like = False, branch = None):
@@ -2811,37 +2842,41 @@ class etpDatabase(TextInterface):
 	return results
 
     def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None):
-	
-	# get category id
-	idcat = -1
-	self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,))
-	idcat = self.cursor.fetchone()
-	if not idcat:
-	    return ()
-	else:
-	    idcat = idcat[0]
+
+        c_hash = str( hash("searchPackagesByNameAndCategory") + hash(name) + hash(category) + hash(sensitive) + hash(branch) )
+        cache = self.fetchSearchCache(c_hash)
+        if cache != None: return cache
+
+        # get category id
+        idcat = -1
+        self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,))
+        idcat = self.cursor.fetchone()
+        if not idcat:
+            return ()
+        else:
+            idcat = idcat[0]
 
         searchkeywords = []
         if sensitive:
             searchkeywords.append(name)
         else:
             searchkeywords.append(name.lower())
-        
+
         searchkeywords.append(idcat)
-        
-	branchstring = ''
-	if branch:
+
+        branchstring = ''
+        if branch:
             searchkeywords.append(branch)
-	    branchstring = ' and branch = (?)'
+            branchstring = ' and branch = (?)'
 
-	if (sensitive):
-	    self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?) AND idcategory = (?) '+branchstring, searchkeywords)
-	else:
-	    self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory = (?) '+branchstring, searchkeywords)
-	
-	results = self.cursor.fetchall()
+        if (sensitive):
+            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?) AND idcategory = (?) '+branchstring, searchkeywords)
+        else:
+            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory = (?) '+branchstring, searchkeywords)
 
-	return results
+        results = self.cursor.fetchall()
+        self.storeSearchCache(c_hash,results)
+        return results
 
     def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False):
 
@@ -2907,8 +2942,13 @@ class etpDatabase(TextInterface):
 
     def listAllBranches(self):
 
+        cache = self.fetchInfoCache(0,'listAllBranches')
+        if cache != None: return cache
+
         self.cursor.execute('SELECT branch FROM baseinfo')
         results = self.fetchall2set(self.cursor.fetchall())
+
+        self.storeInfoCache(0,'listAllBranches',results)
         return results
 
     def listIdPackagesInIdcategory(self,idcategory):
@@ -3201,57 +3241,33 @@ class etpDatabase(TextInterface):
         self.createUseflagsIndex()
 
     def createNeededIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( idneeded,library )')
             self.commitChanges()
 
     def createUseflagsIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( idflag,flagname )')
             self.commitChanges()
 
     def createContentIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex ON content ( file )')
             self.commitChanges()
 
     def createBaseinfoIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
-            self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, slot, branch, revision )')
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
+            self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, versiontag, slot, branch, revision )')
             self.commitChanges()
 
     def createDependenciesIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesindex ON dependencies ( idpackage, iddependency )')
             self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesreferenceindex ON dependenciesreference ( iddependency, dependency )')
             self.commitChanges()
 
     def createExtrainfoIndex(self):
-        if self.dbname != "etpdb" and self.indexing:
-            try:
-                self.checkReadOnly()
-            except exceptionTools.OperationNotPermitted:
-                return
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( idpackage, description, homepage, download, digest, datecreation, size )')
             self.commitChanges()
 
@@ -3381,7 +3397,7 @@ class etpDatabase(TextInterface):
                         idpackage,
                         )
         )
-        if etpConst['uid'] == 0 and self.dbname == "etpdb": # force commit even if readonly, this will allow to automagically fix dependstable server side
+        if etpConst['uid'] == 0 and self.dbname == etpConst['serverdbid']: # force commit even if readonly, this will allow to automagically fix dependstable server side
             self.connection.commit()                        # we don't care much about syncing the database since it's quite trivial
 
     '''
@@ -3427,10 +3443,9 @@ class etpDatabase(TextInterface):
                             hash(multiMatch) + \
                             hash(caseSensitive) + \
                             hash(tuple(matchBranches)) + \
-                            hash(packagesFilter) + \
-                            hash(self.dbname)
+                            hash(packagesFilter)
                         )
-            cached = dumpTools.loadobj(etpCache['dbMatch']+c_hash)
+            cached = dumpTools.loadobj(etpCache['dbMatch']+self.dbname+c_hash)
             if cached != None:
                 return cached
 
@@ -3442,11 +3457,10 @@ class etpDatabase(TextInterface):
                             hash(multiMatch) + \
                             hash(caseSensitive) + \
                             hash(tuple(matchBranches)) + \
-                            hash(packagesFilter) + \
-                            hash(self.dbname)
+                            hash(packagesFilter)
                         )
             try:
-                dumpTools.dumpobj(etpCache['dbMatch']+c_hash,result)
+                dumpTools.dumpobj(etpCache['dbMatch']+self.dbname+c_hash,result)
             except IOError:
                 pass
 
@@ -3536,11 +3550,11 @@ class etpDatabase(TextInterface):
         if not self.dbname.startswith(etpConst['dbnamerepoprefix']):
             return results
 
-        newresults = []
+        newresults = set()
         for item in results:
             rc = self.idpackageValidator(item[1])
             if rc != -1:
-                newresults.append(item)
+                newresults.add(item)
         return newresults
 
     def __filterSlot(self, idpackage, slot):
@@ -3573,7 +3587,7 @@ class etpDatabase(TextInterface):
 
             newlist.add(data)
 
-        return list(newlist)
+        return newlist
 
 
 
@@ -3616,7 +3630,7 @@ class etpDatabase(TextInterface):
 
         justname = entropyTools.isjustname(strippedAtom)
         pkgversion = ''
-        if (not justname):
+        if not justname:
 
             # get version
             data = entropyTools.catpkgsplit(strippedAtom)
@@ -3636,14 +3650,14 @@ class etpDatabase(TextInterface):
         if (matchBranches):
             myBranchIndex = tuple(matchBranches) # force to tuple for security
         else:
-            if (self.dbname == 'client'):
+            if (self.dbname == etpConst['clientdbid']):
                 # collect all available branches
                 myBranchIndex = tuple(self.listAllBranches())
             else:
                 myBranchIndex = (etpConst['branch'],)
 
         # IDs found in the database that match our search
-        foundIDs = []
+        foundIDs = set()
 
         for idx in myBranchIndex:
             results = self.searchPackagesByName(pkgname, sensitive = caseSensitive, branch = idx)
@@ -3698,7 +3712,7 @@ class etpDatabase(TextInterface):
                     continue  # search into another branch
 
                 # if we get here, we have found the needed IDs
-                foundIDs = results
+                foundIDs |= set(results)
                 break
 
             else:
@@ -3711,31 +3725,23 @@ class etpDatabase(TextInterface):
                 if mypkgcat != "null":
                     foundCat = self.retrieveCategory(results[0][1])
                     if mypkgcat == foundCat:
-                        foundIDs.append(results[0])
+                        foundIDs.add(results[0])
                     else:
                         continue
                 else:
-                    foundIDs.append(results[0])
+                    foundIDs.add(results[0])
                     break
 
         ### FILTERING
         ### FILTERING
         ### FILTERING
 
-        if packagesFilter: # keyword filtering
-            foundIDs = self.packagesFilter(foundIDs)
-
-        # filter broken entries
-        if self.dbname == "client":
-            for x in range(len(foundIDs)):
-                try:
-                    self.retrieveAtom(foundIDs[x][1])
-                except TypeError:
-                    del foundIDs[x]
-
         # filter slot and tag
         foundIDs = self.__filterSlotTag(foundIDs, matchSlot, matchTag)
 
+        if packagesFilter: # keyword filtering
+            foundIDs = self.packagesFilter(foundIDs)
+
         ### END FILTERING
         ### END FILTERING
         ### END FILTERING
@@ -3749,7 +3755,7 @@ class etpDatabase(TextInterface):
         ### FILLING dbpkginfo
         ### FILLING dbpkginfo
 
-        dbpkginfo = []
+        dbpkginfo = set()
         # now we have to handle direction
         if (direction) or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")):
 
@@ -3776,15 +3782,15 @@ class etpDatabase(TextInterface):
                         myver = entropyTools.remove_revision(dbver)
                         if myver == pkgversion:
                             # found
-                            dbpkginfo.append([idpackage,dbver])
+                            dbpkginfo.add((idpackage,dbver))
                     else:
                         # media-libs/test-1.2* support
                         if pkgversion[-1] == "*":
                             if dbver.startswith(pkgversion[:-1]):
-                                dbpkginfo.append((idpackage,dbver))
+                                dbpkginfo.add((idpackage,dbver))
                         # do versions match?
                         elif pkgversion == dbver:
-                            dbpkginfo.append((idpackage,dbver))
+                            dbpkginfo.add((idpackage,dbver))
 
             elif (direction.find(">") != -1) or (direction.find("<") != -1):
 
@@ -3801,23 +3807,23 @@ class etpDatabase(TextInterface):
                     if direction == ">": # the --deep mode should really act on this
                         if (pkgcmp < 0):
                             # found
-                            dbpkginfo.append((idpackage,dbver))
+                            dbpkginfo.add((idpackage,dbver))
                     elif direction == "<":
                         if (pkgcmp > 0):
                             # found
-                            dbpkginfo.append((idpackage,dbver))
+                            dbpkginfo.add((idpackage,dbver))
                     elif direction == ">=": # the --deep mode should really act on this
                         if (pkgcmp <= 0):
                             # found
-                            dbpkginfo.append((idpackage,dbver))
+                            dbpkginfo.add((idpackage,dbver))
                     elif direction == "<=":
                         if (pkgcmp >= 0):
                             # found
-                            dbpkginfo.append((idpackage,dbver))
+                            dbpkginfo.add((idpackage,dbver))
 
         else: # just the key
 
-            dbpkginfo = [((x[1]),self.retrieveVersion(x[1])) for x in foundIDs]
+            dbpkginfo = set([((x[1]),self.retrieveVersion(x[1])) for x in foundIDs])
 
         ### END FILLING dbpkginfo
         ### END FILLING dbpkginfo
@@ -3833,11 +3839,18 @@ class etpDatabase(TextInterface):
             return x,0
 
         if len(dbpkginfo) == 1:
-            self.atomMatchStoreCache((dbpkginfo[0][0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
-            return dbpkginfo[0][0],0
+            x = dbpkginfo.pop()
+            self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
+            return x[0],0
 
-        versions = [(x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) for x in dbpkginfo]
-        versionlist = entropyTools.getEntropyNewerVersion(versions)
-        x = dbpkginfo[versions.index(versionlist[0])][0]
+        dbpkginfo = list(dbpkginfo)
+        pkgdata = {}
+        versions = set()
+        for x in dbpkginfo:
+            info_tuple = (x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0]))
+            versions.add(info_tuple)
+            pkgdata[info_tuple] = x[0]
+        newer = entropyTools.getEntropyNewerVersion(list(versions))[0]
+        x = pkgdata[newer]
         self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
         return x,0

libraries/entropy.py

@@ -222,14 +222,12 @@ class EquoInterface(TextInterface):
         const_resetCache()
         if etpConst['uid'] == 0:
             for key in etpCache:
-                if key in ["library_breakage"]: # caches we can skip
-                    continue
-                cachefile = os.path.join(etpConst['dumpstoragedir'],etpCache[key])+"*.dmp"
-                if showProgress: self.updateProgress(darkred("Cleaning %s...") % (cachefile,), importance = 1, type = "warning", back = True)
-                try:
-                    os.system("rm -f "+cachefile)
-                except:
-                    pass
+                cachefiles = [x for x in os.listdir(etpConst['dumpstoragedir']) if x.startswith(etpCache[key]) and x.endswith(".dmp")]
+                for item in cachefiles:
+                    item = os.path.join(etpConst['dumpstoragedir'],item)
+                    if showProgress: self.updateProgress(darkred("Cleaning %s...") % (item,), importance = 1, type = "warning", back = True)
+                    if os.path.isfile(item):
+                        os.remove(item)
 
             if showProgress: self.updateProgress(darkgreen("Cache is now empty."), importance = 2, type = "info")
 
@@ -314,6 +312,7 @@ class EquoInterface(TextInterface):
             update, remove, fine = self.calculate_world_updates()
             del fine, remove
             self.retrieveInstallQueue(update, False, False)
+            self.calculate_available_packages()
         except:
             pass
 
@@ -321,7 +320,11 @@ class EquoInterface(TextInterface):
 
     def clear_dump_cache(self, dump_name):
         dump_file = os.path.join(etpConst['dumpstoragedir'],dump_name)
-        os.system("rm -rf %s*.dmp" % (dump_file,) )
+        files = [x for x in os.listdir(etpConst['dumpstoragedir']) if x.startswith(dump_name) and x.endswith(".dmp")]
+        for item in files:
+            item = os.path.join(etpConst['dumpstoragedir'],item)
+            if os.path.isfile(item):
+                os.remove(item)
 
     '''
        Cache stuff :: end
@@ -567,6 +570,7 @@ class EquoInterface(TextInterface):
         found, match = self.check_package_update("app-admin/equo")
         return found
 
+    # better to use key:slot
     def check_package_update(self, atom):
 
         if self.xcache:
@@ -643,30 +647,42 @@ class EquoInterface(TextInterface):
         fetch_repository_if_not_available_cache[reponame] = rc
         return rc
 
-    '''
-    @description: matches the package that user chose, using dbconnection.atomMatch searching in all available repositories.
-    @input atom: user choosen package name
-    @output: the matched selection, list: [package id,repository name] | if nothing found, returns: ( -1,1 )
-    @ exit errors:
-                -1 => repository cannot be fetched online
-    '''
-    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), packagesFilter = True):
+    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), packagesFilter = True, multiMatch = False, multiRepo = False):
+
+        valid_repos = []
+        for repoid in etpRepositoriesOrder:
+            # open database
+            try:
+                self.openRepositoryDatabase(repoid)
+                valid_repos.append(repoid)
+            except exceptionTools.RepositoryError:
+                continue # repo not available
 
         if self.xcache:
+
+            if multiMatch:
+                m_hash = 2
+            else:
+                m_hash = -2
+            if multiRepo:
+                r_hash = 3
+            else:
+                r_hash = -3
             c_hash = str(   hash(atom) + \
                             hash(caseSensitive) + \
-                            hash(matchSlot) + \
+                            hash(str(matchSlot)) + \
                             hash(tuple(matchBranches)) + \
                             hash(packagesFilter) + \
-                            hash(tuple(etpRepositoriesOrder)) + \
-                            hash(tuple(etpRepositories))
+                            hash(tuple(valid_repos)) + \
+                            hash(tuple(etpRepositories)) + \
+                            m_hash + r_hash
                         )
             cached = self.dumpTools.loadobj(etpCache['atomMatch']+c_hash)
             if cached != None:
                 return cached
 
         repoResults = {}
-        for repo in etpRepositoriesOrder:
+        for repo in valid_repos:
 
             # check if repo exists
             if not repo.endswith(".tbz2"):
@@ -674,41 +690,32 @@ class EquoInterface(TextInterface):
                 if fetch != 0:
                     continue # cannot fetch repo, excluding
 
-            # open database
-            try:
-                dbconn = self.openRepositoryDatabase(repo)
-            except exceptionTools.RepositoryError, e:
-                continue # repo not available
-
             # search
+            dbconn = self.openRepositoryDatabase(repo)
             query = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter)
             if query[1] == 0:
                 # package found, add to our dictionary
                 repoResults[repo] = query[0]
 
-        # handle repoResults
-        packageInformation = {}
 
+        dbpkginfo = (-1,1)
+
+        packageInformation = {}
         # nothing found
         if not repoResults:
-            if self.xcache:
-                try:
-                    self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(-1,1))
-                except IOError:
-                    pass
-            return -1,1
+            dbpkginfo = (-1,1)
+
+        elif multiRepo:
+            data = set()
+            for repoid in repoResults:
+                data.add((repoResults[repoid],repoid))
+            dbpkginfo = (data,0)
 
         elif len(repoResults) == 1:
             # one result found
             repo = repoResults.keys()[0]
-            if self.xcache:
-                try:
-                    self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repo],repo))
-                except IOError:
-                    pass
-            return repoResults[repo],repo
+            dbpkginfo = (repoResults[repo],repo)
 
-        # FIXME: consider to rewrite the code below
         elif len(repoResults) > 1:
             # we have to decide which version should be taken
 
@@ -722,39 +729,44 @@ class EquoInterface(TextInterface):
                     if not x.endswith(".tbz2"):
                         del repoResults[x]
 
+            versions = []
             # get package information for all the entries
             for repo in repoResults:
-
-                # open database
                 dbconn = self.openRepositoryDatabase(repo)
-                # search
                 packageInformation[repo] = {}
-                packageInformation[repo]['version'] = dbconn.retrieveVersion(repoResults[repo])
+                version = dbconn.retrieveVersion(repoResults[repo])
+                packageInformation[repo]['version'] = version
+                versions.append(version)
                 packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(repoResults[repo])
                 packageInformation[repo]['revision'] = dbconn.retrieveRevision(repoResults[repo])
 
-            versions = []
-            repoNames = []
-            # compare versions
-            for repo in packageInformation:
-                repoNames.append(repo)
-                versions.append(packageInformation[repo]['version'])
+            # if no duplicates are found, we're done
+            if len(versions) == len(set(versions)):
 
-            # found duplicates, this mean that we have to look at the revision and then, at the version tag
-            # if all this shait fails, get the uppest repository
-            # if no duplicates, we're done
-            if (len(versions) > len(set(versions))):
-                # there are duplicated results, fetch them
-                # get the newerVersion
-                newerVersion = self.entropyTools.getNewerVersion(versions)
-                newerVersion = newerVersion[0]
-                # is newerVersion, the duplicated one?
+                newerVersion = self.entropyTools.getNewerVersion(versions)[0]
+                for reponame in packageInformation:
+                    if packageInformation[reponame]['version'] == newerVersion:
+                        break
+                dbpkginfo = (repoResults[reponame],reponame)
+
+            else:
+
+                newerVersion = self.entropyTools.getNewerVersion(versions)[0]
                 duplicatedEntries = self.entropyTools.extractDuplicatedEntries(versions)
                 needFiltering = False
                 if newerVersion in duplicatedEntries:
                     needFiltering = True
 
-                if (needFiltering):
+                if not needFiltering:
+
+                    # we are fine, the newerVersion is not one of the duplicated ones
+                    for reponame in packageInformation:
+                        if packageInformation[reponame]['version'] == newerVersion:
+                            break
+                    dbpkginfo = (repoResults[reponame],reponame)
+
+                else:
+
                     # we have to decide which one is good
                     # we have newerVersion
                     conflictingEntries = {}
@@ -766,8 +778,7 @@ class EquoInterface(TextInterface):
 
                     # at this point compare tags
                     tags = [conflictingEntries[x]['versiontag'] for x in conflictingEntries]
-                    newerTag = self.entropyTools.getNewerVersionTag(tags)
-                    newerTag = newerTag[0]
+                    newerTag = self.entropyTools.getNewerVersionTag(tags)[0]
 
                     # is the chosen tag duplicated?
                     duplicatedTags = self.entropyTools.extractDuplicatedEntries(tags)
@@ -775,7 +786,16 @@ class EquoInterface(TextInterface):
                     if newerTag in duplicatedTags:
                         needFiltering = True
 
-                    if (needFiltering):
+                    if not needFiltering:
+
+                        # we're finally done
+                        for reponame in conflictingEntries:
+                            if conflictingEntries[reponame]['versiontag'] == newerTag:
+                                break
+                        dbpkginfo = (repoResults[reponame],reponame)
+
+                    else:
+
                         # yes, it is. we need to compare revisions
                         conflictingTags = {}
                         for repo in conflictingEntries:
@@ -783,81 +803,52 @@ class EquoInterface(TextInterface):
                                 conflictingTags[repo] = {}
                                 conflictingTags[repo]['revision'] = conflictingEntries[repo]['revision']
 
-                        revisions = []
-                        for repo in conflictingTags:
-                            revisions.append(str(conflictingTags[repo]['revision']))
+                        revisions = [conflictingTags[x]['revision'] for x in conflictingTags]
                         newerRevision = max(revisions)
                         duplicatedRevisions = self.entropyTools.extractDuplicatedEntries(revisions)
                         needFiltering = False
                         if newerRevision in duplicatedRevisions:
                             needFiltering = True
 
-                        if (needFiltering):
-                            # ok, we must get the repository with the biggest priority
-                            for repository in etpRepositoriesOrder:
-                                if repository in conflictingTags:
-                                    # found it, WE ARE DOOONE!
-                                    if self.xcache:
-                                        try:
-                                            self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[repository],repository))
-                                        except IOError:
-                                            pass
-                                    return repoResults[repository],repository
-                        else:
-                            # we are done!!!
-                            reponame = ''
-                            for x in conflictingTags:
-                                if str(conflictingTags[x]['revision']) == str(newerRevision):
-                                    reponame = x
+                        if not needFiltering:
+
+                            for reponame in conflictingTags:
+                                if conflictingTags[reponame]['revision'] == newerRevision:
                                     break
-                            if self.xcache:
-                                try:
-                                    self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame))
-                                except IOError:
-                                    pass
-                            return repoResults[reponame],reponame
-                    else:
-                        # we're finally done
-                        reponame = ''
-                        for x in conflictingEntries:
-                            if conflictingEntries[x]['versiontag'] == newerTag:
-                                reponame = x
-                                break
-                        if self.xcache:
-                            try:
-                                self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame))
-                            except IOError:
-                                pass
-                        return repoResults[reponame],reponame
+                            dbpkginfo = (repoResults[reponame],reponame)
+
+                        else:
+
+                            # ok, we must get the repository with the biggest priority
+                            for reponame in valid_repos:
+                                if reponame in conflictingTags:
+                                    break
+                            dbpkginfo = (repoResults[reponame],reponame)
+
+        # multimatch support
+        if multiMatch:
+
+            if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure
+                if multiRepo:
+                    data = set()
+                    for match in dbpkginfo[0]:
+                        dbconn = self.openRepositoryDatabase(match[1])
+                        matches = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, multiMatch = True)
+                        for repoidpackage in matches[0]:
+                            data.add((repoidpackage,match[1]))
+                    dbpkginfo = (data,0)
                 else:
-                    # we are fine, the newerVersion is not one of the duplicated ones
-                    reponame = ''
-                    for x in packageInformation:
-                        if packageInformation[x]['version'] == newerVersion:
-                            reponame = x
-                            break
-                    if self.xcache:
-                        try:
-                            self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame))
-                        except IOError:
-                            pass
-                    return repoResults[reponame],reponame
-            else:
-                # yeah, we're done, just return the info
-                newerVersion = self.entropyTools.getNewerVersion(versions)
-                # get the repository name
-                newerVersion = newerVersion[0]
-                reponame = ''
-                for x in packageInformation:
-                    if packageInformation[x]['version'] == newerVersion:
-                        reponame = x
-                        break
-                if self.xcache:
-                    try:
-                        self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,(repoResults[reponame],reponame))
-                    except IOError:
-                        pass
-                return repoResults[reponame],reponame
+                    dbconn = self.openRepositoryDatabase(dbpkginfo[1])
+                    matches = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, multiMatch = True)
+                    dbpkginfo = (set([(x,dbpkginfo[1]) for x in matches[0]]),0)
+
+        if self.xcache:
+            try:
+                self.dumpTools.dumpobj(etpCache['atomMatch']+c_hash,dbpkginfo)
+            except IOError:
+                pass
+
+        return dbpkginfo
 
 
     def __repository_move_clear_cache(self, repoid):
@@ -866,8 +857,9 @@ class EquoInterface(TextInterface):
         self.clear_dump_cache(etpCache['check_package_update'])
         self.clear_dump_cache(etpCache['filter_satisfied_deps'])
         self.clear_dump_cache(etpCache['atomMatch'])
-        self.clear_dump_cache(etpCache['dbMatch'])
-        self.clear_dump_cache(etpCache['dbInfo'])
+        self.clear_dump_cache(etpCache['dbMatch']+repoid)
+        self.clear_dump_cache(etpCache['dbInfo']+repoid)
+        self.clear_dump_cache(etpCache['dbSearch']+repoid)
 
 
     def addRepository(self, repodata):
@@ -1460,24 +1452,31 @@ class EquoInterface(TextInterface):
             sum_hashes += dbconn.tablesChecksum()
         return sum_hashes
 
+    def get_available_packges_chash(self, branch):
+        repo_digest = self.all_repositories_checksum()
+        # client digest not needed, cache is kept updated
+        c_hash = str(hash(repo_digest) + \
+                    hash(branch) + \
+                    hash(tuple(etpRepositories)) + \
+                    hash(tuple(etpRepositoriesOrder)))
+        return c_hash
+
     # this function searches all the not installed packages available in the repositories
-    def calculate_available_packages(self, branch = etpConst['branch']):
+    def calculate_available_packages(self):
 
         if self.xcache:
-            repo_digest = self.all_repositories_checksum()
-            client_digest = self.clientDbconn.tablesChecksum()
-            c_hash = str(hash(repo_digest) + \
-                     hash(branch) + \
-                     hash(client_digest) + \
-                     hash(tuple(etpRepositories)) + \
-                     hash(tuple(etpRepositoriesOrder)))
-            disk_cache = self.dumpTools.loadobj(etpCache['world_available']+c_hash)
+            c_hash = self.get_available_packges_chash(etpConst['branch'])
+            disk_cache = self.dumpTools.loadobj(etpCache['world_available'])
             if disk_cache != None:
-                return disk_cache
+                try:
+                    if disk_cache['chash'] == c_hash:
+                        return disk_cache['available']
+                except KeyError:
+                    pass
 
         available = set()
-        self.setTotalCycles(len(etpRepositories))
-        for repo in etpRepositories:
+        self.setTotalCycles(len(etpRepositoriesOrder))
+        for repo in etpRepositoriesOrder:
             try:
                 dbconn = self.openRepositoryDatabase(repo)
             except exceptionTools.RepositoryError:
@@ -1488,7 +1487,7 @@ class EquoInterface(TextInterface):
             maxlen = len(idpackages)
             for idpackage in idpackages:
                 count += 1
-                self.updateProgress("Calculating updates for %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = "::")
+                self.updateProgress("Calculating updates for %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::")
                 # ignore masked packages
                 idpackage = dbconn.idpackageValidator(idpackage)
                 if idpackage == -1:
@@ -1502,7 +1501,10 @@ class EquoInterface(TextInterface):
 
         if self.xcache:
             try:
-                disk_cache = self.dumpTools.dumpobj(etpCache['world_available']+c_hash,available)
+                data = {}
+                data['chash'] = c_hash
+                data['available'] = available
+                self.dumpTools.dumpobj(etpCache['world_available'],data)
             except IOError:
                 pass
         return available
@@ -1539,7 +1541,7 @@ class EquoInterface(TextInterface):
         for idpackage in idpackages:
             count += 1
             if (count%10 == 0) or (count == maxlen) or (count == 1):
-                self.updateProgress("Calculating world dependencies", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::")
+                self.updateProgress("Calculating world packages", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::")
             tainted = False
             myscopedata = self.clientDbconn.getScopeData(idpackage)
             #atom = myscopedata[0]
@@ -1583,7 +1585,6 @@ class EquoInterface(TextInterface):
                 matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchSlot = myscopedata[4], matchBranches = (branch,))
                 if matchresults[0] != -1:
                     mdbconn = self.openRepositoryDatabase(matchresults[1])
-                    #matchatom = mdbconn.retrieveAtom(matchresults[0])
                     update.add(matchresults)
                 else:
                     # don't take action if it's just masked
@@ -2408,10 +2409,38 @@ class PackageInterface:
         self.Entropy.clear_dump_cache(etpCache['depends_tree'])
         self.Entropy.clear_dump_cache(etpCache['check_package_update'])
         self.Entropy.clear_dump_cache(etpCache['dep_tree'])
-        self.Entropy.clear_dump_cache(etpCache['world_available'])
         self.Entropy.clear_dump_cache(etpCache['world_update'])
-        self.Entropy.clear_dump_cache(etpCache['dbMatch'])
-        self.Entropy.clear_dump_cache(etpCache['dbInfo'])
+        self.Entropy.clear_dump_cache(etpCache['dbInfo']+etpConst['clientdbid'])
+        self.Entropy.clear_dump_cache(etpCache['dbMatch']+etpConst['clientdbid'])
+        self.Entropy.clear_dump_cache(etpCache['dbSearch']+etpConst['clientdbid'])
+
+        # update world available cache
+        if self.Entropy.xcache and (self.action in ("remove","install")):
+            c_hash = self.Entropy.get_available_packges_chash(etpConst['branch'])
+            disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_available'])
+            if disk_cache != None:
+                try:
+                    if disk_cache['chash'] == c_hash:
+
+                        # remove and old install
+                        if self.infoDict['removeidpackage'] != -1:
+                            key = self.Entropy.entropyTools.dep_getkey(self.infoDict['removeatom'])
+                            slot = self.infoDict['slot']
+                            matches = self.Entropy.atomMatch(key, matchSlot = slot, multiRepo = True, multiMatch = True)
+                            if matches[1] == 0:
+                                disk_cache['available'].update(matches[0])
+
+                        # install, doing here because matches[0] could contain self.matched_atoms
+                        if self.matched_atom in disk_cache['available']:
+                            disk_cache['available'].remove(self.matched_atom)
+
+                        self.Entropy.dumpTools.dumpobj(etpCache['world_available'],disk_cache)
+
+                except KeyError:
+                    try:
+                        self.Entropy.dumpTools.dumpobj(etpCache['world_available'],{})
+                    except IOError:
+                        pass
 
     '''
     @description: install unpacked files, update database and also update gentoo db if requested
@@ -3130,6 +3159,7 @@ class PackageInterface:
         self.infoDict.clear()
         self.infoDict['triggers'] = {}
         self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage)
+        self.infoDict['slot'] = self.Entropy.clientDbconn.retrieveSlot(idpackage)
         self.infoDict['removeidpackage'] = idpackage
         self.infoDict['diffremoval'] = False
         removeConfig = False
@@ -3646,7 +3676,9 @@ class RepoInterface:
 
     def clear_repository_cache(self, repo):
         self.__validate_repository_id(repo)
-        self.Entropy.clear_dump_cache(etpCache['dbInfo'])
+        self.Entropy.clear_dump_cache(etpCache['dbInfo']+repo)
+        self.Entropy.clear_dump_cache(etpCache['dbMatch']+repo)
+        self.Entropy.clear_dump_cache(etpCache['dbSearch']+repo)
 
     # this function can be reimplemented
     def download_item(self, item, repo, cmethod = None):

libraries/entropyConstants.py

@@ -344,8 +344,9 @@ ETP_LOGPRI_ERROR = "[ ERROR ]"
 # disk caching dictionary
 etpCache = {
     'configfiles': 'scanfs', # used to store information about files that should be merged using "equo conf merge"
-    'dbMatch': 'cache_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (dep solving)
-    'dbInfo': 'info_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (info retrival)
+    'dbMatch': 'cache_', # db atom match cache
+    'dbInfo': 'info_', # db data retrieval cache
+    'dbSearch': 'search_', # db search cache
     'atomMatch': 'atom_match_', # used to store info about repository dependencies solving
     'install': 'resume_install', # resume cache (install)
     'remove': 'resume_remove', # resume cache (remove)
@@ -621,6 +622,9 @@ def initConfig_entropyConstants(rootdir):
             'x11-drivers/ati-drivers': ['x11-drivers/ati-drivers'],
         },
 
+        'clientdbid': "client",
+        'serverdbid': "etpdb",
+
     }
     etpConst.update(myConst)
     del myConst