- added package.mask support

- added package.unmask support
- great atomMatch speed improvements on both caching and calculation side


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@771 cd1c1023-2f26-0410-ae45-c471fc1f0318

conf/packages/package.mask

@@ -0,0 +1,18 @@
+# package.mask example file
+#
+# In this file you can specify atoms, one per line, that you would mask.
+# Masking a package (atom) means that you will deny Entropy to pull in a package even if it's marked as "NOT experimental".
+
+# LINE CONSTRUCTION:
+# <atom>
+# See examples below
+
+# EXAMPLES:
+# >=media-libs/foo-1.2.3
+# media-libs/foo
+# <media-libs/foo-1.2.3
+# media-libs/foo:1
+# >=media-libs/foo-1.2.3#2.6.23-sabayon-r1
+# 
+# :1 means package with SLOT="1"
+# #2.6.23-sabayon-r1 means package with kernel tag = 2.6.23-sabayon-r1

conf/packages/package.unmask

@@ -0,0 +1,18 @@
+# package.unmask example file
+#
+# In this file you can specify atoms, one per line, that you would unmask.
+# Unmasking a package (atom) means that you will allow Entropy to pull in a package even if it's marked as "experimental".
+
+# LINE CONSTRUCTION:
+# <atom>
+# See examples below
+
+# EXAMPLES:
+# >=media-libs/foo-1.2.3
+# media-libs/foo
+# <media-libs/foo-1.2.3
+# media-libs/foo:1
+# >=media-libs/foo-1.2.3#2.6.23-sabayon-r1
+# 
+# :1 means package with SLOT="1"
+# #2.6.23-sabayon-r1 means package with kernel tag = 2.6.23-sabayon-r1

libraries/databaseTools.py

@@ -2942,58 +2942,59 @@ class etpDatabase:
 ##   Dependency handling functions
 #
 
-    def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording):
+    def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter):
         if (self.xcache):
             try:
-                cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get(atom)
+                cache_tuple = (atom,matchSlot,matchTag,multiMatch,caseSensitive,matchBranches,packagesFilter)
+                cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get((cache_tuple))
                 if cached:
-                    if (matchSlot == cached['matchSlot']) \
-                            and (multiMatch == cached['multiMatch']) \
-                            and (caseSensitive == cached['caseSensitive']) \
-                            and (matchTag == cached['matchTag']) \
-                            and (keywording == cached['keywording']) \
-                            and (matchBranches == cached['matchBranches']):
-                        return cached['result']
-                    else:
-                        return None
+                    return cached
                 else:
-                    #print atom
-                    #print self.dbname
-                    #print "not in cache"
                     return None
             except KeyError: # issues with dictionaries?
                 return None
         else:
             return None
 
-    def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording):
+    def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter):
         try:
             #print "CACHING",result,atom,self.dbname
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom] = {}
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchSlot'] = matchSlot
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchTag'] = matchTag
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['multiMatch'] = multiMatch
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['caseSensitive'] = caseSensitive
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchBranches'] = matchBranches
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['keywording'] = keywording
-            dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['result'] = result
+            cache_tuple = (atom,matchSlot,matchTag,multiMatch,caseSensitive,matchBranches,packagesFilter)
+            dbCacheStore[etpCache['dbMatch']+self.dbname][cache_tuple] = result
         except KeyError: # againnn, issues with dicts??
             pass
 
     # function that validate one atom by reading keywords settings
-    # keywordValidatorCache = {} << done in entropyConstants
-    def keywordsIdValidator(self,idpackage):
+    # idpackageValidatorCache = {} >> function cache
+    def idpackageValidator(self,idpackage):
         
         reponame = self.dbname[5:]
-        cached = keywordValidatorCache.get((idpackage,reponame))
+        cached = idpackageValidatorCache.get((idpackage,reponame))
         if cached != None: return cached
         
+        # check if package.mask need it masked
+        for atom in etpConst['packagemasking']['mask']:
+            matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
+            if idpackage in matches[0]:
+                # sorry, masked
+                idpackageValidatorCache[(idpackage,reponame)] = -1
+                return -1
+        
         mykeywords = self.retrieveKeywords(idpackage)
         # firstly, check if package keywords are in etpConst['keywords'] (universal keywords have been merged from package.mask)
         for key in etpConst['keywords']:
             if key in mykeywords:
                 # found! all fine
-                keywordValidatorCache[(idpackage,reponame)] = idpackage
+                idpackageValidatorCache[(idpackage,reponame)] = idpackage
+                return idpackage
+        
+        #### IT IS MASKED!!
+        
+        # see if we can unmask by just lookin into package.unmask stuff -> etpConst['packagemasking']['unmask']
+        for atom in etpConst['packagemasking']['unmask']:
+            matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
+            if idpackage in matches[0]:
+                idpackageValidatorCache[(idpackage,reponame)] = idpackage
                 return idpackage
         
         # if we get here, it means we didn't find mykeywords in etpConst['keywords'], we need to seek etpConst['packagemasking']['keywords']
@@ -3005,9 +3006,9 @@ class etpDatabase:
                     for atom in keyword_data:
                         if atom == "*": # all packages in this repo with keyword "keyword" are ok
                             return idpackage
-                        matches = self.atomMatch(atom, multiMatch = True, keywording = False)
+                        matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                         if idpackage in matches[0]:
-                            keywordValidatorCache[(idpackage,reponame)] = idpackage
+                            idpackageValidatorCache[(idpackage,reponame)] = idpackage
                             return idpackage
 
         # if we get here, it means we didn't find a match in repositories
@@ -3020,19 +3021,19 @@ class etpDatabase:
                 # check for relation
                 for atom in keyword_data:
                     # match atom
-                    matches = self.atomMatch(atom, multiMatch = True, keywording = False)
+                    matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                     if idpackage in matches[0]:
                         # valid!
-                        keywordValidatorCache[(idpackage,reponame)] = idpackage
+                        idpackageValidatorCache[(idpackage,reponame)] = idpackage
                         return idpackage
         
         # holy crap, can't validate
-        keywordValidatorCache[(idpackage,reponame)] = -1
+        idpackageValidatorCache[(idpackage,reponame)] = -1
         return -1
 
-    # keywords filter used by atomMatch, input must me foundIDs, a list like this:
+    # packages filter used by atomMatch, input must me foundIDs, a list like this:
     # [(u'x11-libs/qt-4.3.2', 608), (u'x11-libs/qt-3.3.8-r4', 1867)]
-    def keywordsFilter(self,results):
+    def packagesFilter(self,results):
         
         # keywordsFilter ONLY FILTERS results if self.dbname.startswith(etpConst['dbnamerepoprefix']), repository database is open
         if not self.dbname.startswith(etpConst['dbnamerepoprefix']):
@@ -3040,7 +3041,7 @@ class etpDatabase:
 
         newresults = []
         for item in results:
-            rc = self.keywordsIdValidator(item[1])
+            rc = self.idpackageValidator(item[1])
             if rc != -1:
                 newresults.append(item)
         
@@ -3052,12 +3053,12 @@ class etpDatabase:
        @input caseSensitive: bool, should the atom be parsed case sensitive?
        @input matchSlot: string, match atoms with the provided slot
        @input multiMatch: bool, return all the available atoms
-       @input keywording: enable/disable keywords filter
+       @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter
        @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = not found, 2 = need more info, 3 = cannot use direction without specifying version
     '''
-    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, keywording = True):
+    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True):
 
-        cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,keywording)
+        cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter)
         if cached != None:
             return cached
 	
@@ -3162,7 +3163,7 @@ class etpDatabase:
 	        if (not foundCat) and (mypkgcat == "null"):
 		    # got the issue
 		    # gosh, return and complain
-                    self.atomMatchStoreCache((-1,2), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((-1,2), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return -1,2
 	
 	        # I can use foundCat
@@ -3196,8 +3197,8 @@ class etpDatabase:
 	            foundIDs.append(results[0])
 	            break
         
-        if keywording: # keyword filtering
-            foundIDs = self.keywordsFilter(foundIDs)
+        if packagesFilter: # keyword filtering
+            foundIDs = self.packagesFilter(foundIDs)
         
         if (foundIDs):
 	    # now we have to handle direction
@@ -3205,7 +3206,7 @@ class etpDatabase:
 	        # check if direction is used with justname, in this case, return an error
 	        if (justname):
 		    #print "justname"
-                    self.atomMatchStoreCache((-1,3), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((-1,3), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return -1,3 # error, cannot use directions when not specifying version
 	    
 	        if (direction == "~") or (direction == "=") or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")): # any revision within the version specified OR the specified version
@@ -3242,7 +3243,7 @@ class etpDatabase:
 				    dbpkginfo.append([idpackage,dbver])
 
 		    if (not dbpkginfo):
-                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		        return -1,1
 		
 		    versions = []
@@ -3257,7 +3258,7 @@ class etpDatabase:
 		        versions.append(x[1])
 		
 		    if (not versions):
-                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 			return -1,1
 		
 		    # who is newer ?
@@ -3273,7 +3274,7 @@ class etpDatabase:
 		    
 		    if (multiMatch):
                         # filter only valid keywords
-                        self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 			return similarPackages,0
 		    
 		    #print newerPackage
@@ -3288,7 +3289,7 @@ class etpDatabase:
 		        newerPackage = similarPackages[versionTags.index(versiontaglist[0])]
 		
                     # filter only valid keywords
-                    self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return newerPackage[0],0
 
 	        elif (direction.find(">") != -1) or (direction.find("<") != -1):
@@ -3323,7 +3324,7 @@ class etpDatabase:
 		
 		    if (not dbpkginfo):
 		        # this version is not available
-                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		        return -1,1
 		
 		    versions = []
@@ -3344,11 +3345,11 @@ class etpDatabase:
 		    dbpkginfo = _dbpkginfo
 		    
 		    if (multiMatch):
-                        self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 			return multiMatchList,0
 
 		    if (not versions):
-                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 			return -1,1
 
 		    # who is newer ?
@@ -3363,7 +3364,7 @@ class etpDatabase:
 	            similarPackages = self.searchPackagesByNameAndVersionAndCategory(name = newerPkgName, version = newerPkgVersion, category = newerPkgCategory, branch = newerPkgBranch)
 
 		    if (multiMatch):
-                        self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                        self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 			return similarPackages,0
 
 		    #print newerPackage
@@ -3378,11 +3379,11 @@ class etpDatabase:
 		        newerPackage = similarPackages[versionTags.index(versiontaglist[0])]
 		
 
-                    self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return newerPackage[0],0
 
 	        else:
-                    self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return -1,1
 		
 	    else:
@@ -3414,11 +3415,11 @@ class etpDatabase:
 		foundIDs = _foundIDs
 	    
 		if (multiMatch):
-                    self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return multiMatchList,0
 		
 		if (not versionIDs):
-                    self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                    self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 		    return -1,1
 		
 	        versionlist = entropyTools.getNewerVersion(versionIDs)
@@ -3440,10 +3441,10 @@ class etpDatabase:
 		    versiontaglist = entropyTools.getNewerVersionTag(versionTags)
 		    newerPackage = similarPackages[versionTags.index(versiontaglist[0])]
 
-                self.atomMatchStoreCache((newerPackage[1],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+                self.atomMatchStoreCache((newerPackage[1],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 	        return newerPackage[1],0
 
         else:
 	    # package not found in any branch
-            self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording)
+            self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter)
 	    return -1,1

libraries/entropyConstants.py

@@ -476,13 +476,13 @@ dbCacheStore = {}
 atomMatchCache = {}
 atomClientMatchCache = {}
 generateDependsTreeCache = {}
-keywordValidatorCache = {}
+idpackageValidatorCache = {}
 def const_resetCache():
     dbCacheStore.clear()
     atomMatchCache.clear()
     atomClientMatchCache.clear()
     generateDependsTreeCache.clear()
-    keywordValidatorCache.clear()
+    idpackageValidatorCache.clear()
 
 # handle Entropy Version
 ETP_REVISION_FILE = "../libraries/revision"

libraries/maskingparser.py

@@ -23,11 +23,15 @@
 from entropyConstants import ETP_CONF_DIR, etpConst, etpCache, os
 
 etpMaskFiles = {
-    'keywords': ETP_CONF_DIR+"/packages/package.keywords", # keywords configuration files
+    'keywords': ETP_CONF_DIR+"/packages/package.keywords", # keywording configuration files
+    'unmask': ETP_CONF_DIR+"/packages/package.unmask", # unmasking configuration files
+    'mask': ETP_CONF_DIR+"/packages/package.mask", # masking configuration files
 }
 
-etpMtimesFiles = {
-    'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", # keywords configuration files mtime
+etpMtimeFiles = {
+    'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", # keywording configuration files mtime
+    'unmask_mtime': etpConst['dumpstoragedir']+"/unmask.mtime", # unmasking configuration files mtime
+    'mask_mtime': etpConst['dumpstoragedir']+"/mask.mtime", # masking configuration files mtime
 }
 
 '''
@@ -45,31 +49,7 @@ def parse():
 '''
 def keywords_parser():
     
-    # handle on-disk cache validation
-    # in this case, repositories cache
-    # if package.keywords is changed, we must destroy cache
-    if not os.path.isfile(etpMtimesFiles['keywords_mtime']):
-        # we can't know if package.keywords has been updated
-        # remove repositories caches
-        __removeRepoCache()
-        __savePackageKeywordsMtime()
-    else:
-        # check mtime
-        try:
-            f = open(etpMtimesFiles['keywords_mtime'],"r")
-            mtime = int(f.readline().strip())
-            # compare with current mtime
-            try:
-                currmtime = os.path.getmtime(etpMaskFiles['keywords'])
-            except OSError:
-                currmtime = 0
-            if mtime != currmtime:
-                __removeRepoCache()
-                __savePackageKeywordsMtime()
-        except:
-            __removeRepoCache()
-            __savePackageKeywordsMtime()
-
+    __validateEntropyCache(etpMaskFiles['keywords'],etpMtimeFiles['keywords_mtime'])
     
     data = {
             'universal': set(),
@@ -133,6 +113,38 @@ def keywords_parser():
     return data
 
 
+def unmask_parser():
+    __validateEntropyCache(etpMaskFiles['unmask'],etpMtimeFiles['unmask_mtime'])
+    
+    data = set()
+    if os.path.isfile(etpMaskFiles['unmask']):
+        f = open(etpMaskFiles['unmask'],"r")
+        content = f.readlines()
+        f.close()
+        # filter comments and white lines
+        content = [x.strip() for x in content if not x.startswith("#") and x.strip()]
+        for line in content:
+            # FIXME: need validation? probably not since atomMatch handles it all
+            # and doesn't care about badly formatted atoms
+            data.add(line)
+    return data
+
+def mask_parser():
+    __validateEntropyCache(etpMaskFiles['mask'],etpMtimeFiles['mask_mtime'])
+    
+    data = set()
+    if os.path.isfile(etpMaskFiles['mask']):
+        f = open(etpMaskFiles['mask'],"r")
+        content = f.readlines()
+        f.close()
+        # filter comments and white lines
+        content = [x.strip() for x in content if not x.startswith("#") and x.strip()]
+        for line in content:
+            # FIXME: need validation? probably not since atomMatch handles it all
+            # and doesn't care about badly formatted atoms
+            data.add(line)
+    return data
+
 '''
    internal functions
 '''
@@ -143,18 +155,44 @@ def __removeRepoCache():
         if file.startswith(etpCache['dbMatch']+etpConst['dbnamerepoprefix']) and file.endswith(".dmp"):
             os.remove(etpConst['dumpstoragedir']+"/"+file)
 
-def __savePackageKeywordsMtime():
+def __saveFileMtime(toread,tosaveinto):
     
-    if not os.path.isfile(etpMaskFiles['keywords']):
-        currmtime = 0
+    if not os.path.isfile(toread):
+        currmtime = 0.0
     else:
-        currmtime = os.path.getmtime(etpMaskFiles['keywords'])
+        currmtime = os.path.getmtime(toread)
     
     if not os.path.isdir(etpConst['dumpstoragedir']):
         os.makedirs(etpConst['dumpstoragedir'])
     
-    f = open(etpMtimesFiles['keywords_mtime'],"w")
+    f = open(tosaveinto,"w")
     f.write(str(currmtime))
     f.flush()
     f.close()
     
+
+def __validateEntropyCache(maskfile,mtimefile):
+    # handle on-disk cache validation
+    # in this case, repositories cache
+    # if package.keywords is changed, we must destroy cache
+    if not os.path.isfile(mtimefile):
+        # we can't know if package.keywords has been updated
+        # remove repositories caches
+        __removeRepoCache()
+        __saveFileMtime(maskfile,mtimefile)
+    else:
+        # check mtime
+        try:
+            f = open(mtimefile,"r")
+            mtime = float(f.readline().strip())
+            # compare with current mtime
+            try:
+                currmtime = os.path.getmtime(maskfile)
+            except OSError:
+                currmtime = 0.0
+            if mtime != currmtime:
+                __removeRepoCache()
+                __saveFileMtime(maskfile,mtimefile)
+        except:
+            __removeRepoCache()
+            __saveFileMtime(maskfile,mtimefile)