From 45fc794fc67c93ae59703cee290efc3bbfbb9c60 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Wed, 28 Nov 2007 14:28:01 +0000 Subject: [PATCH] - added package.mask support - added package.unmask support - great atomMatch speed improvements on both caching and calculation side git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@771 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- conf/packages/package.mask | 18 ++++++ conf/packages/package.unmask | 18 ++++++ libraries/databaseTools.py | 113 +++++++++++++++++----------------- libraries/entropyConstants.py | 4 +- libraries/maskingparser.py | 104 +++++++++++++++++++++---------- 5 files changed, 166 insertions(+), 91 deletions(-) create mode 100644 conf/packages/package.mask create mode 100644 conf/packages/package.unmask diff --git a/conf/packages/package.mask b/conf/packages/package.mask new file mode 100644 index 000000000..c836e4f8e --- /dev/null +++ b/conf/packages/package.mask @@ -0,0 +1,18 @@ +# package.mask example file +# +# In this file you can specify atoms, one per line, that you would mask. +# Masking a package (atom) means that you will deny Entropy to pull in a package even if it's marked as "NOT experimental". + +# LINE CONSTRUCTION: +# +# See examples below + +# EXAMPLES: +# >=media-libs/foo-1.2.3 +# media-libs/foo +# =media-libs/foo-1.2.3#2.6.23-sabayon-r1 +# +# :1 means package with SLOT="1" +# #2.6.23-sabayon-r1 means package with kernel tag = 2.6.23-sabayon-r1 \ No newline at end of file diff --git a/conf/packages/package.unmask b/conf/packages/package.unmask new file mode 100644 index 000000000..32daad7ed --- /dev/null +++ b/conf/packages/package.unmask @@ -0,0 +1,18 @@ +# package.unmask example file +# +# In this file you can specify atoms, one per line, that you would unmask. +# Unmasking a package (atom) means that you will allow Entropy to pull in a package even if it's marked as "experimental". + +# LINE CONSTRUCTION: +# +# See examples below + +# EXAMPLES: +# >=media-libs/foo-1.2.3 +# media-libs/foo +# =media-libs/foo-1.2.3#2.6.23-sabayon-r1 +# +# :1 means package with SLOT="1" +# #2.6.23-sabayon-r1 means package with kernel tag = 2.6.23-sabayon-r1 \ No newline at end of file diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index eb99acb73..e871d3bd1 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -2942,58 +2942,59 @@ class etpDatabase: ## Dependency handling functions # - def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording): + def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter): if (self.xcache): try: - cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get(atom) + cache_tuple = (atom,matchSlot,matchTag,multiMatch,caseSensitive,matchBranches,packagesFilter) + cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get((cache_tuple)) if cached: - if (matchSlot == cached['matchSlot']) \ - and (multiMatch == cached['multiMatch']) \ - and (caseSensitive == cached['caseSensitive']) \ - and (matchTag == cached['matchTag']) \ - and (keywording == cached['keywording']) \ - and (matchBranches == cached['matchBranches']): - return cached['result'] - else: - return None + return cached else: - #print atom - #print self.dbname - #print "not in cache" return None except KeyError: # issues with dictionaries? return None else: return None - def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording): + def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter): try: #print "CACHING",result,atom,self.dbname - dbCacheStore[etpCache['dbMatch']+self.dbname][atom] = {} - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchSlot'] = matchSlot - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchTag'] = matchTag - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['multiMatch'] = multiMatch - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['caseSensitive'] = caseSensitive - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['matchBranches'] = matchBranches - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['keywording'] = keywording - dbCacheStore[etpCache['dbMatch']+self.dbname][atom]['result'] = result + cache_tuple = (atom,matchSlot,matchTag,multiMatch,caseSensitive,matchBranches,packagesFilter) + dbCacheStore[etpCache['dbMatch']+self.dbname][cache_tuple] = result except KeyError: # againnn, issues with dicts?? pass # function that validate one atom by reading keywords settings - # keywordValidatorCache = {} << done in entropyConstants - def keywordsIdValidator(self,idpackage): + # idpackageValidatorCache = {} >> function cache + def idpackageValidator(self,idpackage): reponame = self.dbname[5:] - cached = keywordValidatorCache.get((idpackage,reponame)) + cached = idpackageValidatorCache.get((idpackage,reponame)) if cached != None: return cached + # check if package.mask need it masked + for atom in etpConst['packagemasking']['mask']: + matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) + if idpackage in matches[0]: + # sorry, masked + idpackageValidatorCache[(idpackage,reponame)] = -1 + return -1 + mykeywords = self.retrieveKeywords(idpackage) # firstly, check if package keywords are in etpConst['keywords'] (universal keywords have been merged from package.mask) for key in etpConst['keywords']: if key in mykeywords: # found! all fine - keywordValidatorCache[(idpackage,reponame)] = idpackage + idpackageValidatorCache[(idpackage,reponame)] = idpackage + return idpackage + + #### IT IS MASKED!! + + # see if we can unmask by just lookin into package.unmask stuff -> etpConst['packagemasking']['unmask'] + for atom in etpConst['packagemasking']['unmask']: + matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) + if idpackage in matches[0]: + idpackageValidatorCache[(idpackage,reponame)] = idpackage return idpackage # if we get here, it means we didn't find mykeywords in etpConst['keywords'], we need to seek etpConst['packagemasking']['keywords'] @@ -3005,9 +3006,9 @@ class etpDatabase: for atom in keyword_data: if atom == "*": # all packages in this repo with keyword "keyword" are ok return idpackage - matches = self.atomMatch(atom, multiMatch = True, keywording = False) + matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if idpackage in matches[0]: - keywordValidatorCache[(idpackage,reponame)] = idpackage + idpackageValidatorCache[(idpackage,reponame)] = idpackage return idpackage # if we get here, it means we didn't find a match in repositories @@ -3020,19 +3021,19 @@ class etpDatabase: # check for relation for atom in keyword_data: # match atom - matches = self.atomMatch(atom, multiMatch = True, keywording = False) + matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if idpackage in matches[0]: # valid! - keywordValidatorCache[(idpackage,reponame)] = idpackage + idpackageValidatorCache[(idpackage,reponame)] = idpackage return idpackage # holy crap, can't validate - keywordValidatorCache[(idpackage,reponame)] = -1 + idpackageValidatorCache[(idpackage,reponame)] = -1 return -1 - # keywords filter used by atomMatch, input must me foundIDs, a list like this: + # packages filter used by atomMatch, input must me foundIDs, a list like this: # [(u'x11-libs/qt-4.3.2', 608), (u'x11-libs/qt-3.3.8-r4', 1867)] - def keywordsFilter(self,results): + def packagesFilter(self,results): # keywordsFilter ONLY FILTERS results if self.dbname.startswith(etpConst['dbnamerepoprefix']), repository database is open if not self.dbname.startswith(etpConst['dbnamerepoprefix']): @@ -3040,7 +3041,7 @@ class etpDatabase: newresults = [] for item in results: - rc = self.keywordsIdValidator(item[1]) + rc = self.idpackageValidator(item[1]) if rc != -1: newresults.append(item) @@ -3052,12 +3053,12 @@ class etpDatabase: @input caseSensitive: bool, should the atom be parsed case sensitive? @input matchSlot: string, match atoms with the provided slot @input multiMatch: bool, return all the available atoms - @input keywording: enable/disable keywords filter + @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = not found, 2 = need more info, 3 = cannot use direction without specifying version ''' - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, keywording = True): + def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True): - cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,keywording) + cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter) if cached != None: return cached @@ -3162,7 +3163,7 @@ class etpDatabase: if (not foundCat) and (mypkgcat == "null"): # got the issue # gosh, return and complain - self.atomMatchStoreCache((-1,2), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,2), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,2 # I can use foundCat @@ -3196,8 +3197,8 @@ class etpDatabase: foundIDs.append(results[0]) break - if keywording: # keyword filtering - foundIDs = self.keywordsFilter(foundIDs) + if packagesFilter: # keyword filtering + foundIDs = self.packagesFilter(foundIDs) if (foundIDs): # now we have to handle direction @@ -3205,7 +3206,7 @@ class etpDatabase: # check if direction is used with justname, in this case, return an error if (justname): #print "justname" - self.atomMatchStoreCache((-1,3), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,3), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,3 # error, cannot use directions when not specifying version if (direction == "~") or (direction == "=") or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")): # any revision within the version specified OR the specified version @@ -3242,7 +3243,7 @@ class etpDatabase: dbpkginfo.append([idpackage,dbver]) if (not dbpkginfo): - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 versions = [] @@ -3257,7 +3258,7 @@ class etpDatabase: versions.append(x[1]) if (not versions): - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 # who is newer ? @@ -3273,7 +3274,7 @@ class etpDatabase: if (multiMatch): # filter only valid keywords - self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return similarPackages,0 #print newerPackage @@ -3288,7 +3289,7 @@ class etpDatabase: newerPackage = similarPackages[versionTags.index(versiontaglist[0])] # filter only valid keywords - self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return newerPackage[0],0 elif (direction.find(">") != -1) or (direction.find("<") != -1): @@ -3323,7 +3324,7 @@ class etpDatabase: if (not dbpkginfo): # this version is not available - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 versions = [] @@ -3344,11 +3345,11 @@ class etpDatabase: dbpkginfo = _dbpkginfo if (multiMatch): - self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return multiMatchList,0 if (not versions): - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 # who is newer ? @@ -3363,7 +3364,7 @@ class etpDatabase: similarPackages = self.searchPackagesByNameAndVersionAndCategory(name = newerPkgName, version = newerPkgVersion, category = newerPkgCategory, branch = newerPkgBranch) if (multiMatch): - self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((similarPackages,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return similarPackages,0 #print newerPackage @@ -3378,11 +3379,11 @@ class etpDatabase: newerPackage = similarPackages[versionTags.index(versiontaglist[0])] - self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((newerPackage[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return newerPackage[0],0 else: - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 else: @@ -3414,11 +3415,11 @@ class etpDatabase: foundIDs = _foundIDs if (multiMatch): - self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((multiMatchList,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return multiMatchList,0 if (not versionIDs): - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 versionlist = entropyTools.getNewerVersion(versionIDs) @@ -3440,10 +3441,10 @@ class etpDatabase: versiontaglist = entropyTools.getNewerVersionTag(versionTags) newerPackage = similarPackages[versionTags.index(versiontaglist[0])] - self.atomMatchStoreCache((newerPackage[1],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((newerPackage[1],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return newerPackage[1],0 else: # package not found in any branch - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, keywording) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter) return -1,1 diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 62a6e64a2..82c7ff5db 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -476,13 +476,13 @@ dbCacheStore = {} atomMatchCache = {} atomClientMatchCache = {} generateDependsTreeCache = {} -keywordValidatorCache = {} +idpackageValidatorCache = {} def const_resetCache(): dbCacheStore.clear() atomMatchCache.clear() atomClientMatchCache.clear() generateDependsTreeCache.clear() - keywordValidatorCache.clear() + idpackageValidatorCache.clear() # handle Entropy Version ETP_REVISION_FILE = "../libraries/revision" diff --git a/libraries/maskingparser.py b/libraries/maskingparser.py index b8770b065..846ae4bf7 100644 --- a/libraries/maskingparser.py +++ b/libraries/maskingparser.py @@ -23,11 +23,15 @@ from entropyConstants import ETP_CONF_DIR, etpConst, etpCache, os etpMaskFiles = { - 'keywords': ETP_CONF_DIR+"/packages/package.keywords", # keywords configuration files + 'keywords': ETP_CONF_DIR+"/packages/package.keywords", # keywording configuration files + 'unmask': ETP_CONF_DIR+"/packages/package.unmask", # unmasking configuration files + 'mask': ETP_CONF_DIR+"/packages/package.mask", # masking configuration files } -etpMtimesFiles = { - 'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", # keywords configuration files mtime +etpMtimeFiles = { + 'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", # keywording configuration files mtime + 'unmask_mtime': etpConst['dumpstoragedir']+"/unmask.mtime", # unmasking configuration files mtime + 'mask_mtime': etpConst['dumpstoragedir']+"/mask.mtime", # masking configuration files mtime } ''' @@ -45,31 +49,7 @@ def parse(): ''' def keywords_parser(): - # handle on-disk cache validation - # in this case, repositories cache - # if package.keywords is changed, we must destroy cache - if not os.path.isfile(etpMtimesFiles['keywords_mtime']): - # we can't know if package.keywords has been updated - # remove repositories caches - __removeRepoCache() - __savePackageKeywordsMtime() - else: - # check mtime - try: - f = open(etpMtimesFiles['keywords_mtime'],"r") - mtime = int(f.readline().strip()) - # compare with current mtime - try: - currmtime = os.path.getmtime(etpMaskFiles['keywords']) - except OSError: - currmtime = 0 - if mtime != currmtime: - __removeRepoCache() - __savePackageKeywordsMtime() - except: - __removeRepoCache() - __savePackageKeywordsMtime() - + __validateEntropyCache(etpMaskFiles['keywords'],etpMtimeFiles['keywords_mtime']) data = { 'universal': set(), @@ -133,6 +113,38 @@ def keywords_parser(): return data +def unmask_parser(): + __validateEntropyCache(etpMaskFiles['unmask'],etpMtimeFiles['unmask_mtime']) + + data = set() + if os.path.isfile(etpMaskFiles['unmask']): + f = open(etpMaskFiles['unmask'],"r") + content = f.readlines() + f.close() + # filter comments and white lines + content = [x.strip() for x in content if not x.startswith("#") and x.strip()] + for line in content: + # FIXME: need validation? probably not since atomMatch handles it all + # and doesn't care about badly formatted atoms + data.add(line) + return data + +def mask_parser(): + __validateEntropyCache(etpMaskFiles['mask'],etpMtimeFiles['mask_mtime']) + + data = set() + if os.path.isfile(etpMaskFiles['mask']): + f = open(etpMaskFiles['mask'],"r") + content = f.readlines() + f.close() + # filter comments and white lines + content = [x.strip() for x in content if not x.startswith("#") and x.strip()] + for line in content: + # FIXME: need validation? probably not since atomMatch handles it all + # and doesn't care about badly formatted atoms + data.add(line) + return data + ''' internal functions ''' @@ -143,18 +155,44 @@ def __removeRepoCache(): if file.startswith(etpCache['dbMatch']+etpConst['dbnamerepoprefix']) and file.endswith(".dmp"): os.remove(etpConst['dumpstoragedir']+"/"+file) -def __savePackageKeywordsMtime(): +def __saveFileMtime(toread,tosaveinto): - if not os.path.isfile(etpMaskFiles['keywords']): - currmtime = 0 + if not os.path.isfile(toread): + currmtime = 0.0 else: - currmtime = os.path.getmtime(etpMaskFiles['keywords']) + currmtime = os.path.getmtime(toread) if not os.path.isdir(etpConst['dumpstoragedir']): os.makedirs(etpConst['dumpstoragedir']) - f = open(etpMtimesFiles['keywords_mtime'],"w") + f = open(tosaveinto,"w") f.write(str(currmtime)) f.flush() f.close() + +def __validateEntropyCache(maskfile,mtimefile): + # handle on-disk cache validation + # in this case, repositories cache + # if package.keywords is changed, we must destroy cache + if not os.path.isfile(mtimefile): + # we can't know if package.keywords has been updated + # remove repositories caches + __removeRepoCache() + __saveFileMtime(maskfile,mtimefile) + else: + # check mtime + try: + f = open(mtimefile,"r") + mtime = float(f.readline().strip()) + # compare with current mtime + try: + currmtime = os.path.getmtime(maskfile) + except OSError: + currmtime = 0.0 + if mtime != currmtime: + __removeRepoCache() + __saveFileMtime(maskfile,mtimefile) + except: + __removeRepoCache() + __saveFileMtime(maskfile,mtimefile)