From b54fd3bc9852463bf70d19af286b8d76d67ac270 Mon Sep 17 00:00:00 2001 From: lxnay Date: Sun, 13 Apr 2008 19:29:30 +0000 Subject: [PATCH] Entropy: - added fakeoutfile, fakeinfile classes to prepare portage.doebuild() logging - improved Entropy idpackageValidator speed by 1000% (by adding a SQL index on keywords), this improved world updates calculation speed by a big 1000% - misc updates and fixes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1721 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- client/text_ui.py | 6 +- libraries/databaseTools.py | 232 +++++++++++++++++++---------- libraries/entropy.py | 298 +++++++++++++++++++++++++++---------- libraries/entropyTools.py | 3 + spritz/src/spritz.py | 2 +- spritz/src/spritz_setup.py | 72 +++++---- 6 files changed, 417 insertions(+), 196 deletions(-) diff --git a/client/text_ui.py b/client/text_ui.py index 6019bc635..015df716e 100644 --- a/client/text_ui.py +++ b/client/text_ui.py @@ -157,7 +157,7 @@ def worldUpdate(onlyfetch = False, replay = False, upgradeTo = None, resume = Fa resume_cache['ask'] = etpUi['ask'] resume_cache['verbose'] = etpUi['verbose'] resume_cache['onlyfetch'] = onlyfetch - resume_cache['remove'] = remove.copy() + resume_cache['remove'] = remove Equo.dumpTools.dumpobj(etpCache['world'],resume_cache) else: # if resume, load cache if possible @@ -170,12 +170,12 @@ def worldUpdate(onlyfetch = False, replay = False, upgradeTo = None, resume = Fa else: try: update = [] - remove = resume_cache['removed'].copy() + remove = resume_cache['remove'] etpUi['ask'] = resume_cache['ask'] etpUi['verbose'] = resume_cache['verbose'] onlyfetch = resume_cache['onlyfetch'] Equo.dumpTools.dumpobj(etpCache['remove'],list(remove)) - except: + except (IOError,KeyError): print_error(red("Resume cache corrupted.")) Equo.dumpTools.dumpobj(etpCache['world'],{}) Equo.dumpTools.dumpobj(etpCache['install'],{}) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index a3dd32123..6b6abe1ad 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -1574,8 +1574,11 @@ class etpDatabase: break return idcat - def getScopeData(self, idpackage): + def getStrictData(self, idpackage): + self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot,baseinfo.version,baseinfo.versiontag,baseinfo.revision,baseinfo.atom FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,)) + return self.cursor.fetchone() + def getScopeData(self, idpackage): self.cursor.execute(""" SELECT baseinfo.atom, @@ -1590,8 +1593,8 @@ class etpDatabase: baseinfo, categories WHERE - baseinfo.idpackage = (?) - and baseinfo.idcategory = categories.idcategory + baseinfo.idpackage = (?) + and baseinfo.idcategory = categories.idcategory """, (idpackage,)) return self.cursor.fetchone() @@ -2261,8 +2264,7 @@ class etpDatabase: return vtag def retrieveMirrorInfo(self, mirrorname): - - self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) + self.cursor.execute('SELECT mirrorlink FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) mirrorlist = self.fetchall2set(self.cursor.fetchall()) return mirrorlist @@ -2732,7 +2734,7 @@ class etpDatabase: self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE (?)'+slotstring+tagstring+branchstring, searchkeywords) return self.cursor.fetchall() - def searchProvide(self, keyword, slot = None, tag = None, branch = None): + def searchProvide(self, keyword, slot = None, tag = None, branch = None, justid = False): slotstring = '' searchkeywords = [keyword] @@ -2747,33 +2749,45 @@ class etpDatabase: if branch: searchkeywords.append(branch) branchstring = ' and baseinfo.branch = (?)' + atomstring = '' + if not justid: + atomstring = 'baseinfo.atom,' - self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,provide WHERE provide.atom = (?) and provide.idpackage = baseinfo.idpackage'+slotstring+tagstring+branchstring, searchkeywords) + self.cursor.execute('SELECT '+atomstring+'baseinfo.idpackage FROM baseinfo,provide WHERE provide.atom = (?) and provide.idpackage = baseinfo.idpackage'+slotstring+tagstring+branchstring, searchkeywords) - results = self.cursor.fetchall() + if justid: + results = self.fetchall2list(self.cursor.fetchall()) + else: + results = self.cursor.fetchall() return results def searchPackagesByDescription(self, keyword): self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo WHERE LOWER(extrainfo.description) LIKE (?) and baseinfo.idpackage = extrainfo.idpackage', ("%"+keyword.lower()+"%",)) return self.cursor.fetchall() - def searchPackagesByName(self, keyword, sensitive = False, branch = None): + def searchPackagesByName(self, keyword, sensitive = False, branch = None, justid = False): if sensitive: searchkeywords = [keyword] else: searchkeywords = [keyword.lower()] branchstring = '' + atomstring = '' + if not justid: + atomstring = 'atom,' if branch: searchkeywords.append(branch) branchstring = ' and branch = (?)' if sensitive: - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords) + self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords) else: - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) + self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) - results = self.cursor.fetchall() + if justid: + results = self.fetchall2list(self.cursor.fetchall()) + else: + results = self.cursor.fetchall() return results @@ -2794,15 +2808,11 @@ class etpDatabase: return results - def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None): + def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None, justid = False): - namestring = 'baseinfo.name' - catstring = 'categories.category' myname = name mycat = category if not sensitive: - namestring = 'LOWER(baseinfo.name)' - catstring = 'LOWER(categories.category)' myname = name.lower() mycat = category.lower() @@ -2810,11 +2820,21 @@ class etpDatabase: branchstring = '' if branch: searchkeywords.append(branch) - branchstring = ' and baseinfo.branch = (?)' + branchstring = ' and branch = (?)' + atomstring = '' + if not justid: + atomstring = 'atom,' - self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE '+namestring+' = (?) AND '+catstring+' = (?) and baseinfo.idcategory = categories.idcategory'+branchstring, searchkeywords) + if sensitive: + self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE name = (?) AND idcategory IN (SELECT idcategory FROM categories WHERE category = (?))'+branchstring, searchkeywords) + else: + self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory IN (SELECT idcategory FROM categories WHERE LOWER(category) = (?))'+branchstring, searchkeywords) + '' - results = self.cursor.fetchall() + if justid: + results = self.fetchall2list(self.cursor.fetchall()) + else: + results = self.cursor.fetchall() return results def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False): @@ -3407,6 +3427,7 @@ class etpDatabase: def createKeywordsIndex(self): if self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname )') + self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsindex_idpackage ON keywords ( idpackage )') self.commitChanges() def createDependenciesIndex(self): @@ -3611,7 +3632,7 @@ class etpDatabase: ## Dependency handling functions # - def __get_match_hash(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): + def __get_match_hash(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults): c_hash = str(hash(atom)) + \ str(hash(matchSlot)) + \ str(hash(matchTag)) + \ @@ -3620,14 +3641,15 @@ class etpDatabase: str(hash(caseSensitive)) + \ str(hash(multiMatch)) + \ str(hash(packagesFilter)) + \ - str(hash(matchRevision)) + str(hash(matchRevision)) + \ + str(hash(extendedResults)) #c_hash = str(hash(c_hash)) return c_hash - def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): + def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults): if self.xcache: - c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) + c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) try: cached = dumpTools.loadobj(etpCache['dbMatch']+"/"+self.dbname+"/"+c_hash) if cached != None: @@ -3635,9 +3657,9 @@ class etpDatabase: except (EOFError, IOError): return None - def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): + def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults): if self.xcache: - c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) + c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) try: sperms = False if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbMatch']+"/"+self.dbname)): @@ -3655,7 +3677,8 @@ class etpDatabase: reponame = self.dbname[5:] cached = idpackageValidatorCache.get((idpackage,reponame)) - if cached != None: return cached + if cached != None: + return cached # check if user package.mask needs it masked user_package_mask_ids = etpConst['packagemasking'].get(reponame+'mask_ids') @@ -3665,7 +3688,7 @@ class etpDatabase: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue - etpConst['packagemasking'][reponame+'mask_ids'] |= matches[0] + etpConst['packagemasking'][reponame+'mask_ids'] |= set(matches[0]) user_package_mask_ids = etpConst['packagemasking'][reponame+'mask_ids'] if idpackage in user_package_mask_ids: # sorry, masked @@ -3680,7 +3703,7 @@ class etpDatabase: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue - etpConst['packagemasking'][reponame+'unmask_ids'] |= matches[0] + etpConst['packagemasking'][reponame+'unmask_ids'] |= set(matches[0]) user_package_unmask_ids = etpConst['packagemasking'][reponame+'unmask_ids'] if idpackage in user_package_unmask_ids: idpackageValidatorCache[(idpackage,reponame)] = idpackage,3 @@ -3699,7 +3722,7 @@ class etpDatabase: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue - etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] |= matches[0] + etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] |= set(matches[0]) all_branches_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] if idpackage in all_branches_mask_ids: idpackageValidatorCache[(idpackage,reponame)] = -1,8 @@ -3715,7 +3738,7 @@ class etpDatabase: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue - etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] |= matches[0] + etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] |= set(matches[0]) branch_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] if idpackage in branch_mask_ids: if self.retrieveBranch(idpackage) == branch: @@ -3793,24 +3816,25 @@ class etpDatabase: return -1,7 # packages filter used by atomMatch, input must me foundIDs, a list like this: - # [(u'x11-libs/qt-4.3.2', 608), (u'x11-libs/qt-3.3.8-r4', 1867)] - def packagesFilter(self,results, atom): - # keywordsFilter ONLY FILTERS results if self.dbname.startswith(etpConst['dbnamerepoprefix']), repository database is open + # [608,1867] + def packagesFilter(self, results, atom): + # keywordsFilter ONLY FILTERS results if + # self.dbname.startswith(etpConst['dbnamerepoprefix']) => repository database is open if not self.dbname.startswith(etpConst['dbnamerepoprefix']): return results newresults = set() - for item in results: - rc = self.idpackageValidator(item[1]) + for idpackage in results: + rc = self.idpackageValidator(idpackage) if rc[0] != -1: - newresults.add(item) + newresults.add(idpackage) else: idreason = rc[1] if not maskingReasonsStorage.has_key(atom): maskingReasonsStorage[atom] = {} if not maskingReasonsStorage[atom].has_key(idreason): maskingReasonsStorage[atom][idreason] = set() - maskingReasonsStorage[atom][idreason].add((item[1],self.dbname[5:])) + maskingReasonsStorage[atom][idreason].add((idpackage,self.dbname[5:])) return newresults def __filterSlot(self, idpackage, slot): @@ -3844,8 +3868,7 @@ class etpDatabase: def __filterSlotTag(self, foundIDs, slot, tag, operators): newlist = set() - for data in foundIDs: - idpackage = data[1] + for idpackage in foundIDs: idpackage = self.__filterSlot(idpackage, slot) if not idpackage: @@ -3855,7 +3878,7 @@ class etpDatabase: if not idpackage: continue - newlist.add(data) + newlist.add(idpackage) return newlist @@ -3870,12 +3893,12 @@ class etpDatabase: @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error ''' - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True, matchRevision = None): + def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True, matchRevision = None, extendedResults = False): if not atom: return -1,1 - cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter, matchRevision) + cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter, matchRevision,extendedResults) if cached != None: return cached @@ -3923,13 +3946,15 @@ class etpDatabase: pkgname = splitkey[0] pkgcat = "null" - if (matchBranches): - myBranchIndex = tuple(matchBranches) # force to tuple for security + if matchBranches: + # force to tuple for security + myBranchIndex = tuple(matchBranches) else: - if (self.dbname == etpConst['clientdbid']): + if self.dbname == etpConst['clientdbid']: # collect all available branches myBranchIndex = tuple(self.listAllBranches()) - elif self.dbname.startswith(etpConst['dbnamerepoprefix']): # repositories should match to any branch <= than the current if none specified + elif self.dbname.startswith(etpConst['dbnamerepoprefix']): + # repositories should match to any branch <= than the current if none specified allbranches = set([x for x in self.listAllBranches() if x <= etpConst['branch']]) allbranches = list(allbranches) allbranches.reverse() @@ -3943,42 +3968,57 @@ class etpDatabase: foundIDs = set() for idx in myBranchIndex: - results = self.searchPackagesByName(pkgname, sensitive = caseSensitive, branch = idx) + + if pkgcat == "null": + results = self.searchPackagesByName( + pkgname, + sensitive = caseSensitive, + branch = idx, + justid = True + ) + else: + results = self.searchPackagesByNameAndCategory( + name = pkgname, + category = pkgcat, + branch = idx, + sensitive = caseSensitive, + justid = True + ) mypkgcat = pkgcat mypkgname = pkgname - virtual = False # if it's a PROVIDE, search with searchProvide # there's no package with that name if (not results) and (mypkgcat == "virtual"): - virtuals = self.searchProvide(pkgkey, branch = idx) - if (virtuals): + virtuals = self.searchProvide(pkgkey, branch = idx, justid = True) + if virtuals: virtual = True - mypkgname = self.retrieveName(virtuals[0][1]) - mypkgcat = self.retrieveCategory(virtuals[0][1]) + mypkgname = self.retrieveName(virtuals[0]) + mypkgcat = self.retrieveCategory(virtuals[0]) results = virtuals # now validate - if (not results): + if not results: continue # search into a stabler branch elif (len(results) > 1): # if it's because category differs, it's a problem - foundCat = "" + foundCat = None cats = set() - for result in results: - idpackage = result[1] + for idpackage in results: cat = self.retrieveCategory(idpackage) cats.add(cat) - if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): # in case of virtual packages only (that they're not stored as provide) + if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): + # in case of virtual packages only (that they're not stored as provide) foundCat = cat # if we found something at least... if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual","null")): foundCat = list(cats)[0] - if (not foundCat): + + if not foundCat: # got the issue continue @@ -3986,10 +4026,18 @@ class etpDatabase: mypkgcat = foundCat # we need to search using the category - if (not multiMatch): - results = self.searchPackagesByNameAndCategory(name = mypkgname, category = mypkgcat, branch = idx, sensitive = caseSensitive) + if (not multiMatch) and pkgcat == "null": + # we searched by name, we need to search using category + results = self.searchPackagesByNameAndCategory( + name = mypkgname, + category = mypkgcat, + branch = idx, + sensitive = caseSensitive, + justid = True + ) + # validate again - if (not results): + if not results: continue # search into another branch # if we get here, we have found the needed IDs @@ -3998,19 +4046,21 @@ class etpDatabase: else: + idpackage = results[0] # if mypkgcat is virtual, we can force - if (mypkgcat == "virtual") and (not virtual): # in case of virtual packages only (that they're not stored as provide) - mypkgcat = self.entropyTools.dep_getkey(results[0][0]).split("/")[0] + if (mypkgcat == "virtual") and (not virtual): + # in case of virtual packages only (that they're not stored as provide) + mypkgcat = self.retrieveCategory(idpackage) # check if category matches if mypkgcat != "null": - foundCat = self.retrieveCategory(results[0][1]) + foundCat = self.retrieveCategory(idpackage) if mypkgcat == foundCat: - foundIDs.add(results[0]) + foundIDs.add(idpackage) else: continue else: - foundIDs.add(results[0]) + foundIDs.add(idpackage) break ### FILTERING @@ -4029,7 +4079,7 @@ class etpDatabase: if not foundIDs: # package not found - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) return -1,1 ### FILLING dbpkginfo @@ -4056,9 +4106,8 @@ class etpDatabase: pkgrevision = self.entropyTools.dep_get_portage_revision(pkgversion) pkgversion = self.entropyTools.remove_revision(pkgversion) - for data in foundIDs: + for idpackage in foundIDs: - idpackage = data[1] dbver = self.retrieveVersion(idpackage) if (direction == "~"): myrev = self.entropyTools.dep_get_portage_revision(dbver) @@ -4087,9 +4136,8 @@ class etpDatabase: # remove self.entropyTools.remove_revision(pkgversion) - for data in foundIDs: + for idpackage in foundIDs: - idpackage = data[1] revcmp = 0 tagcmp = 0 if matchRevision != None: @@ -4160,25 +4208,40 @@ class etpDatabase: else: # just the key - dbpkginfo = set([((x[1]),self.retrieveVersion(x[1])) for x in foundIDs]) + dbpkginfo = set([(x,self.retrieveVersion(x)) for x in foundIDs]) ### END FILLING dbpkginfo ### END FILLING dbpkginfo ### END FILLING dbpkginfo if not dbpkginfo: - self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) - return -1,1 + if extendedResults: + x = (-1,1,None,None,None) + self.atomMatchStoreCache(x, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x + else: + self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return -1,1 if multiMatch: - x = set([x[0] for x in dbpkginfo]) - self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) - return x,0 + if extendedResults: + x = set([(x[0],0,x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) for x in dbpkginfo]),0 + self.atomMatchStoreCache(x, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x + else: + x = set([x[0] for x in dbpkginfo]) + self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x,0 if len(dbpkginfo) == 1: x = dbpkginfo.pop() - self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) - return x[0],0 + if extendedResults: + x = (x[0],0,x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])),0 + self.atomMatchStoreCache(x, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x + else: + self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x[0],0 dbpkginfo = list(dbpkginfo) pkgdata = {} @@ -4189,5 +4252,10 @@ class etpDatabase: pkgdata[info_tuple] = x[0] newer = self.entropyTools.getEntropyNewerVersion(list(versions))[0] x = pkgdata[newer] - self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) - return x,0 + if extendedResults: + x = (x,0,newer[0],newer[1],newer[2]),0 + self.atomMatchStoreCache(x, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x + else: + self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision, extendedResults) + return x,0 diff --git a/libraries/entropy.py b/libraries/entropy.py index 7e11bcd57..58ac29a96 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -55,6 +55,10 @@ class EquoInterface(TextInterface): self.repoDbCache = {} self.securityCache = {} self.spmCache = {} + #self.stdout = fakeoutfile(sys.stdout.fileno()) + #self.stderr = fakeoutfile(sys.stderr.fileno()) + #self.stdout.connect_progress(self.updateProgress) + #self.stderr.connect_progress(self.updateProgress) self.clientLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[client]") import dumpTools @@ -991,7 +995,8 @@ class EquoInterface(TextInterface): matchRevision = None, matchRepo = None, server_repos = [], - serverInstance = None + serverInstance = None, + extendedResults = False ): if not server_repos: @@ -1017,6 +1022,7 @@ class EquoInterface(TextInterface): str(hash(multiRepo)) + \ str(hash(caseSensitive)) + \ str(hash(matchRevision)) + \ + str(hash(extendedResults))+ \ str(u_hash) c_hash = str(hash(c_hash)) cached = self.dumpTools.loadobj(etpCache['atomMatch']+c_hash) @@ -1036,7 +1042,7 @@ class EquoInterface(TextInterface): if server_repos: dbconn = serverInstance.openServerDatabase(read_only = True, no_upload = True, repo = repoid) else: - dbconn = self.openRepositoryDatabase(repo) + dbconn = self.openRepositoryDatabase(repoid) return dbconn repoResults = {} @@ -1050,19 +1056,29 @@ class EquoInterface(TextInterface): # search dbconn = open_db(repo) - query = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, matchRevision = matchRevision) + query = dbconn.atomMatch( atom, + caseSensitive = caseSensitive, + matchSlot = matchSlot, + matchBranches = matchBranches, + packagesFilter = packagesFilter, + matchRevision = matchRevision, + extendedResults = extendedResults + ) if query[1] == 0: # package found, add to our dictionary - repoResults[repo] = query[0] + if extendedResults: + repoResults[repo] = (query[0][0],query[0][2],query[0][3],query[0][4]) + else: + repoResults[repo] = query[0] - dbpkginfo = (-1,1) - - packageInformation = {} - # nothing found - if not repoResults: + if extendedResults: + dbpkginfo = ((-1,None,None,None),1) + else: dbpkginfo = (-1,1) - elif multiRepo: + packageInformation = {} + + if multiRepo and repoResults: data = set() for repoid in repoResults: data.add((repoResults[repoid],repoid)) @@ -1089,15 +1105,22 @@ class EquoInterface(TextInterface): version_duplicates = set() versions = [] for repo in repoResults: - dbconn = open_db(repo) packageInformation[repo] = {} - version = dbconn.retrieveVersion(repoResults[repo]) + if extendedResults: + version = repoResults[repo][1] + packageInformation[repo]['versiontag'] = repoResults[repo][2] + packageInformation[repo]['revision'] = repoResults[repo][3] + packageInformation[repo]['version'] = version + else: + dbconn = open_db(repo) + packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(repoResults[repo]) + packageInformation[repo]['revision'] = dbconn.retrieveRevision(repoResults[repo]) + version = dbconn.retrieveVersion(repoResults[repo]) packageInformation[repo]['version'] = version if version in versions: version_duplicates.add(version) versions.append(version) - packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(repoResults[repo]) - packageInformation[repo]['revision'] = dbconn.retrieveRevision(repoResults[repo]) + newerVersion = self.entropyTools.getNewerVersion(versions)[0] # if no duplicates are found, we're done @@ -1191,10 +1214,15 @@ class EquoInterface(TextInterface): matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, - multiMatch = True + multiMatch = True, + extendedResults = extendedResults ) - for repoidpackage in matches[0]: - data.add((repoidpackage,match[1])) + if extendedResults: + for item in matches[0]: + data.add(((item[0],item[2],item[3],item[4]),match[1])) + else: + for repoidpackage in matches[0]: + data.add((repoidpackage,match[1])) dbpkginfo = (data,0) else: dbconn = open_db(dbpkginfo[1]) @@ -1204,9 +1232,13 @@ class EquoInterface(TextInterface): matchSlot = matchSlot, matchBranches = matchBranches, packagesFilter = packagesFilter, - multiMatch = True + multiMatch = True, + extendedResults = extendedResults ) - dbpkginfo = (set([(x,dbpkginfo[1]) for x in matches[0]]),0) + if extendedResults: + dbpkginfo = (set([((x[0],x[2],x[3],x[4]),dbpkginfo[1]) for x in matches[0]]),0) + else: + dbpkginfo = (set([(x,dbpkginfo[1]) for x in matches[0]]),0) if self.xcache: try: @@ -1966,72 +1998,56 @@ class EquoInterface(TextInterface): maxlen = len(idpackages) count = 0 for idpackage in idpackages: + count += 1 if (count%10 == 0) or (count == maxlen) or (count == 1): self.updateProgress("Calculating world packages", importance = 0, type = "info", back = True, header = ":: ", count = (count,maxlen), percent = True, footer = " ::") - tainted = False - myscopedata = self.clientDbconn.getScopeData(idpackage) - # check against broken entry - if myscopedata == None: + + mystrictdata = self.clientDbconn.getStrictData(idpackage) + # check against broken entries + if mystrictdata == None: continue - #atom = myscopedata[0] - #category = myscopedata[1] - #name = myscopedata[2] - #slot = myscopedata[4] - #revision = myscopedata[6] - #atomkey = myscopedata[1]+"/"+myscopedata[2] - # search in the packages - match = self.atomMatch(myscopedata[0]) - if match[0] == -1: # atom has been changed, or removed? - tainted = True - else: # not changed, is the revision changed? - adbconn = self.openRepositoryDatabase(match[1]) - arevision = adbconn.retrieveRevision(match[0]) - # if revision is 9999, then any revision is fine - if myscopedata[6] == 9999: arevision = 9999 + match = self.atomMatch( mystrictdata[0], + matchSlot = mystrictdata[1], + matchBranches = (branch,), + extendedResults = True + ) + # now compare + # version: mystrictdata[2] + # tag: mystrictdata[3] + # revision: mystrictdata[4] + if (match[0][0] != -1): + # version: match[0][1] + # tag: match[0][2] + # revision: match[0][3] if empty_deps: - tainted = True - elif myscopedata[6] != arevision: - tainted = True - ''' - elif (myscopedata[6] == arevision) and (arevision == 9999): - # check if "needed" are the same, otherwise, pull - # this will avoid having old packages installed just because user ran equo database generate (migrating from gentoo) - # also this helps in environments with multiple repositories, to avoid messing with libraries - aneeded = adbconn.retrieveNeeded(match[0]) - needed = self.clientDbconn.retrieveNeeded(idpackage) - if needed != aneeded: - tainted = True - # - else: - # check use flags too - # it helps for the same reason above and when doing upgrades to different branches - auseflags = adbconn.retrieveUseflags(match[0]) - useflags = self.clientDbconn.retrieveUseflags(idpackage) - if auseflags != useflags: - tainted = True - # - ''' - if (tainted): - # Alice! use the key! ... and the slot - matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchSlot = myscopedata[4], matchBranches = (branch,)) - if matchresults[0] != -1: - #mdbconn = self.openRepositoryDatabase(matchresults[1]) - update.append(matchresults) + update.append((match[0][0],match[1])) + continue + elif (mystrictdata[2] != match[0][1]): + # different versions + update.append((match[0][0],match[1])) + continue + elif (mystrictdata[3] != match[0][2]): + # different tags + update.append((match[0][0],match[1])) + continue + elif (mystrictdata[4] != 9999) and (mystrictdata[4] != match[0][3]): + # different revision + update.append((match[0][0],match[1])) + continue else: - # don't take action if it's just masked - maskedresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchSlot = myscopedata[4], matchBranches = (branch,), packagesFilter = False) - if maskedresults[0] == -1: - remove.append(idpackage) - # look for packages that would match key with any slot (for eg, gcc updates) - matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchBranches = (branch,)) - if matchresults[0] != -1: - update.append(matchresults) + # no difference + fine.append(mystrictdata[5]) + continue - else: - fine.append(myscopedata[0]) - - del idpackages + # don't take action if it's just masked + maskedresults = self.atomMatch(mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), packagesFilter = False) + if maskedresults[0] == -1: + remove.append(idpackage) + # look for packages that would match key with any slot (for eg, gcc updates) + matchresults = self.atomMatch(mystrictdata[0], matchBranches = (branch,)) + if matchresults[0] != -1: + update.append(matchresults) if self.xcache: c_hash = str(hash(db_digest)) + \ @@ -3876,7 +3892,8 @@ class PackageInterface: slot = self.infoDict['slot'] matches = self.Entropy.atomMatch(key, matchSlot = slot, multiRepo = True, multiMatch = True) if matches[1] == 0: - disk_cache['available'].update(matches[0]) + if matches[0] not in disk_cache['available']: + disk_cache['available'].append(matches[0]) # install, doing here because matches[0] could contain self.matched_atoms if self.matched_atom in disk_cache['available']: @@ -7424,16 +7441,35 @@ class TriggerInterface: header = red(" ##") ) try: + if not os.path.isfile(self.pkgdata['unpackdir']+"/portage/"+portage_atom+"/temp/environment"): # if environment is not yet created, we need to run pkg_setup() sys.stdout = stdfile - rc = self.Spm.spm_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) + rc = self.Spm.spm_doebuild( myebuild, + mydo = "setup", + tree = "bintree", + cpv = portage_atom, + portage_tmpdir = self.pkgdata['unpackdir'], + licenses = self.pkgdata['accept_license'] + ) if rc == 1: - self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup()) trigger for "+str(portage_atom)+". Something bad happened.") + self.Entropy.clientLog.log( ETP_LOGPRI_INFO, + ETP_LOGLEVEL_NORMAL, + "[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup())" + " trigger for "+str(portage_atom)+". Something bad happened." + ) sys.stdout = oldstdout - rc = self.Spm.spm_doebuild(myebuild, mydo = "postinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) + + rc = self.Spm.spm_doebuild( myebuild, + mydo = "postinst", + tree = "bintree", + cpv = portage_atom, + portage_tmpdir = self.pkgdata['unpackdir'], + licenses = self.pkgdata['accept_license'] + ) if rc == 1: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_postinst()) trigger for "+str(portage_atom)+". Something bad happened.") + except Exception, e: sys.stdout = oldstdout self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot run Gentoo postinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e)) @@ -8297,6 +8333,104 @@ class PackageMaskingParser: self.__removeRepoCache(repoid = repoid) self.__saveFileMtime(maskfile,mtimefile) +class fakeoutfile: + """ + A general purpose fake output file object. + """ + + def __init__(self, fn): + self.fn = fn + self.updateProgress = None + self.text_written = [] + + def close(self): + pass + + def flush(self): + self.close() + + def fileno(self): + return self.fn + + def isatty(self): + return False + + def read(self, a): + return '' + + def readline(self): + return '' + + def readlines(self): + return [] + + def connect_progress(self, f): + self.updateProgress = f + + def write(self, s): + os.write(self.fn,s) + self.text_written.append(s) + # cut at 1024 entries + if len(self.text_written) > 1024: + self.text_written = self.text_written[-1024:] + if self.updateProgress: + self.updateProgress(s) + + def write_line(self, s): + self.write(s) + + def writelines(self, l): + for s in l: + self.write(s) + + def seek(self, a): + raise IOError, (29, 'Illegal seek') + + def tell(self): + raise IOError, (29, 'Illegal seek') + + def truncate(self): + self.tell() + +class fakeinfile: + """ + A general purpose fake input file object. + """ + def __init__(self, fn): + self.fn = fn + + def close(self): + pass + + flush = close + + def fileno(self): + return self.fn + + def isatty(self): + return False + + def read(self, a): + return self.readline() + + def readline(self): + return os.read(self.fn,2048) + + def readlines(self): return [] + + def write(self, s): + return None + + def writelines(self, l): + return None + + def seek(self, a): + raise IOError, (29, 'Illegal seek') + + def tell(self): + raise IOError, (29, 'Illegal seek') + + truncate = tell class Callable: def __init__(self, anycallable): diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 010603f58..869bf25d0 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -899,6 +899,9 @@ def dep_gettag(dep): def removePackageOperators(atom): + if not atom: + return atom + if atom[0] in [">","<"]: atom = atom[1:] if atom[0] == "=": diff --git a/spritz/src/spritz.py b/spritz/src/spritz.py index 740d9300a..2742bf7b2 100644 --- a/spritz/src/spritz.py +++ b/spritz/src/spritz.py @@ -39,13 +39,13 @@ import thread import exceptions from etpgui.widgets import UI, Controller from etpgui import * +from spritz_setup import fakeoutfile, fakeinfile # spritz imports import filters from gui import SpritzGUI from dialogs import * -from spritz_setup import const, fakeoutfile, fakeinfile from i18n import _ import time diff --git a/spritz/src/spritz_setup.py b/spritz/src/spritz_setup.py index 3082fab75..0e03d8296 100644 --- a/spritz/src/spritz_setup.py +++ b/spritz/src/spritz_setup.py @@ -143,10 +143,19 @@ def cleanMarkupSting(msg): #msg = msg.replace('>',']') return msg +from htmlentitydefs import codepoint2name +def unicode2htmlentities(u): + htmlentities = list() + for c in u: + if ord(c) < 128: + htmlentities.append(c) + else: + htmlentities.append('&%s;' % codepoint2name[ord(c)]) + return ''.join(htmlentities) + class fakeoutfile: """ - A fake output file object. It sends output to a GTK TextView widget, - and if asked for a file number, returns one set on instance creation + A general purpose fake output file object. """ def __init__(self, fn): @@ -180,7 +189,6 @@ class fakeoutfile: # cut at 1024 entries if len(self.text_written) > 1024: self.text_written = self.text_written[-1024:] - #sys.stdout.write(s+"\n") def write_line(self, s): self.write(s) @@ -200,32 +208,40 @@ class fakeoutfile: class fakeinfile: """ - A fake input file object. It receives input from a GTK TextView widget, - and if asked for a file number, returns one set on instance creation + A general purpose fake input file object. """ - def __init__(self, fn): self.fn = fn - def close(self): pass - flush = close - def fileno(self): return self.fn - def isatty(self): return False - def read(self, a): return self.readline() - def readline(self): ## just a fake - return os.read(self.fn,2048) - def readlines(self): return [] - def write(self, s): return None - def writelines(self, l): return None - def seek(self, a): raise IOError, (29, 'Illegal seek') - def tell(self): raise IOError, (29, 'Illegal seek') - truncate = tell -from htmlentitydefs import codepoint2name -def unicode2htmlentities(u): - htmlentities = list() - for c in u: - if ord(c) < 128: - htmlentities.append(c) - else: - htmlentities.append('&%s;' % codepoint2name[ord(c)]) - return ''.join(htmlentities) \ No newline at end of file + def close(self): + pass + + flush = close + + def fileno(self): + return self.fn + + def isatty(self): + return False + + def read(self, a): + return self.readline() + + def readline(self): + return os.read(self.fn,2048) + + def readlines(self): return [] + + def write(self, s): + return None + + def writelines(self, l): + return None + + def seek(self, a): + raise IOError, (29, 'Illegal seek') + + def tell(self): + raise IOError, (29, 'Illegal seek') + + truncate = tell \ No newline at end of file