#!/usr/bin/python ''' # DESCRIPTION: # Entropy Database Interface Copyright (C) 2007-2008 Fabio Erculiani This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ''' from entropyConstants import * import entropyTools from outputTools import * import exceptionTools Text = TextInterface() try: # try with sqlite3 from python 2.5 - default one from sqlite3 import dbapi2 except ImportError: # fallback to embedded pysqlite try: from pysqlite2 import dbapi2 except ImportError, e: raise exceptionTools.SystemError("Entropy needs a working sqlite+pysqlite or Python compiled with sqlite support. Error: %s" % (str(e),)) import dumpTools ############ # Functions and Classes ##################################################################################### ''' @description: open the installed packages database @output: database class instance NOTE: if you are interested using it client side, please USE equoTools.Equo() class instead ''' def openClientDatabase(xcache = True, generate = False, indexing = True, OutputInterface = Text): # extra check for directory existance if not os.path.isdir(os.path.dirname(etpConst['etpdatabaseclientfilepath'])): os.makedirs(os.path.dirname(etpConst['etpdatabaseclientfilepath'])) if (not generate) and (not os.path.isfile(etpConst['etpdatabaseclientfilepath'])): raise exceptionTools.SystemDatabaseError("SystemDatabaseError: system database not found. Either does not exist or corrupted.") else: conn = etpDatabase(readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = etpConst['clientdbid'], xcache = xcache, indexing = indexing, OutputInterface = OutputInterface) # validate database if not generate: conn.validateDatabase() if (not etpConst['dbconfigprotect']): # config protect not prepared if (not generate): etpConst['dbconfigprotect'] = conn.listConfigProtectDirectories() etpConst['dbconfigprotectmask'] = conn.listConfigProtectDirectories(mask = True) etpConst['dbconfigprotect'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] etpConst['dbconfigprotectmask'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] etpConst['dbconfigprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpConst['dbconfigprotect']] etpConst['dbconfigprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpConst['dbconfigprotectmask']] return conn ''' @description: open the entropy server database and returns the pointer. This function must be used only by reagent or activator @output: database class instance ''' def openServerDatabase(readOnly = True, noUpload = True, OutputInterface = Text): if not os.path.isdir(os.path.dirname(etpConst['etpdatabasefilepath'])): try: os.remove(os.path.dirname(etpConst['etpdatabasefilepath'])) except OSError: pass os.makedirs(os.path.dirname(etpConst['etpdatabasefilepath'])) conn = etpDatabase(readOnly = readOnly, dbFile = etpConst['etpdatabasefilepath'], noUpload = noUpload, OutputInterface = OutputInterface) # verify if we need to update the database to sync with portage updates, we just ignore being readonly in the case if not etpConst['treeupdatescalled']: # sometimes, when filling a new server db, we need to avoid tree updates valid = True try: conn.validateDatabase() except exceptionTools.SystemDatabaseError: valid = False if valid: conn.serverUpdatePackagesData() else: OutputInterface.updateProgress(red("Entropy database is probably empty. If you don't agree with what I'm saying, then it's probably corrupted! I won't stop you here btw..."), importance = 1, type = "info", header = red(" * ")) return conn ''' @description: open a generic client database and returns the pointer. @output: database class instance ''' def openGenericDatabase(dbfile, dbname = None, xcache = False, indexing = True, readOnly = False, OutputInterface = Text): if dbname == None: dbname = "generic" conn = etpDatabase(readOnly = readOnly, dbFile = dbfile, clientDatabase = True, dbname = dbname, xcache = xcache, indexing = indexing, OutputInterface = OutputInterface) return conn def backupClientDatabase(): import shutil if os.path.isfile(etpConst['etpdatabaseclientfilepath']): rnd = entropyTools.getRandomNumber() source = etpConst['etpdatabaseclientfilepath'] dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd) shutil.copy2(source,dest) user = os.stat(source)[4] group = os.stat(source)[5] os.chown(dest,user,group) shutil.copystat(source,dest) return dest return "" class etpDatabase: def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], indexing = True, OutputInterface = Text): self.readOnly = readOnly self.noUpload = noUpload self.packagesRemoved = False self.packagesAdded = False self.clientDatabase = clientDatabase self.xcache = xcache self.dbname = dbname self.indexing = indexing if not entropyTools.is_user_in_entropy_group(): # forcing since we won't have write access to db self.indexing = False # live systems don't like wasting RAM if entropyTools.islive(): self.indexing = False self.dbFile = dbFile # setup output interface self.OutputInterface = OutputInterface self.updateProgress = self.OutputInterface.updateProgress self.askQuestion = self.OutputInterface.askQuestion # no caching for non root and server connections if (self.dbname == etpConst['serverdbid']) or (not entropyTools.is_user_in_entropy_group()): self.xcache = False self.live_cache = {} # create connection self.connection = dbapi2.connect(dbFile,timeout=300.0) self.cursor = self.connection.cursor() if not self.clientDatabase and not self.readOnly: # server side is calling # lock mirror remotely and ensure to have latest database revision self.doServerDatabaseSyncLock(self.noUpload) if os.access(self.dbFile,os.W_OK) and self.doesTableExist('baseinfo') and self.doesTableExist('extrainfo'): if entropyTools.islive(): # check where's the file if etpConst['systemroot']: self.databaseStructureUpdates() else: self.databaseStructureUpdates() def doServerDatabaseSyncLock(self, noUpload): from entropy import FtpInterface import activatorTools # check if the database is locked locally if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): self.updateProgress(red("Entropy database is already locked by you :-)"), importance = 1, type = "info", header = red(" * ")) else: # check if the database is locked REMOTELY self.updateProgress(red("Locking and Syncing Entropy database..."), importance = 1, type = "info", header = red(" * "), back = True) for uri in etpConst['activatoruploaduris']: ftp = FtpInterface(uri, self.OutputInterface) try: ftp.setCWD(etpConst['etpurirelativepath']) except: bdir = "" for mydir in etpConst['etpurirelativepath'].split("/"): bdir += "/"+mydir if (not ftp.isFileAvailable(bdir)): try: ftp.mkdir(bdir) except Exception, e: error = str(e) if error.find("550") == -1 and error.find("File exist") == -1: raise ftp.setCWD(etpConst['etpurirelativepath']) if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) and (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])): import time self.updateProgress(bold("WARNING")+red(": online database is already locked. Waiting up to 2 minutes..."), importance = 1, type = "info", header = red(" * "), back = True) unlocked = False count = 120 while count: time.sleep(1) count -= 1 if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])): self.updateProgress(bold("HOORAY")+red(": online database has been unlocked. Locking back and syncing..."), importance = 1, type = "info", header = red(" * ")) unlocked = True break if (unlocked): break self.updateProgress(darkgreen("Mirrors status table:"), importance = 1, type = "info", header = brown(" * ")) dbstatus = activatorTools.getMirrorsLock() for db in dbstatus: db[1] = green("Unlocked") if (db[1]): db[1] = red("Locked") db[2] = green("Unlocked") if (db[2]): db[2] = red("Locked") p_uri = entropyTools.extractFTPHostFromUri(db[0]) self.updateProgress( bold("%s: ")+red("[")+brown("DATABASE: %s")+red("] [")+brown("DOWNLOAD: %s")+red("]") % (p_uri,db[1],db[2],), importance = 1, type = "info", header = "\t" ) del p_uri ftp.closeConnection() raise exceptionTools.OnlineMirrorError("OnlineMirrorError: cannot lock mirror "+entropyTools.extractFTPHostFromUri(uri)) # if we arrive here, it is because all the mirrors are unlocked so... damn, LOCK! activatorTools.lockDatabases(True) # ok done... now sync the new db, if needed activatorTools.syncRemoteDatabases(noUpload) def closeDB(self): # if the class is opened readOnly, close and forget if (self.readOnly): self.cursor.close() self.connection.close() return # if it's equo that's calling the function, just save changes and quit if (self.clientDatabase): self.commitChanges() self.cursor.close() self.connection.close() return # Cleanups if at least one package has been removed # Please NOTE: the client database does not need it if (self.packagesRemoved): self.cleanupUseflags() self.cleanupSources() self.cleanupEclasses() self.cleanupNeeded() self.cleanupDependencies() if (etpDbStatus[etpConst['etpdatabasefilepath']]['tainted']) and (not etpDbStatus[etpConst['etpdatabasefilepath']]['bumped']): # bump revision, setting DatabaseBump causes the session to just bump once etpDbStatus[etpConst['etpdatabasefilepath']]['bumped'] = True self.revisionBump() if (not etpDbStatus[etpConst['etpdatabasefilepath']]['tainted']): # we can unlock it, no changes were made import activatorTools activatorTools.lockDatabases(False) else: self.updateProgress(darkgreen("Mirrors have not been unlocked. Run activator."), importance = 1, type = "info", header = brown(" * ")) # run vacuum cleaner self.vacuum() self.connection.commit() self.cursor.close() self.connection.close() def vacuum(self): self.cursor.execute("vacuum") def commitChanges(self): if (not self.readOnly): try: self.connection.commit() except: pass self.taintDatabase() def taintDatabase(self): if (self.clientDatabase): # if it's equo to open it, this should be avoided return # taint the database status f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'],"w") f.write(etpConst['currentarch']+" database tainted\n") f.flush() f.close() etpDbStatus[etpConst['etpdatabasefilepath']]['tainted'] = True def untaintDatabase(self): if (self.clientDatabase): # if it's equo to open it, this should be avoided return etpDbStatus[etpConst['etpdatabasefilepath']]['tainted'] = False # untaint the database status if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) def revisionBump(self): if (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'])): revision = 0 else: f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'],"r") revision = int(f.readline().strip()) revision += 1 f.close() f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'],"w") f.write(str(revision)+"\n") f.flush() f.close() def isDatabaseTainted(self): if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): return True return False # never use this unless you know what you're doing def initializeDatabase(self): self.checkReadOnly() for sql in etpSQLInitDestroyAll.split(";"): if sql: self.cursor.execute(sql+";") del sql for sql in etpSQLInit.split(";"): if sql: self.cursor.execute(sql+";") self.databaseStructureUpdates() self.commitChanges() def checkReadOnly(self): if (self.readOnly): raise exceptionTools.OperationNotPermitted("OperationNotPermitted: can't do that on a readonly database.") # check for /usr/portage/profiles/updates changes def serverUpdatePackagesData(self): etpConst['treeupdatescalled'] = True repository = etpConst['officialrepositoryid'] repo_updates_file = os.path.join(etpConst['etpdatabasedir'],etpConst['etpdatabaseupdatefile']) doRescan = False if repositoryUpdatesDigestCache_db.has_key(repository): stored_digest = repositoryUpdatesDigestCache_db.get(repository) else: # check database digest stored_digest = self.retrieveRepositoryUpdatesDigest(repository) repositoryUpdatesDigestCache_db[repository] = stored_digest if stored_digest == -1: doRescan = True # check portage files for changes if doRescan is still false portage_dirs_digest = "0" if not doRescan: if repositoryUpdatesDigestCache_disk.has_key(repository): portage_dirs_digest = repositoryUpdatesDigestCache_disk.get(repository) else: from entropy import SpmInterface SpmIntf = SpmInterface(self.OutputInterface) Spm = SpmIntf.intf # grab portdir updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates" if os.path.isdir(updates_dir): # get checksum mdigest = entropyTools.md5sum_directory(updates_dir, get_obj = True) # also checksum etpConst['etpdatabaseupdatefile'] if os.path.isfile(repo_updates_file): f = open(repo_updates_file) block = f.read(1024) while block: mdigest.update(block) block = f.read(1024) f.close() portage_dirs_digest = mdigest.hexdigest() repositoryUpdatesDigestCache_disk[repository] = portage_dirs_digest del updates_dir if doRescan or (str(stored_digest) != str(portage_dirs_digest)): # force parameters self.readOnly = False self.noUpload = True # reset database tables self.clearTreeupdatesEntries(repository) from entropy import SpmInterface SpmIntf = SpmInterface(self.OutputInterface) Spm = SpmIntf.intf updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates" update_files = entropyTools.sortUpdateFiles(os.listdir(updates_dir)) update_files = [os.path.join(updates_dir,x) for x in update_files] # now load actions from files update_actions = [] for update_file in update_files: f = open(update_file,"r") lines = [x.strip() for x in f.readlines() if x.strip()] for line in lines: update_actions.append(line) f.close() del lines # add entropy packages.db.repo_updates content if os.path.isfile(repo_updates_file): f = open(repo_updates_file,"r") lines = [x.strip() for x in f.readlines() if x.strip()] for line in lines: update_actions.append(line) f.close() # now filter the required actions update_actions = self.filterTreeUpdatesActions(update_actions) if update_actions: self.updateProgress( bold("ATTENTION: ")+red("forcing package updates. Syncing with %s") % (blue(updates_dir),), importance = 1, type = "info", header = brown(" * ") ) # lock database self.doServerDatabaseSyncLock(self.noUpload) # now run queue self.runTreeUpdatesActions(update_actions) # store new actions self.addRepositoryUpdatesActions(repository,update_actions) # store new digest into database self.setRepositoryUpdatesDigest(repository, portage_dirs_digest) # client side, no portage dependency # lxnay: it is indeed very similar to serverUpdatePackagesData() but I prefer keeping both separate # also, we reuse the same caching dictionaries of the server function # repositoryUpdatesDigestCache_db -> repository cache # repositoryUpdatesDigestCache_disk -> client database cache # check for repository packages updates # this will read database treeupdates* tables and do # changes required if running as root. def clientUpdatePackagesData(self): etpConst['treeupdatescalled'] = True repository = self.dbname[len(etpConst['dbnamerepoprefix']):] doRescan = False if repositoryUpdatesDigestCache_db.has_key(repository): stored_digest = repositoryUpdatesDigestCache_db.get(repository) else: # check database digest stored_digest = self.retrieveRepositoryUpdatesDigest(repository) repositoryUpdatesDigestCache_db[repository] = stored_digest if stored_digest == -1: doRescan = True try: clientDbconn = openClientDatabase(xcache = False, indexing = False, OutputInterface = self) except exceptionTools.SystemDatabaseError: return # don't run anything for goodness' sake # check stored value in client database client_digest = "0" if not doRescan: if repositoryUpdatesDigestCache_disk.has_key(etpConst['systemroot']): client_digest = repositoryUpdatesDigestCache_disk.get(etpConst['systemroot']) else: client_digest = clientDbconn.retrieveRepositoryUpdatesDigest(repository) if doRescan or (str(stored_digest) != str(client_digest)): # reset database tables clientDbconn.clearTreeupdatesEntries(repository) # load updates update_actions = self.retrieveTreeUpdatesActions(repository) # now filter the required actions update_actions = clientDbconn.filterTreeUpdatesActions(update_actions) if update_actions: self.updateProgress( bold("ATTENTION: ")+red("forcing packages metadata update. Updating system database using repository id: %s") % (blue(repository),), importance = 1, type = "info", header = darkred(" * ") ) # run stuff clientDbconn.runTreeUpdatesActions(update_actions) # store new digest into database clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest) # clear client cache clientDbconn.clearCache() clientDbconn.closeDB() del clientDbconn # this functions will filter either data from /usr/portage/profiles/updates/* # or repository database returning only the needed actions def filterTreeUpdatesActions(self, actions): new_actions = [] for action in actions: doaction = action.split() if doaction[0] == "slotmove": # slot move atom = doaction[1] from_slot = doaction[2] to_slot = doaction[3] category = atom.split("/")[0] matches = self.atomMatch(atom, multiMatch = True) if matches[1] == 0: # found atom, check slot and category for idpackage in matches[0]: myslot = str(self.retrieveSlot(idpackage)) mycategory = self.retrieveCategory(idpackage) if mycategory == category: if (myslot == from_slot) and (myslot != to_slot): new_actions.append(action) elif doaction[0] == "move": atom = doaction[1] category = atom.split("/")[0] matches = self.atomMatch(atom, multiMatch = True) if matches[1] == 0: for idpackage in matches[0]: mycategory = self.retrieveCategory(idpackage) if mycategory == category: new_actions.append(action) return new_actions # this is the place to add extra actions support def runTreeUpdatesActions(self, actions): # just run fixpackages if gentoo-compat is enabled if etpConst['gentoo-compat']: self.updateProgress( bold("GENTOO: ")+red("Running fixpackages, could take a while."), importance = 1, type = "warning", header = darkred(" * ") ) if self.clientDatabase: os.system("fixpackages &> /dev/null") else: os.system("fixpackages") for action in actions: command = action.split() self.updateProgress( bold("ENTROPY: ")+red("action: %s") % (blue(action),), importance = 1, type = "warning", header = darkred(" * ") ) if command[0] == "move": self.runTreeUpdatesMoveAction(command[1:]) elif command[0] == "slotmove": self.runTreeUpdatesSlotmoveAction(command[1:]) # discard cache self.clearCache() # -- move action: # 1) move package key to the new name: category + name + atom # 2) update all the dependencies in dependenciesreference to the new key # 3) run fixpackages which will update /var/db/pkg files # 4) automatically run quickpkg() to build the new binary and tainted binaries owning tainted iddependency and taint database (LOL) def runTreeUpdatesMoveAction(self, move_command): key_from = move_command[0] key_to = move_command[1] cat_to = key_to.split("/")[0] name_to = key_to.split("/")[1] matches = self.atomMatch(key_from, multiMatch = True) for idpackage in matches[0]: slot = self.retrieveSlot(idpackage) old_atom = self.retrieveAtom(idpackage) new_atom = old_atom.replace(key_from,key_to) ### UPDATE DATABASE # update category self.setCategory(idpackage, cat_to) # update name self.setName(idpackage, name_to) # update atom self.setAtom(idpackage, new_atom) # look for packages we need to quickpkg again # note: quickpkg_queue is simply ignored if self.clientDatabase quickpkg_queue = [key_to+":"+str(slot)] iddeps = self.searchDependency(key_from, like = True, multi = True) for iddep in iddeps: # update string mydep = self.retrieveDependencyFromIddependency(iddep) mydep_key = entropyTools.dep_getkey(mydep) if mydep_key != key_from: # avoid changing wrong atoms -> dev-python/qscintilla-python would continue # become x11-libs/qscintilla if we don't do this check mydep = mydep.replace(key_from,key_to) # now update # dependstable on server is always re-generated self.setDependency(iddep, mydep) if self.clientDatabase: continue # ignore quickpkg stuff # we have to repackage also package owning this iddep iddep_owners = self.searchIdpackageFromIddependency(iddep) for idpackage_owner in iddep_owners: quickpkg_queue.append(self.retrieveAtom(idpackage_owner)) if not self.clientDatabase: # check for injection and warn the developer injected = self.isInjected(idpackage) if injected: self.updateProgress( bold("INJECT: ")+red("Package %s has been injected. You need to quickpkg it manually to update embedded database !!! Repository database will be updated anyway.") % (blue(new_atom),), importance = 1, type = "warning", header = darkred(" * ") ) # quickpkg package and packages owning it as a dependency self.runTreeUpdatesQuickpkgAction(quickpkg_queue) self.commitChanges() # -- slotmove action: # 1) move package slot # 2) update all the dependencies in dependenciesreference owning same matched atom + slot # 3) run fixpackages which will update /var/db/pkg files # 4) automatically run quickpkg() to build the new binary and tainted binaries owning tainted iddependency and taint database (LOL) def runTreeUpdatesSlotmoveAction(self, slotmove_command): atom = slotmove_command[0] atomkey = entropyTools.dep_getkey(atom) slot_from = slotmove_command[1] slot_to = slotmove_command[2] matches = self.atomMatch(atom, multiMatch = True) for idpackage in matches[0]: ### UPDATE DATABASE # update slot self.setSlot(idpackage, slot_to) # look for packages we need to quickpkg again # note: quickpkg_queue is simply ignored if self.clientDatabase quickpkg_queue = [atom+":"+str(slot_to)] iddeps = self.searchDependency(atomkey, like = True, multi = True) for iddep in iddeps: # update string mydep = self.retrieveDependencyFromIddependency(iddep) if mydep.find(":"+str(slot_from)) != -1: # probably slotted dep mydep = mydep.replace(":"+str(slot_from),":"+str(slot_to)) else: continue # it's fine # now update # dependstable on server is always re-generated self.setDependency(iddep, mydep) if self.clientDatabase: continue # ignore quickpkg stuff # we have to repackage also package owning this iddep iddep_owners = self.searchIdpackageFromIddependency(iddep) for idpackage_owner in iddep_owners: quickpkg_queue.append(self.retrieveAtom(idpackage_owner)) if not self.clientDatabase: # check for injection and warn the developer injected = self.isInjected(idpackage) if injected: self.updateProgress( bold("INJECT: ")+red("Package %s has been injected. You need to quickpkg it manually to update embedded database !!! Repository database will be updated anyway.") % (blue(atom),), importance = 1, type = "warning", header = darkred(" * ") ) # quickpkg package and packages owning it as a dependency self.runTreeUpdatesQuickpkgAction(quickpkg_queue) self.commitChanges() def runTreeUpdatesQuickpkgAction(self, atoms): import reagentTools reagent_cmds = ["--repackage"] reagent_cmds += atoms # ask branch question rc = self.askQuestion(" Would you like to continue with the default branch \"%s\" ?" % (etpConst['branch'],)) if rc == "No": # ask which mybranch = etpConst['branch'] while 1: mybranch = readtext("Type your branch: ") if mybranch not in self.listAllBranches(): self.updateProgress( bold("ATTENTION: ")+red("Specified branch %s does not exist.") % (blue(mybranch),), importance = 1, type = "warning", header = darkred(" * ") ) continue # ask to confirm rc = self.askQuestion(" Confirm %s ?" % (mybranch,)) if rc == "Yes": break reagent_cmds.append("--branch=%s" % (mybranch,)) rc = reagentTools.update(reagent_cmds) if rc != 0: self.updateProgress( bold("ATTENTION: ")+red("reagent update did not run properly. Please update packages manually"), importance = 1, type = "warning", header = darkred(" * ") ) # this function manages the submitted package # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage def handlePackage(self, etpData, forcedRevision = -1): self.checkReadOnly() # build atom string versiontag = '' if etpData['versiontag']: versiontag = '#'+etpData['versiontag'] foundid = self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag) if (foundid < 0): # same atom doesn't exist in any branch idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, revision = forcedRevision) else: idpk, revision, etpDataUpdated, accepted = self.updatePackage(etpData, forcedRevision) # only when the same atom exists return idpk, revision, etpDataUpdated, accepted def addPackage(self, etpData, revision = -1): self.checkReadOnly() self.live_cache.clear() if revision == -1: try: revision = etpData['revision'] except: etpData['revision'] = 0 # revision not specified revision = 0 # we need to find other packages with the same key and slot, and remove them if (self.clientDatabase): # client database can't care about branch searchsimilar = self.searchPackagesByNameAndCategory(name = etpData['name'], category = etpData['category'], sensitive = True) else: # server supports multiple branches inside a db searchsimilar = self.searchPackagesByNameAndCategory(name = etpData['name'], category = etpData['category'], sensitive = True, branch = etpData['branch']) removelist = set() if not etpData['injected']: # read: if package has been injected, we'll skip the removal of packages in the same slot, usually used server side btw for oldpkg in searchsimilar: # get the package slot idpackage = oldpkg[1] slot = self.retrieveSlot(idpackage) isinjected = self.isInjected(idpackage) if isinjected: continue # we merely ignore packages with negative counters, since they're the injected ones if (etpData['slot'] == slot): # remove! removelist.add(idpackage) for pkg in removelist: self.removePackage(pkg) # create new category if it doesn't exist catid = self.isCategoryAvailable(etpData['category']) if (catid == -1): # create category catid = self.addCategory(etpData['category']) # create new license if it doesn't exist licid = self.isLicenseAvailable(etpData['license']) if (licid == -1): # create category licid = self.addLicense(etpData['license']) # insert license information mylicenses = etpData['licensedata'].keys() for mylicense in mylicenses: found = self.isLicensedataKeyAvailable(mylicense) if not found: text = etpData['licensedata'][mylicense] self.cursor.execute( 'INSERT into licensedata VALUES ' '(?,?,?)' , ( mylicense, buffer(text), 0, )) # look for configured versiontag versiontag = "" if (etpData['versiontag']): versiontag = "#"+etpData['versiontag'] trigger = 0 if etpData['trigger']: trigger = 1 # baseinfo pkgatom = etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag # create new idflag if it doesn't exist idflags = self.areCompileFlagsAvailable(etpData['chost'],etpData['cflags'],etpData['cxxflags']) if (idflags == -1): # create category idflags = self.addCompileFlags(etpData['chost'],etpData['cflags'],etpData['cxxflags']) self.cursor.execute( 'INSERT into baseinfo VALUES ' '(NULL,?,?,?,?,?,?,?,?,?,?,?)' , ( pkgatom, catid, etpData['name'], etpData['version'], etpData['versiontag'], revision, etpData['branch'], etpData['slot'], licid, etpData['etpapi'], trigger, ) ) idpackage = self.cursor.lastrowid # extrainfo self.cursor.execute( 'INSERT into extrainfo VALUES ' '(?,?,?,?,?,?,?,?)' , ( idpackage, etpData['description'], etpData['homepage'], etpData['download'], etpData['size'], idflags, etpData['digest'], etpData['datecreation'], ) ) ### other information iserted below are not as critical as these above # content, a list self.insertContent(idpackage,etpData['content']) etpData['counter'] = int(etpData['counter']) # cast to integer if etpData['counter'] != -1 and not (etpData['injected']): if etpData['counter'] <= -2: # special cases etpData['counter'] = self.getNewNegativeCounter() try: self.cursor.execute( 'INSERT into counters VALUES ' '(?,?,?)' , ( etpData['counter'], idpackage, etpData['branch'], ) ) except dbapi2.IntegrityError: # we have a PRIMARY KEY we need to remove self.migrateCountersTable() self.cursor.execute( 'INSERT into counters VALUES ' '(?,?,?)' , ( etpData['counter'], idpackage, etpData['branch'], ) ) except: if self.dbname == etpConst['clientdbid']: # force only for client database if self.doesTableExist("counters"): raise self.cursor.execute( 'INSERT into counters VALUES ' '(?,?,?)' , ( etpData['counter'], idpackage, etpData['branch'], ) ) elif self.dbname == etpConst['serverdbid']: raise # on disk size self.cursor.execute( 'INSERT into sizes VALUES ' '(?,?)' , ( idpackage, etpData['disksize'], ) ) # trigger blob self.cursor.execute( 'INSERT into triggers VALUES ' '(?,?)' , ( idpackage, buffer(etpData['trigger']), )) # eclasses table for var in etpData['eclasses']: idclass = self.isEclassAvailable(var) if (idclass == -1): # create eclass idclass = self.addEclass(var) self.cursor.execute( 'INSERT into eclasses VALUES ' '(?,?)' , ( idpackage, idclass, ) ) # needed table for var in etpData['needed']: idneeded = self.isNeededAvailable(var) if (idneeded == -1): # create eclass idneeded = self.addNeeded(var) self.cursor.execute( 'INSERT into needed VALUES ' '(?,?)' , ( idpackage, idneeded, ) ) # dependencies, a list self.insertDependencies(idpackage, etpData['dependencies']) # provide for atom in etpData['provide']: self.cursor.execute( 'INSERT into provide VALUES ' '(?,?)' , ( idpackage, atom, ) ) # injected? if etpData['injected']: self.setInjected(idpackage) # compile messages for message in etpData['messages']: self.cursor.execute( 'INSERT into messages VALUES ' '(?,?)' , ( idpackage, message, ) ) # is it a system package? if etpData['systempackage']: self.cursor.execute( 'INSERT into systempackages VALUES ' '(?)' , ( idpackage, ) ) # create new protect if it doesn't exist idprotect = self.isProtectAvailable(etpData['config_protect']) if (idprotect == -1): # create category idprotect = self.addProtect(etpData['config_protect']) # fill configprotect self.cursor.execute( 'INSERT into configprotect VALUES ' '(?,?)' , ( idpackage, idprotect, ) ) idprotect = self.isProtectAvailable(etpData['config_protect_mask']) if (idprotect == -1): # create category idprotect = self.addProtect(etpData['config_protect_mask']) # fill configprotect self.cursor.execute( 'INSERT into configprotectmask VALUES ' '(?,?)' , ( idpackage, idprotect, ) ) # conflicts, a list for conflict in etpData['conflicts']: self.cursor.execute( 'INSERT into conflicts VALUES ' '(?,?)' , ( idpackage, conflict, ) ) # mirrorlinks, always update the table for mirrordata in etpData['mirrorlinks']: mirrorname = mirrordata[0] mirrorlist = mirrordata[1] # remove old self.removeMirrorEntries(mirrorname) # add new self.addMirrors(mirrorname,mirrorlist) # sources, a list for source in etpData['sources']: if (not source) or (source == "") or (not entropyTools.is_valid_string(source)): continue idsource = self.isSourceAvailable(source) if (idsource == -1): # create category idsource = self.addSource(source) self.cursor.execute( 'INSERT into sources VALUES ' '(?,?)' , ( idpackage, idsource, ) ) # useflags, a list for flag in etpData['useflags']: iduseflag = self.isUseflagAvailable(flag) if (iduseflag == -1): # create category iduseflag = self.addUseflag(flag) self.cursor.execute( 'INSERT into useflags VALUES ' '(?,?)' , ( idpackage, iduseflag, ) ) # create new keyword if it doesn't exist for key in etpData['keywords']: idkeyword = self.isKeywordAvailable(key) if (idkeyword == -1): # create category idkeyword = self.addKeyword(key) self.cursor.execute( 'INSERT into keywords VALUES ' '(?,?)' , ( idpackage, idkeyword, ) ) self.clearCache() self.packagesAdded = True self.commitChanges() ### RSS Atom support ### dictionary will be elaborated by activator if etpConst['rss-feed'] and not self.clientDatabase: rssAtom = pkgatom+"~"+str(revision) # store addPackage action rssObj = dumpTools.loadobj(etpConst['rss-dump-name']) if rssObj: global etpRSSMessages etpRSSMessages = rssObj.copy() if rssAtom in etpRSSMessages['removed']: del etpRSSMessages['removed'][rssAtom] etpRSSMessages['added'][rssAtom] = {} etpRSSMessages['added'][rssAtom]['description'] = etpData['description'] etpRSSMessages['added'][rssAtom]['homepage'] = etpData['homepage'] etpRSSMessages['light'][rssAtom] = {} etpRSSMessages['light'][rssAtom]['description'] = etpData['description'] # save dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages) return idpackage, revision, etpData, True # Update already available atom in db # returns True,revision if the package has been updated # returns False,revision if not def updatePackage(self, etpData, forcedRevision = -1): self.checkReadOnly() # build atom string versiontag = '' if etpData['versiontag']: versiontag = '#'+etpData['versiontag'] pkgatom = etpData['category'] + "/" + etpData['name'] + "-" + etpData['version']+versiontag # for client database - the atom if present, must be overwritten with the new one regardless its branch if (self.clientDatabase): atomid = self.isPackageAvailable(pkgatom) if atomid > -1: self.removePackage(atomid) x,y,z,accepted = self.addPackage(etpData, revision = forcedRevision) self.commitChanges() return x,y,z,accepted else: # update package in etpData['branch'] # get its package revision idpackage = self.getIDPackage(pkgatom,etpData['branch']) if (forcedRevision == -1): if (idpackage != -1): curRevision = self.retrieveRevision(idpackage) else: curRevision = 0 else: curRevision = forcedRevision if (idpackage != -1): # remove old package in branch self.removePackage(idpackage) if (forcedRevision == -1): curRevision += 1 # add the new one x,y,z,accepted = self.addPackage(etpData, revision = curRevision) self.commitChanges() return x,y,z,accepted def removePackage(self,idpackage): self.checkReadOnly() self.live_cache.clear() ### RSS Atom support ### dictionary will be elaborated by activator if etpConst['rss-feed'] and not self.clientDatabase: # store addPackage action rssObj = dumpTools.loadobj(etpConst['rss-dump-name']) if rssObj: global etpRSSMessages etpRSSMessages = rssObj.copy() rssAtom = self.retrieveAtom(idpackage) rssRevision = self.retrieveRevision(idpackage) rssAtom += "~"+str(rssRevision) if rssAtom in etpRSSMessages['added']: del etpRSSMessages['added'][rssAtom] etpRSSMessages['removed'][rssAtom] = {} try: etpRSSMessages['removed'][rssAtom]['description'] = self.retrieveDescription(idpackage) except: etpRSSMessages['removed'][rssAtom]['description'] = "N/A" try: etpRSSMessages['removed'][rssAtom]['homepage'] = self.retrieveHomepage(idpackage) except: etpRSSMessages['removed'][rssAtom]['homepage'] = "" # save dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages) idpackage = str(idpackage) # baseinfo self.cursor.execute('DELETE FROM baseinfo WHERE idpackage = '+idpackage) # extrainfo self.cursor.execute('DELETE FROM extrainfo WHERE idpackage = '+idpackage) # content self.cursor.execute('DELETE FROM content WHERE idpackage = '+idpackage) # dependencies self.cursor.execute('DELETE FROM dependencies WHERE idpackage = '+idpackage) # provide self.cursor.execute('DELETE FROM provide WHERE idpackage = '+idpackage) # conflicts self.cursor.execute('DELETE FROM conflicts WHERE idpackage = '+idpackage) # protect self.cursor.execute('DELETE FROM configprotect WHERE idpackage = '+idpackage) # protect_mask self.cursor.execute('DELETE FROM configprotectmask WHERE idpackage = '+idpackage) # sources self.cursor.execute('DELETE FROM sources WHERE idpackage = '+idpackage) # useflags self.cursor.execute('DELETE FROM useflags WHERE idpackage = '+idpackage) # keywords self.cursor.execute('DELETE FROM keywords WHERE idpackage = '+idpackage) # # WARNING: exception won't be handled anymore with 1.0 # try: # messages self.cursor.execute('DELETE FROM messages WHERE idpackage = '+idpackage) except: pass try: # systempackage self.cursor.execute('DELETE FROM systempackages WHERE idpackage = '+idpackage) except: pass try: # counter self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage) except: if self.dbname in [etpConst['clientdbid'],etpConst['serverdbid']]: raise try: # on disk sizes self.cursor.execute('DELETE FROM sizes WHERE idpackage = '+idpackage) except: pass try: # eclasses self.cursor.execute('DELETE FROM eclasses WHERE idpackage = '+idpackage) except: pass try: # needed self.cursor.execute('DELETE FROM needed WHERE idpackage = '+idpackage) except: pass try: # triggers self.cursor.execute('DELETE FROM triggers WHERE idpackage = '+idpackage) except: pass try: # inject table self.cursor.execute('DELETE FROM injected WHERE idpackage = '+idpackage) except: pass # Remove from installedtable if exists self.removePackageFromInstalledTable(idpackage) # Remove from dependstable if exists self.removePackageFromDependsTable(idpackage) # need a final cleanup self.packagesRemoved = True # clear caches self.clearCache() self.commitChanges() def removeMirrorEntries(self,mirrorname): self.cursor.execute('DELETE FROM mirrorlinks WHERE mirrorname = "'+mirrorname+'"') def addMirrors(self,mirrorname,mirrorlist): for x in mirrorlist: self.cursor.execute( 'INSERT into mirrorlinks VALUES ' '(?,?)', (mirrorname,x,) ) def addCategory(self,category): self.cursor.execute( 'INSERT into categories VALUES ' '(NULL,?)', (category,) ) # get info about inserted value and return cat = self.isCategoryAvailable(category) if cat != -1: return cat raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a category but then, fetching it returned -1. There's something broken.") def addProtect(self,protect): self.cursor.execute( 'INSERT into configprotectreference VALUES ' '(NULL,?)', (protect,) ) # get info about inserted value and return prt = self.isProtectAvailable(protect) if prt != -1: return prt raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a protect but then, fetching it returned -1. There's something broken.") def addSource(self,source): self.cursor.execute( 'INSERT into sourcesreference VALUES ' '(NULL,?)', (source,) ) # get info about inserted value and return src = self.isSourceAvailable(source) if src != -1: return src raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a source but then, fetching it returned -1. There's something broken.") def addDependency(self,dependency): self.cursor.execute( 'INSERT into dependenciesreference VALUES ' '(NULL,?)', (dependency,) ) # get info about inserted value and return dep = self.isDependencyAvailable(dependency) if dep != -1: return dep raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a dependency but then, fetching it returned -1. There's something broken.") def addKeyword(self,keyword): self.cursor.execute( 'INSERT into keywordsreference VALUES ' '(NULL,?)', (keyword,) ) # get info about inserted value and return key = self.isKeywordAvailable(keyword) if key != -1: return key raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a keyword but then, fetching it returned -1. There's something broken.") def addUseflag(self,useflag): self.cursor.execute( 'INSERT into useflagsreference VALUES ' '(NULL,?)', (useflag,) ) # get info about inserted value and return use = self.isUseflagAvailable(useflag) if use != -1: return use raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a use flag but then, fetching it returned -1. There's something broken.") def addEclass(self,eclass): self.cursor.execute( 'INSERT into eclassesreference VALUES ' '(NULL,?)', (eclass,) ) # get info about inserted value and return myclass = self.isEclassAvailable(eclass) if myclass != -1: return myclass raise exceptionTools.CorruptionError("CorruptionError: I tried to insert an eclass but then, fetching it returned -1. There's something broken.") def addNeeded(self,needed): self.cursor.execute( 'INSERT into neededreference VALUES ' '(NULL,?)', (needed,) ) # get info about inserted value and return myneeded = self.isNeededAvailable(needed) if myneeded != -1: return myneeded raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a needed library but then, fetching it returned -1. There's something broken.") def addLicense(self,pkglicense): if not entropyTools.is_valid_string(pkglicense): pkglicense = ' ' # workaround for broken license entries self.cursor.execute( 'INSERT into licenses VALUES ' '(NULL,?)', (pkglicense,) ) # get info about inserted value and return lic = self.isLicenseAvailable(pkglicense) if lic != -1: return lic raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a license but then, fetching it returned -1. There's something broken.") def addCompileFlags(self,chost,cflags,cxxflags): self.cursor.execute( 'INSERT into flags VALUES ' '(NULL,?,?,?)', (chost,cflags,cxxflags,) ) # get info about inserted value and return idflag = self.areCompileFlagsAvailable(chost,cflags,cxxflags) if idflag != -1: return idflag raise exceptionTools.CorruptionError("CorruptionError: I tried to insert compile flags but then, fetching it returned -1. There's something broken.") def setInjected(self, idpackage): self.checkReadOnly() if not self.isInjected(idpackage): self.cursor.execute( 'INSERT into injected VALUES ' '(?)' , ( idpackage, ) ) self.commitChanges() # date expressed the unix way def setDateCreation(self, idpackage, date): self.checkReadOnly() self.cursor.execute('UPDATE extrainfo SET datecreation = (?) WHERE idpackage = (?)', (str(date),idpackage,)) self.commitChanges() def setDigest(self, idpackage, digest): self.checkReadOnly() self.cursor.execute('UPDATE extrainfo SET digest = (?) WHERE idpackage = (?)', (digest,idpackage,)) self.commitChanges() def setDownloadURL(self, idpackage, url): self.checkReadOnly() self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (url,idpackage,)) self.commitChanges() def setCategory(self, idpackage, category): self.checkReadOnly() # create new category if it doesn't exist catid = self.isCategoryAvailable(category) if (catid == -1): # create category catid = self.addCategory(category) self.cursor.execute('UPDATE baseinfo SET idcategory = (?) WHERE idpackage = (?)', (catid,idpackage,)) self.commitChanges() def setName(self, idpackage, name): self.checkReadOnly() self.cursor.execute('UPDATE baseinfo SET name = (?) WHERE idpackage = (?)', (name,idpackage,)) self.commitChanges() def setDependency(self, iddependency, dependency): self.checkReadOnly() self.cursor.execute('UPDATE dependenciesreference SET dependency = (?) WHERE iddependency = (?)', (dependency,iddependency,)) self.commitChanges() def setAtom(self, idpackage, atom): self.checkReadOnly() self.cursor.execute('UPDATE baseinfo SET atom = (?) WHERE idpackage = (?)', (atom,idpackage,)) self.commitChanges() def setSlot(self, idpackage, slot): self.checkReadOnly() self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot,idpackage,)) self.commitChanges() def removeLicensedata(self, license_name): if not self.doesTableExist("licensedata"): return self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,)) def removeDependencies(self, idpackage): self.checkReadOnly() self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,)) self.commitChanges() def insertDependencies(self, idpackage, deplist): for dep in deplist: iddep = self.isDependencyAvailable(dep) if (iddep == -1): # create category iddep = self.addDependency(dep) self.cursor.execute( 'INSERT into dependencies VALUES ' '(?,?)' , ( idpackage, iddep, ) ) def removeContent(self, idpackage): self.checkReadOnly() self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,)) self.commitChanges() def insertContent(self, idpackage, content): for xfile in content: contenttype = content[xfile] if type(xfile) is unicode: xfile = xfile.encode('raw_unicode_escape') self.cursor.execute( 'INSERT into content VALUES ' '(?,?,?)' , ( idpackage, xfile, contenttype, ) ) def insertCounter(self, idpackage, counter, branch = None): self.checkReadOnly() if not branch: branch = etpConst['branch'] self.cursor.execute('DELETE FROM counters WHERE counter = (?) OR idpackage = (?)', (counter,idpackage,)) self.cursor.execute('INSERT INTO counters VALUES (?,?,?)', (counter,idpackage,branch,)) self.commitChanges() def setCounter(self, idpackage, counter, branch = None): self.checkReadOnly() branchstring = '' insertdata = [counter,idpackage] if branch: branchstring = ', branch = (?)' insertdata.insert(1,branch) else: branch = etpConst['branch'] try: self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata) except: if self.dbname == etpConst['clientdbid']: raise self.commitChanges() def contentDiff(self, idpackage, dbconn, dbconn_idpackage): self.checkReadOnly() self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") # create a random table and fill randomtable = "cdiff"+str(entropyTools.getRandomNumber()) self.cursor.execute('DROP TABLE IF EXISTS '+randomtable) self.cursor.execute('CREATE TABLE '+randomtable+' ( file VARCHAR )') dbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") dbconn.cursor.execute('select file from content where idpackage = (?)', (dbconn_idpackage,)) xfile = dbconn.cursor.fetchone() while xfile: self.cursor.execute('INSERT INTO '+randomtable+' VALUES (?)', (xfile[0],)) xfile = dbconn.cursor.fetchone() # now compare self.cursor.execute('SELECT file FROM content WHERE content.idpackage = (?) AND content.file NOT IN (SELECT file from '+randomtable+') ', (idpackage,)) diff = self.fetchall2set(self.cursor.fetchall()) self.cursor.execute('DROP TABLE IF EXISTS '+randomtable) return diff def cleanupUseflags(self): self.checkReadOnly() self.cursor.execute('SELECT idflag FROM useflagsreference') idflags = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table orphanedFlags = idflags.copy() foundflags = set() query = 'WHERE idflag = ' counter = 0 run = False for idflag in idflags: run = True counter += 1 query += str(idflag)+' OR idflag = ' if counter > 25: counter = 0 query = query[:-13] self.cursor.execute('SELECT idflag FROM useflags '+query) foundflags.update(self.fetchall2set(self.cursor.fetchall())) query = 'WHERE idflag = ' run = False if (run): query = query[:-13] self.cursor.execute('SELECT idflag FROM useflags '+query) foundflags.update(self.fetchall2set(self.cursor.fetchall())) orphanedFlags.difference_update(foundflags) for idflag in orphanedFlags: self.cursor.execute('DELETE FROM useflagsreference WHERE idflag ='+str(idflag)) self.commitChanges() def cleanupSources(self): self.checkReadOnly() self.cursor.execute('SELECT idsource FROM sourcesreference') idsources = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table orphanedSources = idsources.copy() foundsources = set() query = 'WHERE idsource = ' counter = 0 run = False for idsource in idsources: run = True counter += 1 query += str(idsource)+' OR idsource = ' if counter > 25: counter = 0 query = query[:-15] self.cursor.execute('SELECT idsource FROM sources '+query) foundsources.update(self.fetchall2set(self.cursor.fetchall())) query = 'WHERE idsource = ' run = False if (run): query = query[:-15] self.cursor.execute('SELECT idsource FROM sources '+query) foundsources.update(self.fetchall2set(self.cursor.fetchall())) orphanedSources.difference_update(foundsources) for idsource in orphanedSources: self.cursor.execute('DELETE FROM sourcesreference WHERE idsource = '+str(idsource)) self.commitChanges() def cleanupEclasses(self): self.checkReadOnly() self.cursor.execute('SELECT idclass FROM eclassesreference') idclasses = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table orphanedClasses = idclasses.copy() foundclasses = set() query = 'WHERE idclass = ' counter = 0 run = False for idclass in idclasses: run = True counter += 1 query += str(idclass)+' OR idclass = ' if counter > 25: counter = 0 query = query[:-14] self.cursor.execute('SELECT idclass FROM eclasses '+query) foundclasses.update(self.fetchall2set(self.cursor.fetchall())) query = 'WHERE idclass = ' run = False if (run): query = query[:-14] self.cursor.execute('SELECT idclass FROM eclasses '+query) foundclasses.update(self.fetchall2set(self.cursor.fetchall())) orphanedClasses.difference_update(foundclasses) for idclass in orphanedClasses: self.cursor.execute('DELETE FROM eclassesreference WHERE idclass = '+str(idclass)) self.commitChanges() def cleanupNeeded(self): self.checkReadOnly() self.cursor.execute('SELECT idneeded FROM neededreference') idneededs = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table orphanedNeededs = idneededs.copy() foundneeded = set() query = 'WHERE idneeded = ' counter = 0 run = False for idneeded in idneededs: run = True counter += 1 query += str(idneeded)+' OR idneeded = ' if counter > 25: counter = 0 query = query[:-15] self.cursor.execute('SELECT idneeded FROM needed '+query) foundneeded.update(self.fetchall2set(self.cursor.fetchall())) query = 'WHERE idneeded = ' run = False if (run): query = query[:-15] self.cursor.execute('SELECT idneeded FROM needed '+query) foundneeded.update(self.fetchall2set(self.cursor.fetchall())) orphanedNeededs.difference_update(foundneeded) for idneeded in orphanedNeededs: self.cursor.execute('DELETE FROM neededreference WHERE idneeded = '+str(idneeded)) self.commitChanges() def cleanupDependencies(self): self.checkReadOnly() self.cursor.execute('SELECT iddependency FROM dependenciesreference') iddeps = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table orphanedDeps = iddeps.copy() founddeps = set() query = 'WHERE iddependency = ' counter = 0 run = False for iddep in iddeps: run = True counter += 1 query += str(iddep)+' OR iddependency = ' if counter > 25: counter = 0 query = query[:-19] self.cursor.execute('SELECT iddependency FROM dependencies '+query) founddeps.update(self.fetchall2set(self.cursor.fetchall())) query = 'WHERE iddependency = ' run = False if (run): query = query[:-19] self.cursor.execute('SELECT iddependency FROM dependencies '+query) founddeps.update(self.fetchall2set(self.cursor.fetchall())) orphanedDeps.difference_update(founddeps) for iddep in orphanedDeps: self.cursor.execute('DELETE FROM dependenciesreference WHERE iddependency = '+str(iddep)) self.commitChanges() def getNewNegativeCounter(self): counter = -2 try: mycounters = list(self.listAllCounters(onlycounters = True)) mycounter = min(mycounters) if mycounter >= -1: counter = -2 else: counter = mycounter-1 except: pass return counter def getApi(self): self.cursor.execute('SELECT max(etpapi) FROM baseinfo') return self.cursor.fetchone()[0] def getIDPackage(self, atom, branch = etpConst['branch']): self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = "'+atom+'" AND branch = "'+branch+'"') idpackage = -1 idpackage = self.cursor.fetchone() if idpackage: idpackage = idpackage[0] else: idpackage = -1 return idpackage def getIDPackageFromDownload(self, file, branch = etpConst['branch']): self.cursor.execute('SELECT baseinfo.idpackage FROM content,baseinfo WHERE content.file = (?) and baseinfo.branch = (?)', (file,branch,)) idpackage = self.cursor.fetchone() if idpackage: idpackage = idpackage[0] else: idpackage = -1 return idpackage def getIDPackagesFromFile(self, file): self.cursor.execute('SELECT idpackage FROM content WHERE file = "'+file+'"') idpackages = [] for row in self.cursor: idpackages.append(row[0]) return idpackages def getIDCategory(self, category): self.cursor.execute('SELECT "idcategory" FROM categories WHERE category = "'+str(category)+'"') idcat = -1 for row in self.cursor: idcat = int(row[0]) break return idcat def getIDPackageFromBinaryPackage(self,packageName): self.cursor.execute('SELECT "IDPACKAGE" FROM baseinfo WHERE download = "'+etpConst['binaryurirelativepath']+packageName+'"') idpackage = -1 for row in self.cursor: idpackage = int(row[0]) break return idpackage def getScopeData(self, idpackage): self.cursor.execute(""" SELECT baseinfo.atom, categories.category, baseinfo.name, baseinfo.version, baseinfo.slot, baseinfo.versiontag, baseinfo.revision, baseinfo.branch FROM baseinfo, categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory """, (idpackage,)) return self.cursor.fetchone() def getBaseData(self,idpackage): sql = """ SELECT baseinfo.atom, baseinfo.name, baseinfo.version, baseinfo.versiontag, extrainfo.description, categories.category, flags.chost, flags.cflags, flags.cxxflags, extrainfo.homepage, licenses.license, baseinfo.branch, extrainfo.download, extrainfo.digest, baseinfo.slot, baseinfo.etpapi, extrainfo.datecreation, extrainfo.size, baseinfo.revision FROM baseinfo, extrainfo, categories, flags, licenses WHERE baseinfo.idpackage = '"""+str(idpackage)+"""' and baseinfo.idpackage = extrainfo.idpackage and baseinfo.idcategory = categories.idcategory and extrainfo.idflags = flags.idflags and baseinfo.idlicense = licenses.idlicense """ self.cursor.execute(sql) return self.cursor.fetchone() def getTriggerInfo(self, idpackage): data = {} mydata = self.getScopeData(idpackage) data['atom'] = mydata[0] data['category'] = mydata[1] data['name'] = mydata[2] data['version'] = mydata[3] data['versiontag'] = mydata[5] flags = self.retrieveCompileFlags(idpackage) data['chost'] = flags[0] data['cflags'] = flags[1] data['cxxflags'] = flags[2] data['trigger'] = self.retrieveTrigger(idpackage) data['eclasses'] = self.retrieveEclasses(idpackage) data['content'] = self.retrieveContent(idpackage) return data def getPackageData(self, idpackage, get_content = True): data = {} mydata = self.getBaseData(idpackage) data['name'] = mydata[1] data['version'] = mydata[2] data['versiontag'] = mydata[3] data['description'] = mydata[4] data['category'] = mydata[5] data['chost'] = mydata[6] data['cflags'] = mydata[7] data['cxxflags'] = mydata[8] data['homepage'] = mydata[9] data['useflags'] = self.retrieveUseflags(idpackage) data['license'] = mydata[10] data['keywords'] = self.retrieveKeywords(idpackage) data['branch'] = mydata[11] data['download'] = mydata[12] data['digest'] = mydata[13] data['sources'] = self.retrieveSources(idpackage) data['counter'] = self.retrieveCounter(idpackage) # cannot insert into the sql above data['messages'] = self.retrieveMessages(idpackage) data['trigger'] = self.retrieveTrigger(idpackage) #FIXME: needed for now because of new column if (self.isSystemPackage(idpackage)): data['systempackage'] = 'xxx' else: data['systempackage'] = '' # FIXME: this will be removed when 1.0 will be out data['config_protect'] = self.retrieveProtect(idpackage) data['config_protect_mask'] = self.retrieveProtectMask(idpackage) data['eclasses'] = self.retrieveEclasses(idpackage) data['needed'] = self.retrieveNeeded(idpackage) mirrornames = set() for x in data['sources']: if x.startswith("mirror://"): mirrorname = x.split("/")[2] mirrornames.add(mirrorname) data['mirrorlinks'] = [] for mirror in mirrornames: mirrorlinks = self.retrieveMirrorInfo(mirror) data['mirrorlinks'].append([mirror,mirrorlinks]) data['slot'] = mydata[14] data['injected'] = self.isInjected(idpackage) data['content'] = {} if get_content: mycontent = self.retrieveContent(idpackage, extended = True) for cdata in mycontent: data['content'][cdata[0]] = cdata[1] data['dependencies'] = self.retrieveDependencies(idpackage) data['provide'] = self.retrieveProvide(idpackage) data['conflicts'] = self.retrieveConflicts(idpackage) data['etpapi'] = mydata[15] data['datecreation'] = mydata[16] data['size'] = mydata[17] data['revision'] = mydata[18] data['disksize'] = self.retrieveOnDiskSize(idpackage) # cannot do this too, for backward compat data['licensedata'] = self.retrieveLicensedata(idpackage) return data def fetchall2set(self, item): mycontent = set() for x in item: mycontent |= set(x) return mycontent def fetchall2list(self, item): content = [] for x in item: content += list(x) return content def fetchone2list(self, item): return list(item) def fetchone2set(self, item): return set(item) def clearCache(self): self.live_cache.clear() def do_clear(name): dump_path = os.path.join(etpConst['dumpstoragedir'],name) dump_dir = os.path.dirname(dump_path) if os.path.isdir(dump_dir): for item in os.listdir(dump_dir): item = os.path.join(dump_dir,item) if os.path.isfile(item): os.remove(item) do_clear(etpCache['dbInfo']+"/"+self.dbname+"/") do_clear(etpCache['dbMatch']+"/"+self.dbname+"/") do_clear(etpCache['dbSearch']+"/"+self.dbname+"/") def fetchInfoCache(self, idpackage, function, extra_hash = 0): if self.xcache: c_hash = str(hash(function)) + str(extra_hash) c_match = str(idpackage) try: cached = dumpTools.loadobj(etpCache['dbInfo']+"/"+self.dbname+"/"+c_match+"/"+c_hash) if cached != None: return cached except EOFError: pass def storeInfoCache(self, idpackage, function, info_cache_data, extra_hash = 0): if self.xcache: c_hash = str(hash(function)) + str(extra_hash) c_match = str(idpackage) try: sperms = False if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbInfo']+"/"+self.dbname)): sperms = True dumpTools.dumpobj(etpCache['dbInfo']+"/"+self.dbname+"/"+c_match+"/"+c_hash,info_cache_data) if sperms: const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid']) except IOError: pass def retrieveRepositoryUpdatesDigest(self, repository): self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,)) mydigest = self.cursor.fetchone() if mydigest: return mydigest[0] else: return -1 def listAllTreeUpdatesActions(self): self.cursor.execute('SELECT * FROM treeupdatesactions') return self.cursor.fetchall() def retrieveTreeUpdatesActions(self, repository, forbranch = etpConst['branch']): if not self.doesTableExist("treeupdatesactions"): return set() self.cursor.execute('SELECT command FROM treeupdatesactions where repository = (?) and branch = (?)', (repository,forbranch)) return self.fetchall2set(self.cursor.fetchall()) # mainly used to restore a previous table, used by reagent in --initialize def addTreeUpdatesActions(self, updates): for update in updates: idupdate = update[0] repository = update[1] command = update[2] branch = etpConst['branch'] self.cursor.execute('INSERT INTO treeupdatesactions VALUES (?,?,?,?)', (idupdate,repository,command,branch,)) def setRepositoryUpdatesDigest(self, repository, digest): self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository,digest,)) self.commitChanges() def addRepositoryUpdatesActions(self, repository, actions, forbranch = etpConst['branch']): for command in actions: self.cursor.execute('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?)', (repository,command,forbranch,)) def retrieveAtom(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveAtom') #if cache != None: return cache self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,)) atom = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveAtom',atom) return atom def retrieveBranch(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveBranch') if cache != None: return cache self.cursor.execute('SELECT branch FROM baseinfo WHERE idpackage = (?)', (idpackage,)) br = self.cursor.fetchone()[0] self.storeInfoCache(idpackage,'retrieveBranch',br) return br def retrieveTrigger(self, idpackage): self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,)) trigger = self.cursor.fetchone() if trigger: trigger = trigger[0] else: trigger = '' return trigger def retrieveDownloadURL(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveDownloadURL') if cache != None: return cache self.cursor.execute('SELECT download FROM extrainfo WHERE idpackage = (?)', (idpackage,)) download = self.cursor.fetchone()[0] self.storeInfoCache(idpackage,'retrieveDownloadURL',download) return download def retrieveDescription(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveDescription') #if cache != None: return cache self.cursor.execute('SELECT description FROM extrainfo WHERE idpackage = (?)', (idpackage,)) description = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveDescription',description) return description def retrieveHomepage(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveHomepage') #if cache != None: return cache self.cursor.execute('SELECT homepage FROM extrainfo WHERE idpackage = (?)', (idpackage,)) home = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveHomepage',home) return home def retrieveCounter(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveCounter') #if cache != None: return cache counter = -1 self.cursor.execute('SELECT counter FROM counters WHERE idpackage = (?)', (idpackage,)) mycounter = self.cursor.fetchone() if mycounter: counter = mycounter[0] #self.storeInfoCache(idpackage,'retrieveCounter',int(counter)) return int(counter) def retrieveMessages(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveMessages') if cache != None: return cache messages = [] try: self.cursor.execute('SELECT message FROM messages WHERE idpackage = (?)', (idpackage,)) messages = self.fetchall2list(self.cursor.fetchall()) except: pass self.storeInfoCache(idpackage,'retrieveMessages',messages) return messages # in bytes def retrieveSize(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveSize') #if cache != None: return cache self.cursor.execute('SELECT size FROM extrainfo WHERE idpackage = (?)', (idpackage,)) size = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveSize',size) return size # in bytes def retrieveOnDiskSize(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveOnDiskSize') #if cache != None: return cache self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,)) size = self.cursor.fetchone() # do not use [0]! if not size: size = 0 else: size = size[0] #self.storeInfoCache(idpackage,'retrieveOnDiskSize',size) return size def retrieveDigest(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveDigest') #if cache != None: return cache self.cursor.execute('SELECT digest FROM extrainfo WHERE idpackage = (?)', (idpackage,)) digest = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveDigest',digest) return digest def retrieveName(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveName') #if cache != None: return cache self.cursor.execute('SELECT name FROM baseinfo WHERE idpackage = (?)', (idpackage,)) name = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveName',name) return name def retrieveKeySlot(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveKey') if cache != None: return cache self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,)) data = self.cursor.fetchone() self.storeInfoCache(idpackage,'retrieveKey',data) return data def retrieveVersion(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveVersion') #if cache != None: return cache self.cursor.execute('SELECT version FROM baseinfo WHERE idpackage = (?)', (idpackage,)) ver = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveVersion',ver) return ver def retrieveRevision(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveRevision') #if cache != None: return cache self.cursor.execute('SELECT revision FROM baseinfo WHERE idpackage = (?)', (idpackage,)) rev = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveRevision',rev) return rev def retrieveDateCreation(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveDateCreation') #if cache != None: return cache self.cursor.execute('SELECT "datecreation" FROM extrainfo WHERE idpackage = (?)', (idpackage,)) date = self.cursor.fetchone()[0] if not date: date = "N/A" #FIXME: to be removed? #self.storeInfoCache(idpackage,'retrieveDateCreation',date) return date def retrieveApi(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveApi') #if cache != None: return cache self.cursor.execute('SELECT "etpapi" FROM baseinfo WHERE idpackage = (?)', (idpackage,)) api = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveApi',api) return api def retrieveUseflags(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveUseflags') if cache != None: return cache self.cursor.execute('SELECT flagname FROM useflags,useflagsreference WHERE useflags.idpackage = (?) and useflags.idflag = useflagsreference.idflag', (idpackage,)) flags = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveUseflags',flags) return flags def retrieveEclasses(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveEclasses') if cache != None: return cache self.cursor.execute('SELECT classname FROM eclasses,eclassesreference WHERE eclasses.idpackage = (?) and eclasses.idclass = eclassesreference.idclass', (idpackage,)) classes = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveEclasses',classes) return classes def retrieveNeeded(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveNeeded') if cache != None: return cache self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,)) needed = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveNeeded',needed) return needed def retrieveConflicts(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveConflicts') if cache != None: return cache self.cursor.execute('SELECT "conflict" FROM conflicts WHERE idpackage = (?)', (idpackage,)) confl = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveConflicts',confl) return confl def retrieveProvide(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveProvide') #if cache != None: return cache self.cursor.execute('SELECT atom FROM provide WHERE idpackage = (?)', (idpackage,)) provide = self.fetchall2set(self.cursor.fetchall()) #self.storeInfoCache(idpackage,'retrieveProvide',provide) return provide def retrieveDependenciesList(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveDependenciesList') if cache != None: return cache deps = self.retrieveDependencies(idpackage) conflicts = self.retrieveConflicts(idpackage) for x in conflicts: if x[0] != "!": x = "!"+x deps.add(x) del conflicts self.storeInfoCache(idpackage,'retrieveDependenciesList',deps) return deps def retrieveDependencies(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveDependencies') if cache != None: return cache self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) and dependencies.iddependency = dependenciesreference.iddependency', (idpackage,)) deps = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveDependencies',deps) return deps def retrieveIdDependencies(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveIdDependencies') if cache != None: return cache self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = (?)', (idpackage,)) iddeps = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveIdDependencies',iddeps) return iddeps def retrieveDependencyFromIddependency(self, iddependency): self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = (?)', (iddependency,)) return self.cursor.fetchone()[0] def retrieveKeywords(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveKeywords') if cache != None: return cache self.cursor.execute('SELECT keywordname FROM keywords,keywordsreference WHERE keywords.idpackage = (?) and keywords.idkeyword = keywordsreference.idkeyword', (idpackage,)) kw = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveKeywords',kw) return kw def retrieveProtect(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveProtect') if cache != None: return cache self.cursor.execute('SELECT protect FROM configprotect,configprotectreference WHERE configprotect.idpackage = (?) and configprotect.idprotect = configprotectreference.idprotect', (idpackage,)) protect = self.cursor.fetchone() if not protect: protect = '' else: protect = protect[0] self.storeInfoCache(idpackage,'retrieveProtect',protect) return protect def retrieveProtectMask(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveProtectMask') if cache != None: return cache self.cursor.execute('SELECT protect FROM configprotectmask,configprotectreference WHERE idpackage = (?) and configprotectmask.idprotect= configprotectreference.idprotect', (idpackage,)) protect = self.cursor.fetchone() if not protect: protect = '' else: protect = protect[0] self.storeInfoCache(idpackage,'retrieveProtectMask',protect) return protect def retrieveSources(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveSources') if cache != None: return cache self.cursor.execute('SELECT sourcesreference.source FROM sources,sourcesreference WHERE idpackage = (?) and sources.idsource = sourcesreference.idsource', (idpackage,)) sources = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveSources',sources) return sources def retrieveContent(self, idpackage, extended = False, contentType = None): c_hash = -4 if extended: c_hash += 2 c_hash += hash(contentType) cache = self.fetchInfoCache(idpackage,'retrieveContent', extra_hash = c_hash) if cache != None: return cache # like portage does self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") extstring = '' if extended: extstring = ",type" searchkeywords = [idpackage] contentstring = '' if contentType: searchkeywords.append(contentType) contentstring = ' and type = (?)' self.cursor.execute('SELECT file'+extstring+' FROM content WHERE idpackage = (?) '+contentstring, searchkeywords) if extended: fl = self.cursor.fetchall() else: fl = self.fetchall2set(self.cursor.fetchall()) self.storeInfoCache(idpackage,'retrieveContent',fl, extra_hash = c_hash) return fl def retrieveSlot(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveSlot') #if cache != None: return cache self.cursor.execute('SELECT slot FROM baseinfo WHERE idpackage = (?)', (idpackage,)) ver = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveSlot',ver) return ver def retrieveVersionTag(self, idpackage): #cache = self.fetchInfoCache(idpackage,'retrieveVersionTag') #if cache != None: return cache self.cursor.execute('SELECT versiontag FROM baseinfo WHERE idpackage = (?)', (idpackage,)) vtag = self.cursor.fetchone()[0] #self.storeInfoCache(idpackage,'retrieveVersionTag',ver) return vtag def retrieveMirrorInfo(self, mirrorname): self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) mirrorlist = self.fetchall2set(self.cursor.fetchall()) return mirrorlist def retrieveCategory(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveCategory') if cache != None: return cache self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,)) cat = self.cursor.fetchone()[0] self.storeInfoCache(idpackage,'retrieveCategory',cat) return cat def retrieveLicensedata(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveLicensedata') if cache != None: return cache # insert license information if not self.doesTableExist("licensedata"): return {} licenses = self.retrieveLicense(idpackage) licenses = licenses.split() licdata = {} for licname in licenses: licname = licname.strip() if not entropyTools.is_valid_string(licname): continue self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,)) lictext = self.cursor.fetchone() if lictext != None: #licdata[licname] = unicode(lictext[0],'raw_unicode_escape','replace') licdata[licname] = str(lictext[0]) self.storeInfoCache(idpackage,'retrieveLicensedata',licdata) return licdata def retrieveLicensedataKeys(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveLicensedataKeys') if cache != None: return cache if not self.doesTableExist("licensedata"): return set() licenses = self.retrieveLicense(idpackage) licenses = licenses.split() licdata = set() for licname in licenses: licname = licname.strip() if not entropyTools.is_valid_string(licname): continue self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,)) licidentifier = self.cursor.fetchone() if licidentifier: licdata.add(licidentifier[0]) self.storeInfoCache(idpackage,'retrieveLicensedataKeys',licdata) return licdata def retrieveLicenseText(self, license_name): if not self.doesTableExist("licensedata"): return None self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (license_name,)) text = self.cursor.fetchone() if not text: return None #return unicode(text[0],'raw_unicode_escape','replace') return str(text[0]) def retrieveLicense(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveLicense') if cache != None: return cache self.cursor.execute('SELECT license FROM baseinfo,licenses WHERE baseinfo.idpackage = (?) and baseinfo.idlicense = licenses.idlicense', (idpackage,)) licname = self.cursor.fetchone()[0] self.storeInfoCache(idpackage,'retrieveLicense',licname) return licname def retrieveCompileFlags(self, idpackage): cache = self.fetchInfoCache(idpackage,'retrieveCompileFlags') if cache != None: return cache self.cursor.execute('SELECT "idflags" FROM extrainfo WHERE idpackage = (?)', (idpackage,)) idflag = self.cursor.fetchone()[0] # now get the flags self.cursor.execute('SELECT chost,cflags,cxxflags FROM flags WHERE idflags = (?)', (idflag,)) flags = self.cursor.fetchone() if not flags: flags = ("N/A","N/A","N/A") self.storeInfoCache(idpackage,'retrieveCompileFlags',flags) return flags def retrieveDepends(self, idpackage, atoms = False): # sanity check on the table sanity = self.isDependsTableSane() if not sanity: # is empty, need generation self.regenerateDependsTable(output = False) if atoms: self.cursor.execute('SELECT baseinfo.atom FROM dependstable,dependencies,baseinfo WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency and baseinfo.idpackage = dependencies.idpackage', (idpackage,)) else: self.cursor.execute('SELECT dependencies.idpackage FROM dependstable,dependencies WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency', (idpackage,)) result = self.fetchall2set(self.cursor.fetchall()) return result # You must provide the full atom to this function # WARNING: this function does not support branches # NOTE: server side uses this regardless branch specification because it already handles it in updatePackage() def isPackageAvailable(self,pkgatom): pkgatom = entropyTools.removePackageOperators(pkgatom) self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)', (pkgatom,)) result = self.cursor.fetchone() if result: return result[0] return -1 def isIDPackageAvailable(self,idpackage): self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = (?)', (idpackage,)) result = self.cursor.fetchone() if not result: return False return True # This version is more specific and supports branches def isSpecificPackageAvailable(self, pkgkey, branch, branch_operator = "="): pkgkey = entropyTools.removePackageOperators(pkgkey) self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?) AND branch '+branch_operator+' (?)', (pkgkey,branch,)) result = self.cursor.fetchone() if not result: return False return True def isCategoryAvailable(self,category): self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isProtectAvailable(self,protect): self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isFileAvailable(self, myfile): self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (myfile,)) result = self.cursor.fetchone() rc = False if result: rc = True return rc def isSourceAvailable(self,source): self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = "'+source+'"') result = self.cursor.fetchone() if not result: return -1 return result[0] def isDependencyAvailable(self,dependency): self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = (?)', (dependency,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isKeywordAvailable(self,keyword): self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = (?)', (keyword,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isUseflagAvailable(self,useflag): self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = (?)', (useflag,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isEclassAvailable(self,eclass): self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isNeededAvailable(self,needed): self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isCounterAvailable(self,counter, branch = None, branch_operator = "="): result = False if not branch: branch = etpConst['branch'] self.cursor.execute('SELECT counter FROM counters WHERE counter = (?) and branch '+branch_operator+' (?)', (counter,branch,)) result = self.cursor.fetchone() if result: result = True return result def isLicensedataKeyAvailable(self, license_name): if not self.doesTableExist("licensedata"): return True self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,)) result = self.cursor.fetchone() if not result: return False return True def isLicenseAccepted(self, license_name): self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,)) result = self.cursor.fetchone() if not result: return False return True def acceptLicense(self, license_name): if self.readOnly or (not entropyTools.is_user_in_entropy_group()): return if self.isLicenseAccepted(license_name): return self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,)) self.commitChanges() def isLicenseAvailable(self,pkglicense): if not entropyTools.is_valid_string(pkglicense): pkglicense = ' ' self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def isSystemPackage(self,idpackage): self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) result = self.cursor.fetchone() if result: return True return False def isInjected(self,idpackage): cache = self.fetchInfoCache(idpackage,'isInjected') if cache != None: return cache try: self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,)) except: # readonly database? return False result = self.cursor.fetchone() rslt = False if result: rslt = True self.storeInfoCache(idpackage,'isInjected',rslt) return rslt def areCompileFlagsAvailable(self,chost,cflags,cxxflags): self.cursor.execute('SELECT idflags FROM flags WHERE chost = (?) AND cflags = (?) AND cxxflags = (?)', (chost,cflags,cxxflags,)) result = self.cursor.fetchone() if not result: return -1 return result[0] def searchBelongs(self, file, like = False, branch = None, branch_operator = "="): branchstring = '' searchkeywords = [file] if branch: searchkeywords.append(branch) branchstring = ' and baseinfo.branch '+branch_operator+' (?)' if (like): self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file LIKE (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords) else: self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file = (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords) return self.fetchall2set(self.cursor.fetchall()) ''' search packages that uses the eclass provided ''' def searchEclassedPackages(self, eclass, atoms = False): # atoms = return atoms directly if atoms: self.cursor.execute('SELECT baseinfo.atom,eclasses.idpackage FROM baseinfo,eclasses,eclassesreference WHERE eclassesreference.classname = (?) and eclassesreference.idclass = eclasses.idclass and eclasses.idpackage = baseinfo.idpackage', (eclass,)) return self.cursor.fetchall() else: self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (eclass,)) return self.fetchall2set(self.cursor.fetchall()) ''' search packages whose versiontag matches the one provided ''' def searchTaggedPackages(self, tag, atoms = False): # atoms = return atoms directly if atoms: self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) return self.cursor.fetchall() else: self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) return self.fetchall2set(self.cursor.fetchall()) def searchLicenses(self, mylicense, caseSensitive = False, atoms = False): if not entropyTools.is_valid_string(mylicense): return [] request = "baseinfo.idpackage" if atoms: request = "baseinfo.atom,baseinfo.idpackage" if caseSensitive: self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE licenses.license LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%",)) else: self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE LOWER(licenses.license) LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%".lower(),)) if atoms: return self.cursor.fetchall() return self.fetchall2set(self.cursor.fetchall()) ''' search packages whose slot matches the one provided ''' def searchSlottedPackages(self, slot, atoms = False): # atoms = return atoms directly if atoms: self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE slot = (?)', (slot,)) return self.cursor.fetchall() else: self.cursor.execute('SELECT idpackage FROM baseinfo WHERE slot = (?)', (slot,)) return self.fetchall2set(self.cursor.fetchall()) def searchKeySlot(self, key, slot, branch = None): branchstring = '' params = [key,slot] if branch: params.append(branch) branchstring = ' and baseinfo.branch = (?)' self.cursor.execute('SELECT idpackage FROM baseinfo,categories WHERE categories.category || "/" || baseinfo.name = (?) and baseinfo.slot = (?) and baseinfo.idcategory = categories.idcategory'+branchstring, params) data = self.cursor.fetchall() return data ''' search packages that need the specified library (in neededreference table) specified by keyword ''' def searchNeeded(self, keyword, like = False): if like: self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library LIKE (?) and needed.idneeded = neededreference.idneeded', (keyword,)) else: self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = (?) and needed.idneeded = neededreference.idneeded', (keyword,)) return self.fetchall2set(self.cursor.fetchall()) # FIXME: deprecate and add functionalities to the function above ''' same as above but with branch support ''' def searchNeededInBranch(self, keyword, branch): self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference,baseinfo WHERE library = (?) and needed.idneeded = neededreference.idneeded and baseinfo.branch = (?)', (keyword,branch,)) return self.fetchall2set(self.cursor.fetchall()) ''' search dependency string inside dependenciesreference table and retrieve iddependency ''' def searchDependency(self, dep, like = False, multi = False): sign = "=" if like: sign = "LIKE" dep = "%"+dep+"%" self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency '+sign+' (?)', (dep,)) if multi: return self.fetchall2set(self.cursor.fetchall()) else: iddep = self.cursor.fetchone() if iddep: iddep = iddep[0] else: iddep = -1 return iddep ''' search iddependency inside dependencies table and retrieve idpackages ''' def searchIdpackageFromIddependency(self, iddep): self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,)) return self.fetchall2set(self.cursor.fetchall()) def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None): searchkeywords = ["%"+keyword+"%"] slotstring = '' if slot: searchkeywords.append(slot) slotstring = ' and slot = (?)' tagstring = '' if tag: searchkeywords.append(tag) tagstring = ' and versiontag = (?)' branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and branch = (?)' if (sensitive): self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE (?)'+slotstring+tagstring+branchstring, searchkeywords) else: self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE (?)'+slotstring+tagstring+branchstring, searchkeywords) return self.cursor.fetchall() def searchProvide(self, keyword, slot = None, tag = None, branch = None): slotstring = '' searchkeywords = [keyword] if slot: searchkeywords.append(slot) slotstring = ' and baseinfo.slot = (?)' tagstring = '' if tag: searchkeywords.append(tag) tagstring = ' and baseinfo.versiontag = (?)' branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and baseinfo.branch = (?)' self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,provide WHERE provide.atom = (?) and provide.idpackage = baseinfo.idpackage'+slotstring+tagstring+branchstring, searchkeywords) results = self.cursor.fetchall() return results def searchPackagesByDescription(self, keyword): self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo WHERE LOWER(extrainfo.description) LIKE (?) and baseinfo.idpackage = extrainfo.idpackage', ("%"+keyword.lower()+"%",)) return self.cursor.fetchall() def searchPackagesByName(self, keyword, sensitive = False, branch = None): if sensitive: searchkeywords = [keyword] else: searchkeywords = [keyword.lower()] branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and branch = (?)' if sensitive: self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords) else: self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords) results = self.cursor.fetchall() return results def searchPackagesByCategory(self, keyword, like = False, branch = None): searchkeywords = [keyword] branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and branch = (?)' if like: self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category LIKE (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) else: self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) results = self.cursor.fetchall() return results def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None): namestring = 'baseinfo.name' catstring = 'categories.category' myname = name mycat = category if not sensitive: namestring = 'LOWER(baseinfo.name)' catstring = 'LOWER(categories.category)' myname = name.lower() mycat = category.lower() searchkeywords = [myname,mycat] branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and baseinfo.branch = (?)' self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE '+namestring+' = (?) AND '+catstring+' = (?) and baseinfo.idcategory = categories.idcategory'+branchstring, searchkeywords) results = self.cursor.fetchall() return results def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False): searchkeywords = [] if sensitive: searchkeywords.append(key) else: searchkeywords.append(key.lower()) searchkeywords.append(version) branchstring = '' if branch: searchkeywords.append(branch) branchstring = ' and branch = (?)' if (sensitive): self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category || "/" || baseinfo.name = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) else: self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE LOWER(categories.category) || "/" || LOWER(baseinfo.name) = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords) results = self.cursor.fetchall() return results def listAllPackages(self): self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo') return self.cursor.fetchall() def listAllInjectedPackages(self, justFiles = False): self.cursor.execute('SELECT idpackage FROM injected') injecteds = self.fetchall2set(self.cursor.fetchall()) results = set() # get download for injected in injecteds: download = self.retrieveDownloadURL(injected) if justFiles: results.add(download) else: results.add((download,injected)) return results def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="): branchstring = '' if branch: branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"' if onlycounters: self.cursor.execute('SELECT counter FROM counters'+branchstring) return self.fetchall2set(self.cursor.fetchall()) else: self.cursor.execute('SELECT counter,idpackage FROM counters'+branchstring) return self.cursor.fetchall() def listAllIdpackages(self, branch = None, branch_operator = "="): # XXX if you will decide to re-enable this, consider that you have to clean this cache # each time a package is updated/installed/removed #cache = self.fetchInfoCache(hash(branch),'listAllIdpackages') #if cache != None: return cache branchstring = '' searchkeywords = [] if branch: searchkeywords.append(branch) branchstring = ' where branch %s (?)' % (str(branch_operator),) self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring, searchkeywords) results = self.fetchall2set(self.cursor.fetchall()) #self.storeInfoCache(hash(branch),'listAllIdpackages',results) return results def listAllDependencies(self): self.cursor.execute('SELECT * FROM dependenciesreference') return self.cursor.fetchall() def listAllBranches(self): cache = self.live_cache.get('listAllBranches') if cache != None: return cache self.cursor.execute('SELECT distinct branch FROM baseinfo') results = self.fetchall2set(self.cursor.fetchall()) self.live_cache['listAllBranches'] = results.copy() return results def listIdPackagesInIdcategory(self,idcategory): self.cursor.execute('SELECT idpackage FROM baseinfo where idcategory = (?)', (idcategory,)) return self.fetchall2set(self.cursor.fetchall()) def listIdpackageDependencies(self, idpackage): self.cursor.execute('SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies WHERE dependencies.idpackage = (?) AND dependenciesreference.iddependency = dependencies.iddependency', (idpackage,)) return set(self.cursor.fetchall()) def listBranchPackagesTbz2(self, branch): result = set() pkglist = self.listBranchPackages(branch) for pkg in pkglist: idpackage = pkg[1] url = self.retrieveDownloadURL(idpackage) if url: result.add(os.path.basename(url)) if (result): result = list(result) result.sort() return result def listBranchPackages(self, branch): self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE branch = (?)', (branch,)) return self.cursor.fetchall() def listAllFiles(self, clean = False): self.cursor.execute('SELECT file FROM content') if clean: return self.fetchall2set(self.cursor.fetchall()) else: return self.fetchall2list(self.cursor.fetchall()) def listAllCategories(self): self.cursor.execute('SELECT idcategory,category FROM categories') return self.cursor.fetchall() def listConfigProtectDirectories(self, mask = False): query = 'SELECT max(idprotect) FROM configprotect' if mask: query += 'mask' self.cursor.execute(query) r = self.cursor.fetchone() if not r: return [] mymax = r[0] self.cursor.execute('SELECT protect FROM configprotectreference where idprotect >= (?) and idprotect <= (?) order by protect', (1,mymax,)) results = self.cursor.fetchall() dirs = [] for row in results: mydirs = row[0].split() for x in mydirs: if x not in dirs: dirs.append(x) return dirs def switchBranch(self, idpackage, tobranch): self.checkReadOnly() mycat = self.retrieveCategory(idpackage) myname = self.retrieveName(idpackage) myslot = self.retrieveSlot(idpackage) mybranch = self.retrieveBranch(idpackage) mydownload = self.retrieveDownloadURL(idpackage) import re out = re.subn('/'+mybranch+'/','/'+tobranch+'/',mydownload) newdownload = out[0] # remove package with the same key+slot and tobranch if exists match = self.atomMatch(mycat+"/"+myname, matchSlot = myslot, matchBranches = (tobranch,)) if match[0] != -1: self.removePackage(match[0]) # now switch selected idpackage to the new branch self.cursor.execute('UPDATE baseinfo SET branch = (?) WHERE idpackage = (?)', (tobranch,idpackage,)) self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (newdownload,idpackage,)) self.commitChanges() def databaseStructureUpdates(self): if not self.doesTableExist("licensedata"): self.createLicensedataTable() if not self.doesTableExist("licenses_accepted") and (self.dbname == etpConst['clientdbid']): self.createLicensesAcceptedTable() if not self.doesColumnInTableExist("baseinfo","trigger"): self.createTriggerColumn() if not self.doesTableExist("counters"): self.createCountersTable() elif not self.doesColumnInTableExist("counters","branch"): self.createCountersBranchColumn() if not self.doesTableExist("sizes"): self.createSizesTable() if not self.doesTableExist("triggers"): self.createTriggerTable() if not self.doesTableExist("messages"): self.createMessagesTable() if not self.doesTableExist("injected"): self.createInjectedTable() if not self.doesTableExist("systempackages"): self.createSystemPackagesTable() if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): self.createProtectTable() if not self.doesColumnInTableExist("content","type"): self.createContentTypeColumn() if not self.doesTableExist("eclasses"): self.createEclassesTable() if not self.doesTableExist("treeupdates"): self.createTreeupdatesTable() if not self.doesTableExist("treeupdatesactions"): self.createTreeupdatesactionsTable() elif not self.doesColumnInTableExist("treeupdatesactions","branch"): self.createTreeupdatesactionsBranchColumn() if not self.doesTableExist("needed"): self.createNeededTable() if not self.doesTableExist("installedtable") and (self.dbname == etpConst['clientdbid']): self.createInstalledTable() # these are the tables moved to INTEGER PRIMARY KEY AUTOINCREMENT autoincrement_tables = [ 'treeupdatesactions', 'neededreference', 'eclassesreference', 'configprotectreference', 'flags', 'licenses', 'categories', 'keywordsreference', 'useflagsreference', 'sourcesreference', 'dependenciesreference', 'baseinfo' ] autoinc = False for table in autoincrement_tables: x = self.migrateTableToAutoincrement(table) if x: autoinc = True if autoinc: self.updateProgress( red("Entropy database: regenerating indexes after migration."), importance = 1, type = "warning", header = blue(" !!! ") ) self.createAllIndexes() def migrateTableToAutoincrement(self, table): self.cursor.execute('select sql from sqlite_master where type = (?) and name = (?);', ("table",table)) schema = self.cursor.fetchone() if not schema: return False schema = schema[0] if schema.find("AUTOINCREMENT") != -1: return False schema = schema.replace('PRIMARY KEY','PRIMARY KEY AUTOINCREMENT') new_schema = schema totable = table+"_autoincrement" schema = schema.replace('CREATE TABLE '+table,'CREATE TEMPORARY TABLE '+totable) self.updateProgress( red("Entropy database: migrating table ")+blue(table), importance = 1, type = "warning", header = blue(" !!! ") ) # create table self.cursor.execute('DROP TABLE IF EXISTS '+totable) self.cursor.execute(schema) columns = ','.join(self.getColumnsInTable(table)) temp_query = 'INSERT INTO '+totable+' SELECT '+columns+' FROM '+table self.cursor.execute(temp_query) self.cursor.execute('DROP TABLE '+table) self.cursor.execute(new_schema) temp_query = 'INSERT INTO '+table+' SELECT '+columns+' FROM '+totable self.cursor.execute(temp_query) self.cursor.execute('DROP TABLE '+totable) self.commitChanges() return True def validateDatabase(self): self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","baseinfo")) rslt = self.cursor.fetchone() if rslt == None: raise exceptionTools.SystemDatabaseError("SystemDatabaseError: table baseinfo not found. Either does not exist or corrupted.") self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","extrainfo")) rslt = self.cursor.fetchone() if rslt == None: raise exceptionTools.SystemDatabaseError("SystemDatabaseError: table extrainfo not found. Either does not exist or corrupted.") def checkDatabaseApi(self): dbapi = self.getApi() if dbapi > etpConst['etpapi']: self.updateProgress( red("Repository EAPI > Entropy EAPI. Please update Equo/Entropy as soon as possible !"), importance = 1, type = "warning", header = " * ! * ! * ! * " ) def doDatabaseImport(self, dumpfile, dbfile): import subprocess sqlite3_exec = "/usr/bin/sqlite3 %s < %s" % (dbfile,dumpfile,) retcode = subprocess.call(sqlite3_exec, shell = True) return retcode def doDatabaseExport(self, dumpfile): dumpfile.write("BEGIN TRANSACTION;\n") self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'") for name, type, sql in self.cursor.fetchall(): self.updateProgress( red("Exporting database table ")+"["+blue(str(name))+"]", importance = 0, type = "info", back = True, header = " " ) if name == "sqlite_sequence": dumpfile.write("DELETE FROM sqlite_sequence;\n") elif name == "sqlite_stat1": dumpfile.write("ANALYZE sqlite_master;\n") elif name.startswith("sqlite_"): continue else: dumpfile.write("%s;\n" % sql) self.cursor.execute("PRAGMA table_info('%s')" % name) cols = [str(r[1]) for r in self.cursor.fetchall()] q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES(" q += ", ".join(["'||quote(" + c + ")||'" for c in cols]) q += ")' FROM '%(tbl_name)s'" self.cursor.execute(q % {'tbl_name': name}) self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") for row in self.cursor: dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape'))) self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'") for name, type, sql in self.cursor.fetchall(): dumpfile.write("%s;\n" % sql) dumpfile.write("COMMIT;\n") try: dumpfile.flush() except: pass self.updateProgress( red("Database Export completed."), importance = 0, type = "info", header = " " ) # remember to close the file # FIXME: this is only compatible with SQLITE def doesTableExist(self, table): self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table",table)) rslt = self.cursor.fetchone() if rslt == None: return False return True # FIXME: this is only compatible with SQLITE def doesColumnInTableExist(self, table, column): self.cursor.execute('PRAGMA table_info( '+table+' )') rslt = self.cursor.fetchall() if not rslt: return False found = False for row in rslt: if row[1] == column: found = True break return found # FIXME: this is only compatible with SQLITE def getColumnsInTable(self, table): self.cursor.execute('PRAGMA table_info( '+table+' )') rslt = self.cursor.fetchall() columns = [] for row in rslt: columns.append(row[1]) return columns def tablesChecksum(self): # NOTE: if you will add dependstable to the validation # please have a look at EquoInterface.__install_package_into_database world calculation cache stuff self.cursor.execute('select * from baseinfo') c_hash = 0 for row in self.cursor: c_hash += hash(str(row)) self.cursor.execute('select * from dependenciesreference') for row in self.cursor: c_hash += hash(str(row)) return str(c_hash) ######################################################## #### ## Client Database API / but also used by server part # def addPackageToInstalledTable(self, idpackage, repositoryName): self.checkReadOnly() self.cursor.execute( 'INSERT into installedtable VALUES ' '(?,?)' , ( idpackage, repositoryName, ) ) self.commitChanges() def retrievePackageFromInstalledTable(self, idpackage): self.checkReadOnly() result = 'Not available' try: self.cursor.execute('SELECT repositoryname FROM installedtable WHERE idpackage = (?)', (idpackage,)) return self.cursor.fetchone()[0] # it's ok because it's inside try/except except: pass return result def removePackageFromInstalledTable(self, idpackage): self.cursor.execute('DELETE FROM installedtable WHERE idpackage = (?)', (idpackage,)) def removePackageFromDependsTable(self, idpackage): try: self.cursor.execute('DELETE FROM dependstable WHERE idpackage = (?)', (idpackage,)) return 0 except: return 1 # need reinit def removeDependencyFromDependsTable(self, iddependency): self.checkReadOnly() try: self.cursor.execute('DELETE FROM dependstable WHERE iddependency = (?)',(iddependency,)) self.commitChanges() return 0 except: return 1 # need reinit # temporary/compat functions def createDependsTable(self): self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS dependstable;') self.cursor.execute('CREATE TABLE dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER );') # this will be removed when dependstable is refilled properly self.cursor.execute( 'INSERT into dependstable VALUES ' '(?,?)' , ( -1, -1, ) ) self.commitChanges() def sanitizeDependsTable(self): self.cursor.execute('DELETE FROM dependstable where iddependency = -1') self.commitChanges() def isDependsTableSane(self): try: self.cursor.execute('SELECT iddependency FROM dependstable WHERE iddependency = -1') except: return False # table does not exist, please regenerate and re-run status = self.cursor.fetchone() if status: return False self.cursor.execute('select count(*) from dependstable') dependstable_count = self.cursor.fetchone() if dependstable_count == 0: return False return True def createXpakTable(self): self.checkReadOnly() self.cursor.execute('CREATE TABLE xpakdata ( idpackage INTEGER PRIMARY KEY, data BLOB );') self.commitChanges() def storeXpakMetadata(self, idpackage, blob): self.cursor.execute( 'INSERT into xpakdata VALUES ' '(?,?)', ( int(idpackage), buffer(blob), ) ) self.commitChanges() def retrieveXpakMetadata(self, idpackage): try: self.cursor.execute('SELECT data from xpakdata where idpackage = (?)', (idpackage,)) mydata = self.cursor.fetchone() if not mydata: return "" else: return mydata[0] except: return "" pass def createCountersTable(self): self.cursor.execute('DROP TABLE IF EXISTS counters;') self.cursor.execute('CREATE TABLE counters ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );') def CreatePackedDataTable(self): self.cursor.execute('CREATE TABLE packed_data ( idpack INTEGER PRIMARY KEY, data BLOB );') def createAllIndexes(self): #if not self.dbname.startswith(etpConst['dbnamerepoprefix']): self.createContentIndex() self.createBaseinfoIndex() self.createKeywordsIndex() self.createDependenciesIndex() self.createExtrainfoIndex() self.createNeededIndex() self.createUseflagsIndex() self.createLicensedataIndex() self.createLicensesIndex() self.createConfigProtectReferenceIndex() def createNeededIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library )') self.commitChanges() def createUseflagsIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( flagname )') self.commitChanges() def createContentIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex_couple ON content ( idpackage )') self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex_file ON content ( file )') self.commitChanges() def createConfigProtectReferenceIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )') self.commitChanges() def createBaseinfoIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_atom ON baseinfo ( atom )') self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_branch_name ON baseinfo ( name,branch )') self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_branch_name_idcategory ON baseinfo ( name,idcategory,branch )') self.commitChanges() def createLicensedataIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: if not self.doesTableExist("licensedata"): return self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )') self.commitChanges() def createLicensesIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS licensesindex ON licenses ( license )') self.commitChanges() def createKeywordsIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname )') self.commitChanges() def createDependenciesIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesindex_idpackage_iddependency ON dependencies ( idpackage, iddependency )') self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesreferenceindex_dependency ON dependenciesreference ( dependency )') self.commitChanges() def createExtrainfoIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( description )') self.commitChanges() def regenerateCountersTable(self, vdb_path, output = False): self.checkReadOnly() self.createCountersTable() # assign a counter to an idpackage myids = self.listAllIdpackages() for myid in myids: # get atom myatom = self.retrieveAtom(myid) mybranch = self.retrieveBranch(myid) myatom = entropyTools.remove_tag(myatom) myatomcounterpath = vdb_path+myatom+"/"+etpConst['spm']['xpak_entries']['counter'] if os.path.isfile(myatomcounterpath): try: f = open(myatomcounterpath,"r") counter = int(f.readline().strip()) f.close() except: if output: self.updateProgress(red("ATTENTION: cannot open Gentoo counter file for: %s") % (bold(myatom),), importance = 1, type = "warning") continue # insert id+counter try: self.cursor.execute( 'INSERT into counters VALUES ' '(?,?,?)', ( counter, myid, mybranch ) ) except: if output: self.updateProgress(red("ATTENTION: counter for atom %s")+red(" is duplicated. Ignoring.") % (bold(myatom),), importance = 1, type = "warning") continue # don't trust counters, they might not be unique self.commitChanges() def clearTreeupdatesEntries(self, repository): self.checkReadOnly() # treeupdates self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,)) def resetTreeupdatesDigests(self): self.checkReadOnly() self.cursor.execute('UPDATE treeupdates SET digest = "-1"') # # FIXME: remove these when 1.0 will be out # def migrateCountersTable(self): self.cursor.execute('DROP TABLE IF EXISTS counterstemp;') self.cursor.execute('CREATE TABLE counterstemp ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );') self.cursor.execute('select * from counters') countersdata = self.cursor.fetchall() if countersdata: for row in countersdata: self.cursor.execute('INSERT INTO counterstemp VALUES = (?,?,?)',row) self.cursor.execute('DROP TABLE counters') self.cursor.execute('ALTER TABLE counterstemp RENAME TO counters') self.commitChanges() def createTreeupdatesTable(self): self.cursor.execute('DROP TABLE IF EXISTS treeupdates;') self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );') def createTreeupdatesactionsTable(self): self.cursor.execute('DROP TABLE IF EXISTS treeupdatesactions;') self.cursor.execute('CREATE TABLE treeupdatesactions ( idupdate INTEGER PRIMARY KEY AUTOINCREMENT, repository VARCHAR, command VARCHAR, branch VARCHAR );') def createSizesTable(self): self.cursor.execute('DROP TABLE IF EXISTS sizes;') self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );') def createCountersBranchColumn(self): self.cursor.execute('ALTER TABLE counters ADD COLUMN branch VARCHAR;') idpackages = self.listAllIdpackages() for idpackage in idpackages: branch = self.retrieveBranch(idpackage) self.cursor.execute('UPDATE counters SET branch = (?) WHERE idpackage = (?)', (branch,idpackage,)) def createTreeupdatesactionsBranchColumn(self): try: # if database disk image is malformed, won't raise exception here self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN branch VARCHAR;') self.cursor.execute('UPDATE treeupdatesactions SET branch = (?)', (str(etpConst['branch']),)) except: pass def createContentTypeColumn(self): try: # if database disk image is malformed, won't raise exception here self.cursor.execute('ALTER TABLE content ADD COLUMN type VARCHAR;') self.cursor.execute('UPDATE content SET type = "0"') except: pass def createLicensedataTable(self): self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );') def createLicensesAcceptedTable(self): self.cursor.execute('CREATE TABLE licenses_accepted ( licensename VARCHAR UNIQUE );') def createTriggerTable(self): self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );') def createTriggerColumn(self): self.checkReadOnly() self.cursor.execute('ALTER TABLE baseinfo ADD COLUMN trigger INTEGER;') self.cursor.execute('UPDATE baseinfo SET trigger = 0') def createMessagesTable(self): self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);") def createEclassesTable(self): self.cursor.execute('DROP TABLE IF EXISTS eclasses;') self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;') self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );') self.cursor.execute('CREATE TABLE eclassesreference ( idclass INTEGER PRIMARY KEY AUTOINCREMENT, classname VARCHAR );') def createNeededTable(self): self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER );') self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY AUTOINCREMENT, library VARCHAR );') def createSystemPackagesTable(self): self.cursor.execute('CREATE TABLE systempackages ( idpackage INTEGER PRIMARY KEY );') def createInjectedTable(self): self.cursor.execute('CREATE TABLE injected ( idpackage INTEGER PRIMARY KEY );') def createProtectTable(self): self.cursor.execute('DROP TABLE IF EXISTS configprotect;') self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;') self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;') self.cursor.execute('CREATE TABLE configprotect ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );') self.cursor.execute('CREATE TABLE configprotectmask ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );') self.cursor.execute('CREATE TABLE configprotectreference ( idprotect INTEGER PRIMARY KEY AUTOINCREMENT, protect VARCHAR );') def createInstalledTable(self): self.cursor.execute('DROP TABLE IF EXISTS installedtable;') self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR );') def addDependRelationToDependsTable(self, iddependency, idpackage): self.cursor.execute( 'INSERT into dependstable VALUES ' '(?,?)' , ( iddependency, idpackage, ) ) if (entropyTools.is_user_in_entropy_group()) and (self.dbname == etpConst['serverdbid']): # force commit even if readonly, this will allow to automagically fix dependstable server side self.connection.commit() # we don't care much about syncing the database since it's quite trivial ''' @description: recreate dependstable table in the chosen database, it's used for caching searchDepends requests @input Nothing @output: Nothing ''' def regenerateDependsTable(self, output = True): self.createDependsTable() depends = self.listAllDependencies() count = 0 total = len(depends) for depend in depends: count += 1 atom = depend[1] iddep = depend[0] if output: self.updateProgress( red("Resolving %s") % (darkgreen(atom),), importance = 0, type = "info", back = True, count = (count,total) ) match = self.atomMatch(atom) if (match[0] != -1): self.addDependRelationToDependsTable(iddep,match[0]) del depends # now validate dependstable self.sanitizeDependsTable() ######################################################## #### ## Dependency handling functions # def __get_match_hash(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): c_hash = str(hash(atom)) + \ str(hash(matchSlot)) + \ str(hash(matchTag)) + \ str(hash(matchRevision)) + \ str(hash(tuple(matchBranches))) + \ str(hash(caseSensitive)) + \ str(hash(multiMatch)) + \ str(hash(packagesFilter)) + \ str(hash(matchRevision)) #c_hash = str(hash(c_hash)) return c_hash def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): if self.xcache: c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) try: cached = dumpTools.loadobj(etpCache['dbMatch']+"/"+self.dbname+"/"+c_hash) if cached != None: return cached except EOFError, IOError: return None def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision): if self.xcache: c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) try: sperms = False if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbMatch']+"/"+self.dbname)): sperms = True dumpTools.dumpobj(etpCache['dbMatch']+"/"+self.dbname+"/"+c_hash,result) if sperms: const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid']) except IOError: pass # function that validate one atom by reading keywords settings # idpackageValidatorCache = {} >> function cache def idpackageValidator(self,idpackage): reponame = self.dbname[5:] cached = idpackageValidatorCache.get((idpackage,reponame)) if cached != None: return cached # check if user package.mask needs it masked user_package_mask_ids = etpConst['packagemasking'].get(reponame+'mask_ids') if user_package_mask_ids == None: etpConst['packagemasking'][reponame+'mask_ids'] = set() for atom in etpConst['packagemasking']['mask']: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking'][reponame+'mask_ids'] |= matches[0] user_package_mask_ids = etpConst['packagemasking'][reponame+'mask_ids'] if idpackage in user_package_mask_ids: # sorry, masked idpackageValidatorCache[(idpackage,reponame)] = -1,1 return -1,1 # see if we can unmask by just lookin into user package.unmask stuff -> etpConst['packagemasking']['unmask'] user_package_unmask_ids = etpConst['packagemasking'].get(reponame+'unmask_ids') if user_package_unmask_ids == None: etpConst['packagemasking'][reponame+'unmask_ids'] = set() for atom in etpConst['packagemasking']['unmask']: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking'][reponame+'unmask_ids'] |= matches[0] user_package_unmask_ids = etpConst['packagemasking'][reponame+'unmask_ids'] if idpackage in user_package_unmask_ids: idpackageValidatorCache[(idpackage,reponame)] = idpackage,3 return idpackage,3 # check if repository packages.db.mask needs it masked repomask = etpConst['packagemasking']['repos_mask'].get(reponame) if repomask != None: # first, seek into generic masking, all branches all_branches_mask = repomask.get("*") if all_branches_mask: all_branches_mask_ids = repomask.get("*_ids") if all_branches_mask_ids == None: etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] = set() for atom in all_branches_mask: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] |= matches[0] all_branches_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] if idpackage in all_branches_mask_ids: idpackageValidatorCache[(idpackage,reponame)] = -1,8 return -1,8 # no universal mask branches_mask = repomask.get("branch") if branches_mask: for branch in branches_mask: branch_mask_ids = branches_mask.get(branch+"_ids") if branch_mask_ids == None: etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] = set() for atom in branches_mask[branch]: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] |= matches[0] branch_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] if idpackage in branch_mask_ids: if self.retrieveBranch(idpackage) == branch: idpackageValidatorCache[(idpackage,reponame)] = -1,9 return -1,9 if etpConst['packagemasking']['license_mask']: mylicenses = self.retrieveLicense(idpackage) mylicenses = mylicenses.strip().split() if mylicenses: for mylicense in mylicenses: if mylicense in etpConst['packagemasking']['license_mask']: idpackageValidatorCache[(idpackage,reponame)] = -1,10 return -1,10 mykeywords = self.retrieveKeywords(idpackage) # XXX WORKAROUND if not mykeywords: mykeywords = [''] # ** is fine then # firstly, check if package keywords are in etpConst['keywords'] (universal keywords have been merged from package.mask) for key in etpConst['keywords']: if key in mykeywords: # found! all fine idpackageValidatorCache[(idpackage,reponame)] = idpackage,2 return idpackage,2 # if we get here, it means we didn't find mykeywords in etpConst['keywords'], we need to seek etpConst['packagemasking']['keywords'] # seek in repository first if reponame in etpConst['packagemasking']['keywords']['repositories']: for keyword in etpConst['packagemasking']['keywords']['repositories'][reponame]: if keyword in mykeywords: keyword_data = etpConst['packagemasking']['keywords']['repositories'][reponame].get(keyword) if keyword_data: if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok idpackageValidatorCache[(idpackage,reponame)] = idpackage,4 #self.storeInfoCache(idpackage,'idpackageValidator',(idpackage,4)) return idpackage,4 keyword_data_ids = etpConst['packagemasking']['keywords']['repositories'][reponame].get(keyword+"_ids") if keyword_data_ids == None: etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"] = set() for atom in keyword_data: matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"] |= matches[0] keyword_data_ids = etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"] if idpackage in keyword_data_ids: idpackageValidatorCache[(idpackage,reponame)] = idpackage,5 return idpackage,5 # if we get here, it means we didn't find a match in repositories # so we scan packages, last chance for keyword in etpConst['packagemasking']['keywords']['packages']: # first of all check if keyword is in mykeywords if keyword in mykeywords: keyword_data = etpConst['packagemasking']['keywords']['packages'].get(keyword) # check for relation if keyword_data: keyword_data_ids = etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] if keyword_data_ids == None: etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] = set() for atom in keyword_data: # match atom matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False) if matches[1] != 0: continue etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] |= matches[0] keyword_data_ids = etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] if idpackage in keyword_data_ids: # valid! idpackageValidatorCache[(idpackage,reponame)] = idpackage,6 return idpackage,6 # holy crap, can't validate idpackageValidatorCache[(idpackage,reponame)] = -1,7 return -1,7 # packages filter used by atomMatch, input must me foundIDs, a list like this: # [(u'x11-libs/qt-4.3.2', 608), (u'x11-libs/qt-3.3.8-r4', 1867)] def packagesFilter(self,results, atom): # keywordsFilter ONLY FILTERS results if self.dbname.startswith(etpConst['dbnamerepoprefix']), repository database is open if not self.dbname.startswith(etpConst['dbnamerepoprefix']): return results newresults = set() for item in results: rc = self.idpackageValidator(item[1]) if rc[0] != -1: newresults.add(item) else: idreason = rc[1] if not maskingReasonsStorage.has_key(atom): maskingReasonsStorage[atom] = {} if not maskingReasonsStorage[atom].has_key(idreason): maskingReasonsStorage[atom][idreason] = set() maskingReasonsStorage[atom][idreason].add((item[1],self.dbname[5:])) return newresults def __filterSlot(self, idpackage, slot): if slot == None: return idpackage dbslot = self.retrieveSlot(idpackage) if str(dbslot) == str(slot): return idpackage def __filterTag(self, idpackage, tag, operators): if tag == None: return idpackage dbtag = self.retrieveVersionTag(idpackage) compare = cmp(tag,dbtag) if not operators or operators == "=": if compare == 0: return idpackage else: return self.__do_operator_compare(idpackage, operators, compare) def __do_operator_compare(self, token, operators, compare): if operators == ">" and compare == -1: return token elif operators == ">=" and compare < 1: return token elif operators == "<" and compare == 1: return token elif operators == "<=" and compare > -1: return token def __filterSlotTag(self, foundIDs, slot, tag, operators): newlist = set() for data in foundIDs: idpackage = data[1] idpackage = self.__filterSlot(idpackage, slot) if not idpackage: continue idpackage = self.__filterTag(idpackage, tag, operators) if not idpackage: continue newlist.add(data) return newlist ''' @description: matches the user chosen package name+ver, if possibile, in a single repository @input atom: string, atom to match @input caseSensitive: bool, should the atom be parsed case sensitive? @input matchSlot: string, match atoms with the provided slot @input multiMatch: bool, return all the available atoms @input matchBranches: tuple or list, match packages only in the specified branches @input matchTag: match packages only for the specified tag @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error ''' def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True, matchRevision = None): if not atom: return -1,1 cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter, matchRevision) if cached != None: return cached atomTag = entropyTools.dep_gettag(atom) atomSlot = entropyTools.dep_getslot(atom) atomRev = entropyTools.dep_get_entropy_revision(atom) # tag match scan_atom = entropyTools.remove_tag(atom) if (matchTag == None) and (atomTag != None): matchTag = atomTag # slot match scan_atom = entropyTools.remove_slot(scan_atom) if (matchSlot == None) and (atomSlot != None): matchSlot = atomSlot # revision match scan_atom = entropyTools.remove_entropy_revision(scan_atom) if (matchRevision == None) and (atomRev != None): matchRevision = atomRev # check for direction strippedAtom = entropyTools.dep_getcpv(scan_atom) if scan_atom[-1] == "*": strippedAtom += "*" direction = scan_atom[0:len(scan_atom)-len(strippedAtom)] justname = entropyTools.isjustname(strippedAtom) pkgversion = '' if not justname: # get version data = entropyTools.catpkgsplit(strippedAtom) if data == None: return -1,1 # atom is badly formatted pkgversion = data[2]+"-"+data[3] pkgkey = entropyTools.dep_getkey(strippedAtom) splitkey = pkgkey.split("/") if (len(splitkey) == 2): pkgname = splitkey[1] pkgcat = splitkey[0] else: pkgname = splitkey[0] pkgcat = "null" if (matchBranches): myBranchIndex = tuple(matchBranches) # force to tuple for security else: if (self.dbname == etpConst['clientdbid']): # collect all available branches myBranchIndex = tuple(self.listAllBranches()) elif self.dbname.startswith(etpConst['dbnamerepoprefix']): # repositories should match to any branch <= than the current if none specified allbranches = set([x for x in self.listAllBranches() if x <= etpConst['branch']]) allbranches = list(allbranches) allbranches.reverse() if etpConst['branch'] not in allbranches: allbranches.insert(0,etpConst['branch']) myBranchIndex = tuple(allbranches) else: myBranchIndex = (etpConst['branch'],) # IDs found in the database that match our search foundIDs = set() for idx in myBranchIndex: results = self.searchPackagesByName(pkgname, sensitive = caseSensitive, branch = idx) mypkgcat = pkgcat mypkgname = pkgname virtual = False # if it's a PROVIDE, search with searchProvide # there's no package with that name if (not results) and (mypkgcat == "virtual"): virtuals = self.searchProvide(pkgkey, branch = idx) if (virtuals): virtual = True mypkgname = self.retrieveName(virtuals[0][1]) mypkgcat = self.retrieveCategory(virtuals[0][1]) results = virtuals # now validate if (not results): continue # search into a stabler branch elif (len(results) > 1): # if it's because category differs, it's a problem foundCat = "" cats = set() for result in results: idpackage = result[1] cat = self.retrieveCategory(idpackage) cats.add(cat) if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): # in case of virtual packages only (that they're not stored as provide) foundCat = cat # if we found something at least... if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual","null")): foundCat = list(cats)[0] if (not foundCat): # got the issue continue # we can use foundCat mypkgcat = foundCat # we need to search using the category if (not multiMatch): results = self.searchPackagesByNameAndCategory(name = mypkgname, category = mypkgcat, branch = idx, sensitive = caseSensitive) # validate again if (not results): continue # search into another branch # if we get here, we have found the needed IDs foundIDs |= set(results) break else: # if mypkgcat is virtual, we can force if (mypkgcat == "virtual") and (not virtual): # in case of virtual packages only (that they're not stored as provide) mypkgcat = entropyTools.dep_getkey(results[0][0]).split("/")[0] # check if category matches if mypkgcat != "null": foundCat = self.retrieveCategory(results[0][1]) if mypkgcat == foundCat: foundIDs.add(results[0]) else: continue else: foundIDs.add(results[0]) break ### FILTERING ### FILTERING ### FILTERING # filter slot and tag foundIDs = self.__filterSlotTag(foundIDs, matchSlot, matchTag, direction) if packagesFilter: # keyword filtering foundIDs = self.packagesFilter(foundIDs, atom) ### END FILTERING ### END FILTERING ### END FILTERING if not foundIDs: # package not found self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) return -1,1 ### FILLING dbpkginfo ### FILLING dbpkginfo ### FILLING dbpkginfo dbpkginfo = set() # now we have to handle direction if (direction) or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")): if (not justname) and \ ((direction == "~") or (direction == "=") or \ (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*"))): # any revision within the version specified OR the specified version if (direction == '' and not justname): direction = "=" # remove gentoo revision (-r0 if none) if (direction == "="): if (pkgversion.split("-")[-1] == "r0"): pkgversion = entropyTools.remove_revision(pkgversion) if (direction == "~"): pkgrevision = entropyTools.dep_get_portage_revision(pkgversion) pkgversion = entropyTools.remove_revision(pkgversion) for data in foundIDs: idpackage = data[1] dbver = self.retrieveVersion(idpackage) if (direction == "~"): myrev = entropyTools.dep_get_portage_revision(dbver) myver = entropyTools.remove_revision(dbver) if myver == pkgversion and pkgrevision <= myrev: # found dbpkginfo.add((idpackage,dbver)) else: # media-libs/test-1.2* support if pkgversion[-1] == "*": if dbver.startswith(pkgversion[:-1]): dbpkginfo.add((idpackage,dbver)) elif (matchRevision != None) and (pkgversion == dbver): dbrev = self.retrieveRevision(idpackage) if dbrev == matchRevision: dbpkginfo.add((idpackage,dbver)) elif (pkgversion == dbver) and (matchRevision == None): dbpkginfo.add((idpackage,dbver)) elif (direction.find(">") != -1) or (direction.find("<") != -1): if not justname: # remove revision (-r0 if none) if pkgversion.endswith("r0"): # remove entropyTools.remove_revision(pkgversion) for data in foundIDs: idpackage = data[1] revcmp = 0 tagcmp = 0 if matchRevision != None: dbrev = self.retrieveRevision(idpackage) revcmp = cmp(matchRevision,dbrev) if matchTag != None: dbtag = self.retrieveVersionTag(idpackage) tagcmp = cmp(matchTag,dbtag) dbver = self.retrieveVersion(idpackage) pkgcmp = entropyTools.compareVersions(pkgversion,dbver) if type(pkgcmp) is tuple: failed = pkgcmp[1] if failed == 0: failed = pkgversion else: failed = dbver # I am sorry, but either pkgversion or dbver are invalid self.updateProgress( bold("atomMatch: ")+red("comparison between %s and %s failed. Wrong syntax for: %s") % (pkgversion,dbver,failed,), importance = 1, type = "error", header = darkred(" !!! ") ) raise exceptionTools.InvalidVersionString( "InvalidVersionString: from atom: %s, cmp: %s, failed: %s" % ( atom, pkgcmp, failed, ) ) if direction == ">": if pkgcmp < 0: dbpkginfo.add((idpackage,dbver)) elif (matchRevision != None) and pkgcmp <= 0 and revcmp < 0: #print "found >",self.retrieveAtom(idpackage) dbpkginfo.add((idpackage,dbver)) elif (matchTag != None) and tagcmp < 0: dbpkginfo.add((idpackage,dbver)) elif direction == "<": if pkgcmp > 0: dbpkginfo.add((idpackage,dbver)) elif (matchRevision != None) and pkgcmp >= 0 and revcmp > 0: #print "found <",self.retrieveAtom(idpackage) dbpkginfo.add((idpackage,dbver)) elif (matchTag != None) and tagcmp > 0: dbpkginfo.add((idpackage,dbver)) elif direction == ">=": if (matchRevision != None) and pkgcmp <= 0: if pkgcmp == 0: if revcmp <= 0: dbpkginfo.add((idpackage,dbver)) #print "found >=",self.retrieveAtom(idpackage) else: dbpkginfo.add((idpackage,dbver)) elif pkgcmp <= 0 and matchRevision == None: dbpkginfo.add((idpackage,dbver)) elif (matchTag != None) and tagcmp <= 0: dbpkginfo.add((idpackage,dbver)) elif direction == "<=": if (matchRevision != None) and pkgcmp >= 0: if pkgcmp == 0: if revcmp >= 0: dbpkginfo.add((idpackage,dbver)) #print "found <=",self.retrieveAtom(idpackage) else: dbpkginfo.add((idpackage,dbver)) elif pkgcmp >= 0 and matchRevision == None: dbpkginfo.add((idpackage,dbver)) elif (matchTag != None) and tagcmp >= 0: dbpkginfo.add((idpackage,dbver)) else: # just the key dbpkginfo = set([((x[1]),self.retrieveVersion(x[1])) for x in foundIDs]) ### END FILLING dbpkginfo ### END FILLING dbpkginfo ### END FILLING dbpkginfo if not dbpkginfo: self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) return -1,1 if multiMatch: x = set([x[0] for x in dbpkginfo]) self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) return x,0 if len(dbpkginfo) == 1: x = dbpkginfo.pop() self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) return x[0],0 dbpkginfo = list(dbpkginfo) pkgdata = {} versions = set() for x in dbpkginfo: info_tuple = (x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) versions.add(info_tuple) pkgdata[info_tuple] = x[0] newer = entropyTools.getEntropyNewerVersion(list(versions))[0] x = pkgdata[newer] self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision) return x,0