#!/usr/bin/python ''' # DESCRIPTION: # Entropy Object Oriented Interface Copyright (C) 2007-2008 Fabio Erculiani This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ''' from __future__ import with_statement import shutil, commands, urllib2, time, thread, copy, subprocess from entropyConstants import * from outputTools import TextInterface, \ print_info, print_warning, print_error, \ red, brown, blue, green, purple, darkgreen, \ darkred, bold, darkblue, readtext, nocolor import exceptionTools from entropy_i18n import _ try: # try with sqlite3 from python 2.5 - default one from sqlite3 import dbapi2 except ImportError: # fallback to embedded pysqlite try: from pysqlite2 import dbapi2 except ImportError, e: raise exceptionTools.SystemError( "%s. %s: %s" % ( _("Entropy needs a working sqlite+pysqlite or Python compiled with sqlite support"), _("Error"), e, ) ) class urlFetcher: def __init__(self, url, path_to_save, checksum = True, show_speed = True, resume = True, abort_check_func = None, disallow_redirect = False): import entropyTools, socket self.entropyTools, self.socket = entropyTools, socket self.url = url self.resume = resume self.url = self.encode_url(self.url) self.path_to_save = path_to_save self.checksum = checksum self.show_speed = show_speed self.init_vars() self.progress = None self.abort_check_func = abort_check_func self.disallow_redirect = disallow_redirect uname = os.uname() self.user_agent = "Entropy/%s (compatible; %s; %s: %s %s %s)" % ( etpConst['entropyversion'], "Entropy", os.path.basename(self.url), uname[0], uname[4], uname[2], ) self.extra_header_data = {} self.setup_resume_support() self.setup_proxy() def setup_resume_support(self): # resume support if os.path.isfile(self.path_to_save) and os.access(self.path_to_save,os.R_OK) and self.resume: self.localfile = open(self.path_to_save,"awb") self.localfile.seek(0,2) self.startingposition = int(self.localfile.tell()) self.resumed = True else: if os.path.lexists(self.path_to_save) and not self.entropyTools.is_valid_path(self.path_to_save): try: os.remove(self.path_to_save) except OSError: # I won't stop you here pass self.localfile = open(self.path_to_save,"wb") def setup_proxy(self): # setup proxy, doing here because config is dynamic mydict = {} if etpConst['proxy']['ftp']: mydict['ftp'] = etpConst['proxy']['ftp'] if etpConst['proxy']['http']: mydict['http'] = etpConst['proxy']['http'] if mydict: mydict['username'] = etpConst['proxy']['username'] mydict['password'] = etpConst['proxy']['password'] self.entropyTools.add_proxy_opener(urllib2, mydict) else: # unset urllib2._opener = None def encode_url(self, url): import urllib url = os.path.join(os.path.dirname(url),urllib.quote(os.path.basename(url))) return url def init_vars(self): self.resumed = False self.bufferSize = 8192 self.status = None self.remotefile = None self.downloadedsize = 0 self.average = 0 self.remotesize = 0 self.oldaverage = 0.0 # transfer status data self.startingposition = 0 self.datatransfer = 0 self.time_remaining = "(infinite)" self.elapsed = 0.0 self.updatestep = 0.2 self.speedlimit = etpConst['downloadspeedlimit'] # kbytes/sec self.transferpollingtime = float(1)/4 def download(self): self.init_vars() self.speedUpdater = self.entropyTools.TimeScheduled( self.update_speed, self.transferpollingtime ) self.speedUpdater.start() # set timeout self.socket.setdefaulttimeout(20) if self.url.startswith("http://"): headers = { 'User-Agent' : self.user_agent } req = urllib2.Request(self.url, self.extra_header_data, headers) else: req = self.url u_agent_error = False while 1: # get file size if available try: self.remotefile = urllib2.urlopen(req) except KeyboardInterrupt: self.close() raise except urllib2.HTTPError, e: if (e.code == 405) and not u_agent_error: # server doesn't like our user agent req = self.url u_agent_error = True continue self.close() self.status = "-3" return self.status except: self.close() self.status = "-3" return self.status break try: self.remotesize = int(self.remotefile.headers.get("content-length")) self.remotefile.close() except KeyboardInterrupt: self.close() raise except: pass # handle user stupidity try: request = self.url if ((self.startingposition > 0) and (self.remotesize > 0)) and (self.startingposition < self.remotesize): try: request = urllib2.Request( self.url, headers = { "Range" : "bytes=" + str(self.startingposition) + "-" + str(self.remotesize) } ) except KeyboardInterrupt: self.close() raise except: pass elif (self.startingposition == self.remotesize): return self.prepare_return() else: self.localfile = open(self.path_to_save,"wb") self.remotefile = urllib2.urlopen(request) except KeyboardInterrupt: self.close() raise except: self.close() self.status = "-3" return self.status if self.remotesize > 0: self.remotesize = float(int(self.remotesize))/1024 if self.disallow_redirect and (self.url != self.remotefile.geturl()): self.close() self.status = "-3" return self.status while 1: try: rsx = self.remotefile.read(self.bufferSize) if rsx == '': break if self.abort_check_func != None: self.abort_check_func() except KeyboardInterrupt: self.close() raise except: # python 2.4 timeouts go here self.close() self.status = "-3" return self.status self.commit(rsx) if self.show_speed: self.updateProgress() self.oldaverage = self.average if self.speedlimit: while self.datatransfer > self.speedlimit*1024: time.sleep(0.1) if self.show_speed: self.updateProgress() self.oldaverage = self.average # kill thread self.close() return self.prepare_return() def prepare_return(self): if self.checksum: self.status = self.entropyTools.md5sum(self.path_to_save) return self.status else: self.status = "-2" return self.status def commit(self, mybuffer): # writing file buffer self.localfile.write(mybuffer) # update progress info self.downloadedsize = self.localfile.tell() kbytecount = float(self.downloadedsize)/1024 self.average = int((kbytecount/self.remotesize)*100) def close(self): try: self.localfile.flush() self.localfile.close() except: pass try: self.remotefile.close() except: pass self.speedUpdater.kill() self.socket.setdefaulttimeout(2) def update_speed(self): self.elapsed += self.transferpollingtime # we have the diff size self.datatransfer = (self.downloadedsize-self.startingposition) / self.elapsed try: self.time_remaining = int(round((int(round(self.remotesize*1024,0))-int(round(self.downloadedsize,0)))/self.datatransfer,0)) self.time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.time_remaining) except: self.time_remaining = "(%s)" % (_("infinite"),) def updateProgress(self): mytxt = _("[F]") eta_txt = _("ETA") sec_txt = _("sec") # as in XX kb/sec currentText = darkred(" %s: " % (mytxt,)) + \ darkgreen(str(round(float(self.downloadedsize)/1024,1))) + "/" + \ red(str(round(self.remotesize,1))) + " kB" # create progress bar barsize = 10 bartext = "[" curbarsize = 1 if self.average > self.oldaverage+self.updatestep: averagesize = (self.average*barsize)/100 while averagesize > 0: curbarsize += 1 bartext += "=" averagesize -= 1 bartext += ">" diffbarsize = barsize-curbarsize while diffbarsize > 0: bartext += " " diffbarsize -= 1 if self.show_speed: bartext += "] => %s" % (self.entropyTools.bytesIntoHuman(self.datatransfer),) bartext += "/%s : %s: %s" % (sec_txt,eta_txt,self.time_remaining,) else: bartext += "]" average = str(self.average) if len(average) < 2: average = " "+average currentText += " <-> "+average+"% "+bartext print_info(currentText,back = True) class EntropyCacher: import entropyTools import dumpTools import threading def __init__(self): self.alive = False self.CacheBuffer = self.entropyTools.lifobuffer() self.CacheLock = self.threading.Lock() def start(self): self.CacheWriter = self.entropyTools.TimeScheduled(self.Cacher,1) self.CacheWriter.delay_before = True self.CacheWriter.start() while not self.CacheWriter.isAlive(): continue self.alive = True def sync(self, wait = False): if not self.alive: return wd = 10 while self.CacheBuffer.is_filled() and ((wd > 0) or wait): if not wait: wd -= 1 time.sleep(0.5) def push(self, key, data, async = True): if not self.alive: return if async: with self.CacheLock: self.CacheBuffer.push((key,data,)) else: self.dumpTools.dumpobj(key,data) def pop(self, key): with self.CacheLock: l_o = self.dumpTools.loadobj if not l_o: return return l_o(key) def Cacher(self): while 1: if not self.alive: break with self.CacheLock: data = self.CacheBuffer.pop() if data == None: break key, data = data d_o = self.dumpTools.dumpobj if not d_o: break d_o(key,data) def __del__(self): self.stop() def stop(self): if not self.alive: return if hasattr(self,"CacheBuffer"): if self.CacheBuffer and self.alive: wd = 20 while self.CacheBuffer.is_filled() and wd: wd -= 1 time.sleep(0.5) self.CacheBuffer.clear() self.alive = False if hasattr(self,"CacheWriter"): self.CacheWriter.kill() ''' Main Entropy (client side) package management class ''' class EquoInterface(TextInterface): def __init__(self, indexing = True, noclientdb = 0, xcache = True, user_xcache = False, repo_validation = True, load_ugc = True, url_fetcher = urlFetcher): # modules import import dumpTools, entropyTools self.dumpTools = dumpTools self.entropyTools = entropyTools self.atomMatchCacheKey = etpCache['atomMatch'] self.dbapi2 = dbapi2 # export for third parties self.FileUpdates = None self.repoDbCache = {} self.securityCache = {} self.QACache = {} self.spmCache = {} self.mirrorDownloadFailures = {} self.repo_error_messages_cache = set() self.package_match_validator_cache = {} self.memoryDbInstances = {} self.validRepositories = [] self.clientLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[client]") self.urlFetcher = url_fetcher # in this way, can be reimplemented (so you can override updateProgress) self.progress = None # supporting external updateProgress stuff, you can point self.progress to your progress bar # and reimplement updateProgress self.clientDbconn = None self.safe_mode = 0 self.FtpInterface = FtpInterface # for convenience self.indexing = indexing self.repo_validation = repo_validation self.noclientdb = False self.openclientdb = True if noclientdb in (False,0): self.noclientdb = False elif noclientdb in (True,1): self.noclientdb = True elif noclientdb == 2: self.noclientdb = True self.openclientdb = False self.xcache = xcache shell_xcache = os.getenv("ETP_NOCACHE") if shell_xcache: self.xcache = False self.Cacher = EntropyCacher() do_validate_repo_cache = False # now if we are on live, we should disable it # are we running on a livecd? (/proc/cmdline has "cdroot") if self.entropyTools.islive(): self.xcache = False elif (not self.entropyTools.is_user_in_entropy_group()) and not user_xcache: self.xcache = False elif not user_xcache: do_validate_repo_cache = True if not self.xcache and (self.entropyTools.is_user_in_entropy_group()): try: self.purge_cache(False) except: pass if self.openclientdb: self.openClientDatabase() self.FileUpdates = self.FileUpdatesInterfaceLoader() # setup package settings (masking and other stuff) self.SystemSettings = SystemSettings(self) # settle settings, especially package masking ones # useful for third party softwares that use this interface directly self.SystemSettings.scan() # needs to be started here otherwise repository cache will be # always dropped, trust me, it will be stopped later in the init # if xcache is disabled. self.Cacher.start() if do_validate_repo_cache: self.validate_repositories_cache() if self.repo_validation: self.validate_repositories() # load User Generated Content Interface self.UGC = None if load_ugc: self.UGC = UGCClientInterface(self) def destroy(self): if hasattr(self,'clientDbconn'): if self.clientDbconn != None: self.clientDbconn.closeDB() del self.clientDbconn if hasattr(self,'FileUpdates'): del self.FileUpdates if hasattr(self,'clientLog'): self.clientLog.close() if hasattr(self,'Cacher'): self.Cacher.stop() self.closeAllRepositoryDatabases(mask_clear = False) self.closeAllSecurity() self.closeAllQA() def __del__(self): self.destroy() def reload_constants(self): initConfig_entropyConstants(etpSys['rootdir']) initConfig_clientConstants() def validate_repositories(self): self.mirrorDownloadFailures.clear() self.repo_error_messages_cache.clear() self.package_match_validator_cache.clear() # valid repositories del self.validRepositories[:] for repoid in etpRepositoriesOrder: # open database try: dbc = self.openRepositoryDatabase(repoid) dbc.listConfigProtectDirectories() dbc.validateDatabase() self.validRepositories.append(repoid) except exceptionTools.RepositoryError: t = _("Repository") + " " + repoid + " " + _("is not available") + ". " + _("Cannot validate") t2 = _("Please update your repositories now in order to remove this message!") self.updateProgress( darkred(t), importance = 1, type = "warning" ) self.updateProgress( purple(t2), header = bold("!!! "), importance = 1, type = "warning" ) continue # repo not available except (dbapi2.OperationalError,dbapi2.DatabaseError,exceptionTools.SystemDatabaseError,): t = _("Repository") + " " + repoid + " " + _("is corrupted") + ". " + _("Cannot validate") self.updateProgress( darkred(t), importance = 1, type = "warning" ) continue # to avoid having zillions of open files when loading a lot of EquoInterfaces self.closeAllRepositoryDatabases(mask_clear = False) def init_generic_memory_repository(self, repoid, description, package_mirrors = []): dbc = self.openMemoryDatabase(dbname = repoid) self.memoryDbInstances[repoid] = dbc # add to etpRepositories repodata = { 'repoid': repoid, 'in_memory': True, 'description': description, 'packages': package_mirrors, 'dbpath': ':memory:', } self.addRepository(repodata) return dbc def setup_default_file_perms(self, filepath): # setup file permissions os.chmod(filepath,0664) if etpConst['entropygid'] != None: os.chown(filepath,-1,etpConst['entropygid']) def _resources_run_create_lock(self): self.create_pid_file_lock(etpConst['locks']['using_resources']) def _resources_run_remove_lock(self): if os.path.isfile(etpConst['locks']['using_resources']): os.remove(etpConst['locks']['using_resources']) def _resources_run_check_lock(self): rc = self.check_pid_file_lock(etpConst['locks']['using_resources']) return rc def check_pid_file_lock(self, pidfile): if not os.path.isfile(pidfile): return False # not locked f = open(pidfile) s_pid = f.readline().strip() f.close() try: s_pid = int(s_pid) except ValueError: return False # not locked # is it our pid? mypid = os.getpid() if (s_pid != mypid) and os.path.isdir("%s/proc/%s" % (etpConst['systemroot'],s_pid,)): # is it running return True # locked return False def create_pid_file_lock(self, pidfile, mypid = None): lockdir = os.path.dirname(pidfile) if not os.path.isdir(lockdir): os.makedirs(lockdir,0775) const_setup_perms(lockdir,etpConst['entropygid']) if mypid == None: mypid = os.getpid() f = open(pidfile,"w") f.write(str(mypid)) f.flush() f.close() def application_lock_check(self, silent = False): # check if another instance is running etpConst['applicationlock'] = False const_setupEntropyPid(just_read = True) locked = self.entropyTools.applicationLockCheck(option = None, gentle = True, silent = True) if locked: if not silent: self.updateProgress( red(_("Another Entropy instance is currently active, cannot satisfy your request.")), importance = 1, type = "error", header = darkred(" @@ ") ) return True return False def lock_check(self, check_function): lock_count = 0 max_lock_count = 600 sleep_seconds = 0.5 # check lock file while 1: locked = check_function() if not locked: if lock_count > 0: self.updateProgress( blue(_("Resources unlocked, let's go!")), importance = 1, type = "info", header = darkred(" @@ ") ) break if lock_count >= max_lock_count: mycalc = max_lock_count*sleep_seconds/60 self.updateProgress( blue(_("Resources still locked after %s minutes, giving up!")) % (mycalc,), importance = 1, type = "warning", header = darkred(" @@ ") ) return True # gave up lock_count += 1 self.updateProgress( blue(_("Resources locked, sleeping %s seconds, check #%s/%s")) % ( sleep_seconds, lock_count, max_lock_count, ), importance = 1, type = "warning", header = darkred(" @@ "), back = True ) time.sleep(sleep_seconds) return False # yay! def validate_repositories_cache(self): # is the list of repos changed? cached = self.Cacher.pop(etpCache['repolist']) if cached == None: # invalidate matching cache try: self.repository_move_clear_cache() except IOError: pass elif isinstance(cached,tuple): difflist = [x for x in cached if x not in etpRepositoriesOrder] for repoid in difflist: try: self.repository_move_clear_cache(repoid) except IOError: pass self.store_repository_list_cache() def store_repository_list_cache(self): self.Cacher.push(etpCache['repolist'],tuple(etpRepositoriesOrder)[:], async = False) def backup_setting(self, setting_name): if etpConst.has_key(setting_name): myinst = etpConst[setting_name] if type(etpConst[setting_name]) in (list,tuple): myinst = etpConst[setting_name][:] elif type(etpConst[setting_name]) in (dict,set): myinst = etpConst[setting_name].copy() else: myinst = etpConst[setting_name] etpConst['backed_up'].update({setting_name: myinst}) else: t = _("Nothing to backup in etpConst with %s key") % (setting_name,) raise exceptionTools.InvalidData("InvalidData: %s" % (t,)) def set_priority(self, low = 0): return const_setNiceLevel(low) def reloadRepositoriesConfigProtect(self, repositories = None): if repositories == None: repositories = self.validRepositories for repoid in repositories: dbconn = self.openRepositoryDatabase(repoid) self.loadRepositoryConfigProtect(repoid, dbconn) def switchChroot(self, chroot = ""): # clean caches self.purge_cache() self.closeAllRepositoryDatabases() if chroot.endswith("/"): chroot = chroot[:-1] etpSys['rootdir'] = chroot self.reload_constants() self.validate_repositories() self.reopenClientDbconn() if chroot: try: self.clientDbconn.resetTreeupdatesDigests() except: pass # I don't think it's safe to keep them open # isn't it? self.closeAllSecurity() self.closeAllQA() def Security(self): chroot = etpConst['systemroot'] cached = self.securityCache.get(chroot) if cached != None: return cached cached = SecurityInterface(self) self.securityCache[chroot] = cached return cached def QA(self): chroot = etpConst['systemroot'] cached = self.QACache.get(chroot) if cached != None: return cached cached = QAInterface(self) self.QACache[chroot] = cached return cached def closeAllQA(self): self.QACache.clear() def closeAllSecurity(self): self.securityCache.clear() def reopenClientDbconn(self): self.clientDbconn.closeDB() self.openClientDatabase() def closeAllRepositoryDatabases(self, mask_clear = True): for item in self.repoDbCache: self.repoDbCache[item].closeDB() self.repoDbCache.clear() if mask_clear: self.SystemSettings.clear() def openClientDatabase(self): def load_db_from_ram(): self.safe_mode = etpConst['safemodeerrors']['clientdb'] mytxt = "%s, %s" % (_("System database not found or corrupted"),_("running in safe mode using empty database from RAM"),) self.updateProgress( darkred(mytxt), importance = 1, type = "warning", header = bold("!!!"), ) conn = self.openMemoryDatabase(dbname = etpConst['clientdbid']) return conn if not os.path.isdir(os.path.dirname(etpConst['etpdatabaseclientfilepath'])): os.makedirs(os.path.dirname(etpConst['etpdatabaseclientfilepath'])) if (not self.noclientdb) and (not os.path.isfile(etpConst['etpdatabaseclientfilepath'])): conn = load_db_from_ram() self.entropyTools.printTraceback(f = self.clientLog) else: conn = EntropyDatabaseInterface( readOnly = False, dbFile = etpConst['etpdatabaseclientfilepath'], clientDatabase = True, dbname = etpConst['clientdbid'], xcache = self.xcache, indexing = self.indexing, OutputInterface = self, ServiceInterface = self ) # validate database if not self.noclientdb: try: conn.validateDatabase() except exceptionTools.SystemDatabaseError: try: conn.closeDB() except: pass self.entropyTools.printTraceback(f = self.clientLog) conn = load_db_from_ram() if not etpConst['dbconfigprotect']: if conn.doesTableExist('configprotect') and conn.doesTableExist('configprotectreference'): etpConst['dbconfigprotect'] = conn.listConfigProtectDirectories() if conn.doesTableExist('configprotectmask') and conn.doesTableExist('configprotectreference'): etpConst['dbconfigprotectmask'] = conn.listConfigProtectDirectories(mask = True) etpConst['dbconfigprotect'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] etpConst['dbconfigprotectmask'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] etpConst['dbconfigprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpConst['dbconfigprotect']] etpConst['dbconfigprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpConst['dbconfigprotectmask']] self.clientDbconn = conn return self.clientDbconn def clientDatabaseSanityCheck(self): self.updateProgress( darkred(_("Sanity Check") + ": " + _("system database")), importance = 2, type = "warning" ) idpkgs = self.clientDbconn.listAllIdpackages() length = len(idpkgs) count = 0 errors = False scanning_txt = _("Scanning...") for x in idpkgs: count += 1 self.updateProgress( darkgreen(scanning_txt), importance = 0, type = "info", back = True, count = (count,length), percent = True ) try: self.clientDbconn.getPackageData(x) except Exception ,e: self.entropyTools.printTraceback() errors = True self.updateProgress( darkred(_("Errors on idpackage %s, error: %s")) % (x,str(e)), importance = 0, type = "warning" ) if not errors: t = _("Sanity Check") + ": %s" % (bold(_("PASSED")),) self.updateProgress( darkred(t), importance = 2, type = "warning" ) return 0 else: t = _("Sanity Check") + ": %s" % (bold(_("CORRUPTED")),) self.updateProgress( darkred(t), importance = 2, type = "warning" ) return -1 def openRepositoryDatabase(self, repoid): t_ident = 1 # thread.get_ident() disabled for now if not self.repoDbCache.has_key((repoid,etpConst['systemroot'],t_ident,)): dbconn = self.loadRepositoryDatabase(repoid, xcache = self.xcache, indexing = self.indexing) try: dbconn.checkDatabaseApi() except: pass self.repoDbCache[(repoid,etpConst['systemroot'],t_ident,)] = dbconn return dbconn else: return self.repoDbCache.get((repoid,etpConst['systemroot'],t_ident,)) def is_installed_idpackage_in_system_mask(self, idpackage): if idpackage in self.SystemSettings['repos_system_mask_installed']: return True return False ''' @description: open the repository database @input repositoryName: name of the client database @input xcache: loads on-disk cache @input indexing: indexes SQL tables @output: database class instance NOTE: DO NOT USE THIS DIRECTLY, BUT USE EquoInterface.openRepositoryDatabase ''' def loadRepositoryDatabase(self, repoid, xcache = True, indexing = True): if isinstance(repoid,basestring): if repoid.endswith(etpConst['packagesext']): xcache = False if repoid not in etpRepositories: t = _("bad repository id specified") if repoid not in self.repo_error_messages_cache: self.updateProgress( darkred(t), importance = 2, type = "warning" ) self.repo_error_messages_cache.add(repoid) raise exceptionTools.RepositoryError("RepositoryError: %s" % (t,)) dbfile = etpRepositories[repoid]['dbpath']+"/"+etpConst['etpdatabasefile'] if not os.path.isfile(dbfile): t = _("Repository %s hasn't been downloaded yet.") % (repoid,) if repoid not in self.repo_error_messages_cache: self.updateProgress( darkred(t), importance = 2, type = "warning" ) self.repo_error_messages_cache.add(repoid) raise exceptionTools.RepositoryError("RepositoryError: %s" % (t,)) conn = EntropyDatabaseInterface( readOnly = True, dbFile = dbfile, clientDatabase = True, dbname = etpConst['dbnamerepoprefix']+repoid, xcache = xcache, indexing = indexing, OutputInterface = self, ServiceInterface = self ) # initialize CONFIG_PROTECT if (etpRepositories[repoid]['configprotect'] == None) or \ (etpRepositories[repoid]['configprotectmask'] == None): self.loadRepositoryConfigProtect(repoid, conn) if (repoid not in etpConst['client_treeupdatescalled']) and (self.entropyTools.is_user_in_entropy_group()) and (not repoid.endswith(etpConst['packagesext'])): updated = False try: updated = conn.clientUpdatePackagesData(self.clientDbconn) except (dbapi2.OperationalError, dbapi2.DatabaseError): pass if updated: self.clear_dump_cache(etpCache['world_update']) self.clear_dump_cache(etpCache['world']) self.clear_dump_cache(etpCache['install']) self.clear_dump_cache(etpCache['remove']) self.calculate_world_updates(use_cache = False) return conn def loadRepositoryConfigProtect(self, repoid, dbconn): try: etpRepositories[repoid]['configprotect'] = dbconn.listConfigProtectDirectories() except (dbapi2.OperationalError, dbapi2.DatabaseError): etpRepositories[repoid]['configprotect'] = [] try: etpRepositories[repoid]['configprotectmask'] = dbconn.listConfigProtectDirectories(mask = True) except (dbapi2.OperationalError, dbapi2.DatabaseError): etpRepositories[repoid]['configprotectmask'] = [] etpRepositories[repoid]['configprotect'] = [etpConst['systemroot']+x for x in etpRepositories[repoid]['configprotect']] etpRepositories[repoid]['configprotectmask'] = [etpConst['systemroot']+x for x in etpRepositories[repoid]['configprotectmask']] etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotect']] etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotectmask']] def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False, indexing_override = None, skipChecks = False): if xcache == None: xcache = self.xcache if indexing_override != None: indexing = indexing_override else: indexing = self.indexing if dbname == None: dbname = etpConst['genericdbid'] return EntropyDatabaseInterface( readOnly = readOnly, dbFile = dbfile, clientDatabase = True, dbname = dbname, xcache = xcache, indexing = indexing, OutputInterface = self, skipChecks = skipChecks ) def openMemoryDatabase(self, dbname = None): if dbname == None: dbname = etpConst['genericdbid'] dbc = EntropyDatabaseInterface( readOnly = False, dbFile = ':memory:', clientDatabase = True, dbname = dbname, xcache = False, indexing = False, OutputInterface = self, skipChecks = True ) dbc.initializeDatabase() return dbc def backupDatabase(self, dbpath, backup_dir = None, silent = False, compress_level = 9): if compress_level not in range(1,10): compress_level = 9 backup_dir = os.path.dirname(dbpath) if not backup_dir: backup_dir = os.path.dirname(dbpath) dbname = os.path.basename(dbpath) bytes_required = 1024000*300 if not (os.access(backup_dir,os.W_OK) and \ os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \ os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)): if not silent: mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),) self.updateProgress( mytxt, importance = 1, type = "error", header = red(" @@ ") ) return False, mytxt def get_ts(): from datetime import datetime ts = datetime.fromtimestamp(time.time()) return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second) comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),) comp_dbpath = os.path.join(backup_dir,comp_dbname) if not silent: mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),) self.updateProgress( mytxt, importance = 1, type = "info", header = blue(" @@ "), back = True ) import bz2 try: self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) except: if not silent: self.entropyTools.printTraceback() return False, _("Unable to compress") if not silent: mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),) self.updateProgress( mytxt, importance = 1, type = "info", header = blue(" @@ "), back = True ) return True, _("All fine") def restoreDatabase(self, backup_path, db_destination, silent = False): bytes_required = 1024000*300 if not (os.access(os.path.dirname(db_destination),os.W_OK) and \ os.path.isdir(os.path.dirname(db_destination)) and os.path.isfile(backup_path) and \ os.access(backup_path,os.R_OK) and self.entropyTools.check_required_space(os.path.dirname(db_destination), bytes_required)): if not silent: mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")),blue(backup_path),darkred(_("permission denied")),) self.updateProgress( mytxt, importance = 1, type = "error", header = red(" @@ ") ) return False, mytxt if not silent: mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")),blue(os.path.basename(backup_path)),blue(db_destination),) self.updateProgress( mytxt, importance = 1, type = "info", header = blue(" @@ "), back = True ) import bz2 try: self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File) except: if not silent: self.entropyTools.printTraceback() return False, _("Unable to unpack") if not silent: mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")),blue(db_destination),) self.updateProgress( mytxt, importance = 1, type = "info", header = blue(" @@ "), back = True ) self.purge_cache() return True, _("All fine") def list_backedup_client_databases(self, client_dbdir = None): if not client_dbdir: client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \ if x.startswith(etpConst['dbbackupprefix']) and \ os.access(os.path.join(client_dbdir,x),os.R_OK) ] def get_branch_from_download_relative_uri(self, db_download_uri): return db_download_uri.split("/")[2] ''' Cache stuff :: begin ''' def purge_cache(self, showProgress = True, client_purge = True): if self.entropyTools.is_user_in_entropy_group(): skip = set() if not client_purge: skip.add("/"+etpCache['dbMatch']+"/"+etpConst['clientdbid']) # it's ok this way skip.add("/"+etpCache['dbSearch']+"/"+etpConst['clientdbid']) # it's ok this way for key in etpCache: if showProgress: self.updateProgress( darkred(_("Cleaning %s => dumps...")) % (etpCache[key],), importance = 1, type = "warning", back = True ) self.clear_dump_cache(etpCache[key], skip = skip) if showProgress: self.updateProgress( darkgreen(_("Cache is now empty.")), importance = 2, type = "info" ) def generate_cache(self, depcache = True, configcache = True, client_purge = True, install_queue = True): self.Cacher.stop() # clean first of all self.purge_cache(client_purge = client_purge) if depcache: self.do_depcache(do_install_queue = install_queue) if configcache: self.do_configcache() self.Cacher.start() def do_configcache(self): self.updateProgress( darkred(_("Configuration files")), importance = 2, type = "warning" ) self.updateProgress( red(_("Scanning hard disk")), importance = 1, type = "warning" ) self.FileUpdates.scanfs(dcache = False, quiet = True) self.updateProgress( darkred(_("Cache generation complete.")), importance = 2, type = "info" ) def do_depcache(self, do_install_queue = True): self.updateProgress( darkgreen(_("Resolving metadata")), importance = 1, type = "warning" ) # we can barely ignore any exception from here # especially cases where client db does not exist try: update, remove, fine = self.calculate_world_updates() del fine, remove if do_install_queue: self.retrieveInstallQueue(update, False, False) self.calculate_available_packages() except: pass self.updateProgress( darkred(_("Dependencies cache filled.")), importance = 2, type = "warning" ) def clear_dump_cache(self, dump_name, skip = []): self.Cacher.sync(wait = True) dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) dump_dir = os.path.dirname(dump_path) #dump_file = os.path.basename(dump_path) for currentdir, subdirs, files in os.walk(dump_dir): path = os.path.join(dump_dir,currentdir) if skip: found = False for myskip in skip: if path.find(myskip) != -1: found = True break if found: continue for item in files: if item.endswith(etpConst['cachedumpext']): item = os.path.join(path,item) try: os.remove(item) except OSError: pass if not os.listdir(path): os.rmdir(path) ''' Cache stuff :: end ''' def unused_packages_test(self, dbconn = None): if dbconn == None: dbconn = self.clientDbconn return [x for x in dbconn.retrieveUnusedIdpackages() if self.validatePackageRemoval(x)] def dependencies_test(self, dbconn = None): if dbconn == None: dbconn = self.clientDbconn # get all the installed packages installedPackages = dbconn.listAllIdpackages() deps_not_matched = set() # now look length = len(installedPackages) count = 0 for xidpackage in installedPackages: count += 1 atom = dbconn.retrieveAtom(xidpackage) self.updateProgress( darkgreen(_("Checking %s") % (bold(atom),)), importance = 0, type = "info", back = True, count = (count,length), header = darkred(" @@ ") ) xdeps = dbconn.retrieveDependencies(xidpackage) needed_deps = set() for xdep in xdeps: xmatch = dbconn.atomMatch(xdep) if xmatch[0] == -1: needed_deps.add(xdep) deps_not_matched |= needed_deps return deps_not_matched def find_belonging_dependency(self, matched_atoms): crying_atoms = set() for atom in matched_atoms: for repo in self.validRepositories: rdbconn = self.openRepositoryDatabase(repo) riddep = rdbconn.searchDependency(atom) if riddep != -1: ridpackages = rdbconn.searchIdpackageFromIddependency(riddep) for i in ridpackages: i,r = rdbconn.idpackageValidator(i) if i == -1: continue iatom = rdbconn.retrieveAtom(i) crying_atoms.add((iatom,repo)) return crying_atoms def get_licenses_to_accept(self, install_queue): if not install_queue: return {} licenses = {} for match in install_queue: repoid = match[1] dbconn = self.openRepositoryDatabase(repoid) wl = self.SystemSettings['repos_license_whitelist'].get(repoid) if not wl: continue keys = dbconn.retrieveLicensedataKeys(match[0]) for key in keys: if key not in wl: found = self.clientDbconn.isLicenseAccepted(key) if found: continue if not licenses.has_key(key): licenses[key] = set() licenses[key].add(match) return licenses def get_text_license(self, license_name, repoid): dbconn = self.openRepositoryDatabase(repoid) text = dbconn.retrieveLicenseText(license_name) tempfile = self.entropyTools.getRandomTempFile() f = open(tempfile,"w") f.write(text) f.flush() f.close() return tempfile def get_file_viewer(self): viewer = None if os.access("/usr/bin/less",os.X_OK): viewer = "/usr/bin/less" elif os.access("/bin/more",os.X_OK): viewer = "/bin/more" if not viewer: viewer = self.get_file_editor() return viewer def get_file_editor(self): editor = None if os.getenv("EDITOR"): editor = "$EDITOR" elif os.access("/bin/nano",os.X_OK): editor = "/bin/nano" elif os.access("/bin/vi",os.X_OK): editor = "/bin/vi" elif os.access("/usr/bin/vi",os.X_OK): editor = "/usr/bin/vi" elif os.access("/usr/bin/emacs",os.X_OK): editor = "/usr/bin/emacs" elif os.access("/bin/emacs",os.X_OK): editor = "/bin/emacs" return editor def libraries_test(self, dbconn = None, broken_symbols = False, task_bombing_func = None): if dbconn == None: dbconn = self.clientDbconn self.updateProgress( blue(_("Libraries test")), importance = 2, type = "info", header = red(" @@ ") ) if not etpConst['systemroot']: myroot = "/" else: myroot = etpConst['systemroot']+"/" # run ldconfig first subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) # open /etc/ld.so.conf if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"): self.updateProgress( blue(_("Cannot find "))+red(etpConst['systemroot']+"/etc/ld.so.conf"), importance = 1, type = "error", header = red(" @@ ") ) return {},set(),-1 ldpaths = set(self.entropyTools.collectLinkerPaths()) ldpaths |= self.entropyTools.collectPaths() # speed up when /usr/lib is a /usr/lib64 symlink if "/usr/lib64" in ldpaths and "/usr/lib" in ldpaths: if os.path.realpath("/usr/lib64") == "/usr/lib": ldpaths.discard("/usr/lib") # some crappy packages put shit here too ldpaths.add("/usr/share") # always force /usr/libexec too ldpaths.add("/usr/libexec") executables = set() total = len(ldpaths) count = 0 sys_root_len = len(etpConst['systemroot']) for ldpath in ldpaths: if callable(task_bombing_func): task_bombing_func() count += 1 self.updateProgress( blue("Tree: ")+red(etpConst['systemroot']+ldpath), importance = 0, type = "info", count = (count,total), back = True, percent = True, header = " " ) ldpath = ldpath.encode(sys.getfilesystemencoding()) mywalk_iter = os.walk(etpConst['systemroot']+ldpath) def mywimf(dt): currentdir, subdirs, files = dt def mymf(item): filepath = os.path.join(currentdir,item) if filepath in etpConst['libtest_files_blacklist']: return 0 if not os.access(filepath,os.R_OK): return 0 if not os.path.isfile(filepath): return 0 if not self.entropyTools.is_elf_file(filepath): return 0 return filepath[sys_root_len:] return set([x for x in map(mymf,files) if type(x) != int]) for x in map(mywimf,mywalk_iter): executables |= x self.updateProgress( blue(_("Collecting broken executables")), importance = 2, type = "info", header = red(" @@ ") ) t = red(_("Attention")) + ": " + \ blue(_("don't worry about libraries that are shown here but not later.")) self.updateProgress( t, importance = 1, type = "info", header = red(" @@ ") ) myQA = self.QA() plain_brokenexecs = set() total = len(executables) count = 0 scan_txt = blue("%s ..." % (_("Scanning libraries"),)) for executable in executables: if callable(task_bombing_func): task_bombing_func() count += 1 if (count%10 == 0) or (count == total) or (count == 1): self.updateProgress( scan_txt, importance = 0, type = "info", count = (count,total), back = True, percent = True, header = " " ) myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable) def mymf2(mylib): return not myQA.resolve_dynamic_library(mylib, executable) mylibs = set(filter(mymf2,myelfs)) broken_sym_found = set() if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable) if not (mylibs or broken_sym_found): continue if mylibs: alllibs = blue(' :: ').join(list(mylibs)) self.updateProgress( red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]", importance = 1, type = "info", percent = True, count = (count,total), header = " " ) elif broken_sym_found: allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)]) if len(allsyms) > 50: allsyms = brown(_('various broken symbols')) self.updateProgress( red(etpConst['systemroot']+executable)+" { "+allsyms+" }", importance = 1, type = "info", percent = True, count = (count,total), header = " " ) plain_brokenexecs.add(executable) del executables packagesMatched = {} if not etpSys['serverside']: self.updateProgress( blue(_("Matching broken libraries/executables")), importance = 1, type = "info", header = red(" @@ ") ) matched = set() for brokenlib in plain_brokenexecs: idpackages = self.clientDbconn.searchBelongs(brokenlib) for idpackage in idpackages: key, slot = self.clientDbconn.retrieveKeySlot(idpackage) mymatch = self.atomMatch(key, matchSlot = slot) if mymatch[0] == -1: matched.add(brokenlib) continue cmpstat = self.get_package_action(mymatch) if cmpstat == 0: continue if not packagesMatched.has_key(brokenlib): packagesMatched[brokenlib] = set() packagesMatched[brokenlib].add(mymatch) matched.add(brokenlib) plain_brokenexecs -= matched return packagesMatched,plain_brokenexecs,0 def move_to_branch(self, branch, pretend = False): if pretend: return 0 if branch != etpConst['branch']: # update configuration self.entropyTools.writeNewBranch(branch) # reset treeupdatesactions self.clientDbconn.resetTreeupdatesDigests() # clean cache self.purge_cache(showProgress = False) # reopen Client Database, this will make treeupdates to be re-read self.closeAllRepositoryDatabases() etpConst['branch'] = branch self.reload_constants() etpConst['branch'] = branch self.validate_repositories() self.reopenClientDbconn() return 0 # tell if a new equo release is available, returns True or False def check_equo_updates(self): found, match = self.check_package_update("app-admin/equo", deep = True) return found ''' @input: matched atom (idpackage,repoid) @output: upgrade: int(2) install: int(1) reinstall: int(0) downgrade: int(-1) ''' def get_package_action(self, match): dbconn = self.openRepositoryDatabase(match[1]) pkgkey, pkgslot = dbconn.retrieveKeySlot(match[0]) results = self.clientDbconn.searchKeySlot(pkgkey, pkgslot) if not results: return 1 installed_idpackage = results[0][0] pkgver, pkgtag, pkgrev = dbconn.getVersioningData(match[0]) installedVer, installedTag, installedRev = self.clientDbconn.getVersioningData(installed_idpackage) pkgcmp = self.entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) if pkgcmp == 0: return 0 elif pkgcmp > 0: return 2 return -1 def get_meant_packages(self, search_term, from_installed = False, valid_repos = []): pkg_data = [] atom_srch = False if "/" in search_term: atom_srch = True if not valid_repos: valid_repos = self.validRepositories if from_installed: valid_repos = [1] for repo in valid_repos: if isinstance(repo,basestring): dbconn = self.openRepositoryDatabase(repo) elif isinstance(repo,EntropyDatabaseInterface): dbconn = repo elif hasattr(self,'clientDbconn'): dbconn = self.clientDbconn else: continue pkg_data.extend([(x,repo,) for x in dbconn.searchSimilarPackages(search_term, atom = atom_srch)]) return pkg_data # better to use key:slot def check_package_update(self, atom, deep = False): c_hash = "%s%s" % (etpCache['check_package_update'],hash("%s%s" % (atom,hash(deep),)),) if self.xcache: cached = self.Cacher.pop(c_hash) if cached != None: return cached found = False match = self.clientDbconn.atomMatch(atom) matched = None if match[0] != -1: myatom = self.clientDbconn.retrieveAtom(match[0]) mytag = self.entropyTools.dep_gettag(myatom) myatom = self.entropyTools.remove_tag(myatom) myrev = self.clientDbconn.retrieveRevision(match[0]) pkg_match = "="+myatom+"~"+str(myrev) if mytag != None: pkg_match += "#%s" % (mytag,) pkg_unsatisfied = self.get_unsatisfied_dependencies([pkg_match], deep_deps = deep) if pkg_unsatisfied: found = True del pkg_unsatisfied matched = self.atomMatch(pkg_match) del match if self.xcache: self.Cacher.push(c_hash,(found,matched)[:]) return found, matched # @returns -1 if the file does not exist or contains bad data # @returns int>0 if the file exists def get_repository_revision(self, reponame): if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r") try: revision = int(f.readline().strip()) except: revision = -1 f.close() else: revision = -1 return revision def update_repository_revision(self, reponame): r = self.get_repository_revision(reponame) etpRepositories[reponame]['dbrevision'] = "0" if r != -1: etpRepositories[reponame]['dbrevision'] = str(r) # @returns -1 if the file does not exist # @returns int>0 if the file exists def get_repository_db_file_checksum(self, reponame): if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile']): f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") try: mhash = f.readline().strip().split()[0] except: mhash = "-1" f.close() else: mhash = "-1" return mhash def update_ugc_cache(self, repository): if not self.UGC.is_repository_eapi3_aware(repository): return None status = True votes_dict, err_msg = self.UGC.get_all_votes(repository) if isinstance(votes_dict,dict): self.UGC.UGCCache.save_vote_cache(repository, votes_dict) else: status = False downloads_dict, err_msg = self.UGC.get_all_downloads(repository) if isinstance(downloads_dict,dict): self.UGC.UGCCache.save_downloads_cache(repository, downloads_dict) else: status = False return status def __handle_multi_repo_matches(self, results, extended_results, valid_repos, open_db): packageInformation = {} versionInformation = {} # .tbz2 repos have always the precedence, so if we find them, # we should second what user wants, installing his tbz2 tbz2repos = [x for x in results if x.endswith(etpConst['packagesext'])] if tbz2repos: del tbz2repos newrepos = results.copy() for x in newrepos: if x.endswith(etpConst['packagesext']): continue del results[x] version_duplicates = set() versions = set() for repo in results: packageInformation[repo] = {} if extended_results: version = results[repo][1] packageInformation[repo]['versiontag'] = results[repo][2] packageInformation[repo]['revision'] = results[repo][3] else: dbconn = open_db(repo) packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(results[repo]) packageInformation[repo]['revision'] = dbconn.retrieveRevision(results[repo]) version = dbconn.retrieveVersion(results[repo]) packageInformation[repo]['version'] = version versionInformation[version] = repo if version in versions: version_duplicates.add(version) versions.add(version) newerVersion = self.entropyTools.getNewerVersion(list(versions))[0] # if no duplicates are found or newer version is not in duplicates we're done if (not version_duplicates) or (newerVersion not in version_duplicates): reponame = versionInformation.get(newerVersion) return (results[reponame],reponame) # we have two repositories with >two packages with the same version # check package tag conflictingEntries = {} tags_duplicates = set() tags = set() tagsInfo = {} for repo in packageInformation: if packageInformation[repo]['version'] != newerVersion: continue conflictingEntries[repo] = {} versiontag = packageInformation[repo]['versiontag'] if versiontag in tags: tags_duplicates.add(versiontag) tags.add(versiontag) tagsInfo[versiontag] = repo conflictingEntries[repo]['versiontag'] = versiontag conflictingEntries[repo]['revision'] = packageInformation[repo]['revision'] # tags will always be != [] newerTag = sorted(list(tags), reverse = True)[0] if newerTag not in tags_duplicates: reponame = tagsInfo.get(newerTag) return (results[reponame],reponame) # in this case, we have >two packages with the same version and tag # check package revision conflictingRevisions = {} revisions = set() revisions_duplicates = set() revisionInfo = {} for repo in conflictingEntries: if conflictingEntries[repo]['versiontag'] == newerTag: conflictingRevisions[repo] = {} versionrev = conflictingEntries[repo]['revision'] if versionrev in revisions: revisions_duplicates.add(versionrev) revisions.add(versionrev) revisionInfo[versionrev] = repo conflictingRevisions[repo]['revision'] = versionrev newerRevision = max(revisions) if newerRevision not in revisions_duplicates: reponame = revisionInfo.get(newerRevision) return (results[reponame],reponame) # final step, in this case we have >two packages with the same version, tag and revision # get the repository with the biggest priority for reponame in valid_repos: if reponame in conflictingRevisions: return (results[reponame],reponame) def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), matchTag = None, packagesFilter = True, multiMatch = False, multiRepo = False, matchRevision = None, matchRepo = None, server_repos = [], serverInstance = None, extendedResults = False, useCache = True): # support match in repository from shell # atom@repo1,repo2,repo3 atom, repos = self.entropyTools.dep_get_match_in_repos(atom) if (matchRepo == None) and (repos != None): matchRepo = repos u_hash = "" m_hash = "" k_ms = "//" k_mt = "@#@" k_mr = "-1" if isinstance(matchRepo,(list,tuple,set,)): u_hash = hash(frozenset(matchRepo)) if isinstance(matchBranches,(list,tuple,set,)): m_hash = hash(frozenset(matchBranches)) if isinstance(matchSlot,basestring): k_ms = matchSlot if isinstance(matchTag,basestring): k_mt = matchTag if isinstance(matchRevision,basestring): k_mr = matchRevision c_hash = "|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" % ( atom,k_ms,k_mt,hash(packagesFilter), hash(frozenset(self.validRepositories)), hash(frozenset(etpRepositories)), hash(multiMatch),hash(multiRepo),hash(caseSensitive), k_mr,hash(extendedResults), u_hash, m_hash ) c_hash = "%s%s" % (self.atomMatchCacheKey,hash(c_hash),) if self.xcache and useCache: cached = self.Cacher.pop(c_hash) if cached != None: return cached if server_repos: if not serverInstance: t = _("server_repos needs serverInstance") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (t,)) valid_repos = server_repos[:] else: valid_repos = self.validRepositories if matchRepo and (type(matchRepo) in (list,tuple,set)): valid_repos = list(matchRepo) def open_db(repoid): if server_repos: dbconn = serverInstance.openServerDatabase(just_reading = True, repo = repoid) else: dbconn = self.openRepositoryDatabase(repoid) return dbconn repoResults = {} for repo in valid_repos: # search dbconn = open_db(repo) use_cache = useCache while 1: try: query_data,query_rc = dbconn.atomMatch( atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, matchTag = matchTag, packagesFilter = packagesFilter, matchRevision = matchRevision, extendedResults = extendedResults, useCache = use_cache ) if query_rc == 0: # package found, add to our dictionary if extendedResults: repoResults[repo] = (query_data[0],query_data[2],query_data[3],query_data[4]) else: repoResults[repo] = query_data except TypeError: if not use_cache: raise use_cache = False continue break if extendedResults: dbpkginfo = ((-1,None,None,None),1) else: dbpkginfo = (-1,1) if multiRepo and repoResults: data = set() for repoid in repoResults: data.add((repoResults[repoid],repoid)) dbpkginfo = (data,0) elif len(repoResults) == 1: # one result found repo = repoResults.keys()[0] dbpkginfo = (repoResults[repo],repo) elif len(repoResults) > 1: # we have to decide which version should be taken mypkginfo = self.__handle_multi_repo_matches(repoResults, extendedResults, valid_repos, open_db) if mypkginfo != None: dbpkginfo = mypkginfo # multimatch support if multiMatch: if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure if multiRepo: data = set() for q_id,q_repo in dbpkginfo[0]: dbconn = open_db(q_repo) query_data, query_rc = dbconn.atomMatch( atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, matchTag = matchTag, packagesFilter = packagesFilter, multiMatch = True, extendedResults = extendedResults ) if extendedResults: for item in query_data: data.add(((item[0],item[2],item[3],item[4]),q_repo)) else: for x in query_data: data.add((x,q_repo)) dbpkginfo = (data,0) else: dbconn = open_db(dbpkginfo[1]) query_data, query_rc = dbconn.atomMatch( atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches, matchTag = matchTag, packagesFilter = packagesFilter, multiMatch = True, extendedResults = extendedResults ) if extendedResults: dbpkginfo = (set([((x[0],x[2],x[3],x[4]),dbpkginfo[1]) for x in query_data]),0) else: dbpkginfo = (set([(x,dbpkginfo[1]) for x in query_data]),0) if self.xcache and useCache: self.Cacher.push(c_hash,dbpkginfo[:]) return dbpkginfo # expands package sets, and in future something more perhaps def packagesExpand(self, packages): new_packages = [] for pkg_id in range(len(packages)): package = packages[pkg_id] # expand package sets if package.startswith(etpConst['packagesetprefix']): set_pkgs = sorted(list(self.packageSetExpand(package, raise_exceptions = False))) new_packages.extend([x for x in set_pkgs if x not in packages]) # atomMatch below will filter dupies else: new_packages.append(package) return new_packages def packageSetExpand(self, package_set, raise_exceptions = True): max_recursion_level = 50 recursion_level = 0 def do_expand(myset, recursion_level, max_recursion_level): recursion_level += 1 if recursion_level > max_recursion_level: raise exceptionTools.InvalidPackageSet('InvalidPackageSet: corrupted, too many recursions: %s' % (myset,)) set_data, set_rc = self.packageSetMatch(myset[len(etpConst['packagesetprefix']):]) if not set_rc: raise exceptionTools.InvalidPackageSet('InvalidPackageSet: not found: %s' % (myset,)) (set_from, package_set, mydata,) = set_data mypkgs = set() for fset in mydata: # recursively if fset.startswith(etpConst['packagesetprefix']): mypkgs |= do_expand(fset, recursion_level, max_recursion_level) else: mypkgs.add(fset) return mypkgs if not package_set.startswith(etpConst['packagesetprefix']): package_set = "%s%s" % (etpConst['packagesetprefix'],package_set,) try: mylist = do_expand(package_set, recursion_level, max_recursion_level) except exceptionTools.InvalidPackageSet: if raise_exceptions: raise mylist = set() return mylist def packageSetList(self, server_repos = [], serverInstance = None, matchRepo = None): return self.packageSetMatch('', matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] def packageSetSearch(self, package_set, server_repos = [], serverInstance = None, matchRepo = None): # search support if package_set == '*': package_set = '' return self.packageSetMatch(package_set, matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] def packageSetMatch(self, package_set, multiMatch = False, matchRepo = None, server_repos = [], serverInstance = None, search = False): # support match in repository from shell # set@repo1,repo2,repo3 package_set, repos = self.entropyTools.dep_get_match_in_repos(package_set) if (matchRepo == None) and (repos != None): matchRepo = repos if server_repos: if not serverInstance: t = _("server_repos needs serverInstance") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (t,)) valid_repos = server_repos[:] else: valid_repos = self.validRepositories if matchRepo and (type(matchRepo) in (list,tuple,set)): valid_repos = list(matchRepo) def open_db(repoid): if server_repos: dbconn = serverInstance.openServerDatabase(just_reading = True, repo = repoid) else: dbconn = self.openRepositoryDatabase(repoid) return dbconn # if we search, we return all the matches available if search: multiMatch = True set_data = [] while 1: # check inside SystemSettings if not server_repos: if search: mysets = [x for x in self.SystemSettings['system_package_sets'].keys() if (x.find(package_set) != -1)] for myset in mysets: mydata = self.SystemSettings['system_package_sets'].get(myset) set_data.append((etpConst['userpackagesetsid'], unicode(myset), mydata.copy(),)) else: mydata = self.SystemSettings['system_package_sets'].get(package_set) if mydata != None: set_data.append((etpConst['userpackagesetsid'], unicode(package_set), mydata,)) if not multiMatch: break for repoid in valid_repos: dbconn = open_db(repoid) if search: mysets = dbconn.searchSets(package_set) for myset in mysets: mydata = dbconn.retrievePackageSet(myset) set_data.append((repoid, myset, mydata.copy(),)) else: mydata = dbconn.retrievePackageSet(package_set) if mydata: set_data.append((repoid, package_set, mydata,)) if not multiMatch: break break if not set_data: return (),False if multiMatch: return set_data,True return set_data.pop(0),True def repository_move_clear_cache(self, repoid = None): self.clear_dump_cache(etpCache['world_available']) self.clear_dump_cache(etpCache['world_update']) self.clear_dump_cache(etpCache['check_package_update']) self.clear_dump_cache(etpCache['filter_satisfied_deps']) self.clear_dump_cache(self.atomMatchCacheKey) self.clear_dump_cache(etpCache['dep_tree']) if repoid != None: self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,)) def addRepository(self, repodata): # update etpRepositories try: etpRepositories[repodata['repoid']] = {} etpRepositories[repodata['repoid']]['description'] = repodata['description'] etpRepositories[repodata['repoid']]['configprotect'] = None etpRepositories[repodata['repoid']]['configprotectmask'] = None except KeyError: t = _("repodata dictionary is corrupted") raise exceptionTools.InvalidData("InvalidData: %s" % (t,)) if repodata['repoid'].endswith(etpConst['packagesext']) or repodata.get('in_memory'): # dynamic repository try: # no need # etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:] smart_package = repodata.get('smartpackage') if smart_package != None: etpRepositories[repodata['repoid']]['smartpackage'] = smart_package etpRepositories[repodata['repoid']]['dbpath'] = repodata.get('dbpath') etpRepositories[repodata['repoid']]['pkgpath'] = repodata.get('pkgpath') except KeyError: raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted") # put at top priority, shift others etpRepositoriesOrder.insert(0,repodata['repoid']) else: # XXX it's boring to keep this in sync with entropyConstants stuff, solutions? etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] etpRepositories[repodata['repoid']]['packages'] = [x+"/"+etpConst['product'] for x in repodata['plain_packages']] etpRepositories[repodata['repoid']]['plain_database'] = repodata['plain_database'] etpRepositories[repodata['repoid']]['database'] = repodata['plain_database'] + \ "/" + etpConst['product'] + "/database/" + etpConst['currentarch'] + "/" + etpConst['branch'] if not repodata['dbcformat'] in etpConst['etpdatabasesupportedcformats']: repodata['dbcformat'] = etpConst['etpdatabasesupportedcformats'][0] etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat'] etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + \ "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch'] + "/" + etpConst['branch'] # set dbrevision myrev = self.get_repository_revision(repodata['repoid']) if myrev == -1: myrev = 0 etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev) if repodata.has_key("position"): etpRepositoriesOrder.insert(repodata['position'],repodata['repoid']) else: etpRepositoriesOrder.append(repodata['repoid']) if not repodata.has_key("service_port"): repodata['service_port'] = int(etpConst['socket_service']['port']) if not repodata.has_key("ssl_service_port"): repodata['ssl_service_port'] = int(etpConst['socket_service']['ssl_port']) etpRepositories[repodata['repoid']]['service_port'] = repodata['service_port'] etpRepositories[repodata['repoid']]['ssl_service_port'] = repodata['ssl_service_port'] self.repository_move_clear_cache(repodata['repoid']) # save new etpRepositories to file self.entropyTools.saveRepositorySettings(repodata) self.reload_constants() self.validate_repositories() def removeRepository(self, repoid, disable = False): # ensure that all dbs are closed self.closeAllRepositoryDatabases() done = False if etpRepositories.has_key(repoid): del etpRepositories[repoid] done = True if etpRepositoriesExcluded.has_key(repoid): del etpRepositoriesExcluded[repoid] done = True if done: if repoid in etpRepositoriesOrder: etpRepositoriesOrder.remove(repoid) self.repository_move_clear_cache(repoid) # save new etpRepositories to file repodata = {} repodata['repoid'] = repoid if disable: self.entropyTools.saveRepositorySettings(repodata, disable = True) else: self.entropyTools.saveRepositorySettings(repodata, remove = True) self.reload_constants() self.validate_repositories() def shiftRepository(self, repoid, toidx): # update etpRepositoriesOrder etpRepositoriesOrder.remove(repoid) etpRepositoriesOrder.insert(toidx,repoid) self.entropyTools.writeOrderedRepositoriesEntries() self.reload_constants() self.repository_move_clear_cache(repoid) self.validate_repositories() def enableRepository(self, repoid): self.repository_move_clear_cache(repoid) # save new etpRepositories to file repodata = {} repodata['repoid'] = repoid self.entropyTools.saveRepositorySettings(repodata, enable = True) self.reload_constants() self.validate_repositories() def disableRepository(self, repoid): # update etpRepositories done = False try: del etpRepositories[repoid] done = True except: pass if done: try: etpRepositoriesOrder.remove(repoid) except: pass # it's not vital to reset etpRepositoriesOrder counters self.repository_move_clear_cache(repoid) # save new etpRepositories to file repodata = {} repodata['repoid'] = repoid self.entropyTools.saveRepositorySettings(repodata, disable = True) self.reload_constants() self.validate_repositories() def get_unsatisfied_dependencies(self, dependencies, deep_deps = False, depcache = {}): if self.xcache: c_data = sorted(list(dependencies)) client_checksum = self.clientDbconn.database_checksum() c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps))+client_checksum c_hash = "unsat_%s" % (hash(c_hash),) cached = self.dumpTools.loadobj(etpCache['filter_satisfied_deps']+c_hash) if cached != None: return cached cdb_am = self.clientDbconn.atomMatch am = self.atomMatch open_repo = self.openRepositoryDatabase intf_error = self.dbapi2.InterfaceError cdb_getversioning = self.clientDbconn.getVersioningData cdb_retrieveneededraw = self.clientDbconn.retrieveNeededRaw etp_cmp = self.entropyTools.entropyCompareVersions do_needed_check = False def fm_dep(dependency): cached = depcache.get(dependency) if cached != None: return cached ### conflict if dependency.startswith("!"): idpackage,rc = cdb_am(dependency[1:]) if idpackage != -1: depcache[dependency] = dependency return dependency depcache[dependency] = 0 return 0 c_id,c_rc = cdb_am(dependency) if c_id == -1: depcache[dependency] = dependency return dependency #if not deep_deps and not do_needed_check: # depcache[dependency] = 0 # return 0 r_id,r_repo = am(dependency) if r_id == -1: depcache[dependency] = dependency return dependency if do_needed_check: dbconn = open_repo(r_repo) installed_needed = cdb_retrieveneededraw(c_id) repo_needed = dbconn.retrieveNeededRaw(r_id) if installed_needed != repo_needed: return dependency #elif not deep_deps: # return 0 dbconn = open_repo(r_repo) try: repo_pkgver, repo_pkgtag, repo_pkgrev = dbconn.getVersioningData(r_id) except (intf_error,TypeError,): # package entry is broken return dependency try: installedVer, installedTag, installedRev = cdb_getversioning(c_id) except TypeError: # corrupted entry? installedVer = "0" installedTag = '' installedRev = 0 vcmp = etp_cmp((repo_pkgver,repo_pkgtag,repo_pkgrev,), (installedVer,installedTag,installedRev,)) if vcmp != 0: if not deep_deps and ((repo_pkgver,repo_pkgtag,) == (installedVer,installedTag,)) and (repo_pkgrev != installedRev): depcache[dependency] = 0 return 0 depcache[dependency] = dependency return dependency depcache[dependency] = 0 return 0 unsatisfied = map(fm_dep,dependencies) unsatisfied = set([x for x in unsatisfied if x != 0]) if self.xcache: try: self.dumpTools.dumpobj(etpCache['filter_satisfied_deps']+c_hash,unsatisfied) except IOError: pass return unsatisfied def get_masked_package_reason(self, match): idpackage, repoid = match dbconn = self.openRepositoryDatabase(repoid) idpackage, idreason = dbconn.idpackageValidator(idpackage) masked = False if idpackage == -1: masked = True return masked, idreason, self.SystemSettings['pkg_masking_reasons'].get(idreason) def get_masked_packages_tree(self, match, atoms = False, flat = False, matchfilter = None): if not isinstance(matchfilter,set): matchfilter = set() maskedtree = {} mybuffer = self.entropyTools.lifobuffer() depcache = set() treelevel = -1 match_id, match_repo = match mydbconn = self.openRepositoryDatabase(match_repo) myatom = mydbconn.retrieveAtom(match_id) idpackage, idreason = mydbconn.idpackageValidator(match_id) if idpackage == -1: treelevel += 1 if atoms: mydict = {myatom: idreason,} else: mydict = {match: idreason,} if flat: maskedtree.update(mydict) else: maskedtree[treelevel] = {} maskedtree[treelevel].update(mydict) mydeps = mydbconn.retrieveDependencies(match_id) for mydep in mydeps: mybuffer.push(mydep) mydep = mybuffer.pop() open_db = self.openRepositoryDatabase am = self.atomMatch while mydep: if mydep in depcache: mydep = mybuffer.pop() continue depcache.add(mydep) idpackage, repoid = am(mydep) if (idpackage, repoid) in matchfilter: mydep = mybuffer.pop() continue if idpackage != -1: # doing even here because atomMatch with packagesFilter = False can pull # something different matchfilter.add((idpackage, repoid)) # collect masked if idpackage == -1: idpackage, repoid = am(mydep, packagesFilter = False) if idpackage != -1: treelevel += 1 if not maskedtree.has_key(treelevel) and not flat: maskedtree[treelevel] = {} dbconn = open_db(repoid) vidpackage, idreason = dbconn.idpackageValidator(idpackage) if atoms: mydict = {dbconn.retrieveAtom(idpackage): idreason} else: mydict = {(idpackage,repoid): idreason} if flat: maskedtree.update(mydict) else: maskedtree[treelevel].update(mydict) # push its dep into the buffer if idpackage != -1: matchfilter.add((idpackage, repoid)) dbconn = open_db(repoid) owndeps = dbconn.retrieveDependencies(idpackage) for owndep in owndeps: mybuffer.push(owndep) mydep = mybuffer.pop() return maskedtree def generate_dependency_tree(self, atomInfo, empty_deps = False, deep_deps = False, matchfilter = None, flat = False, filter_unsat_cache = None, treecache = None, keyslotcache = None): if not isinstance(matchfilter,set): matchfilter = set() if not isinstance(filter_unsat_cache,dict): filter_unsat_cache = {} if not isinstance(treecache,set): treecache = set() if not isinstance(keyslotcache,set): keyslotcache = set() mydbconn = self.openRepositoryDatabase(atomInfo[1]) myatom = mydbconn.retrieveAtom(atomInfo[0]) # caches # special events deps_not_found = set() conflicts = set() mydep = (1,myatom) mybuffer = self.entropyTools.lifobuffer() deptree = set() if atomInfo not in matchfilter: deptree.add((1,atomInfo)) virgin = True open_repo = self.openRepositoryDatabase atom_match = self.atomMatch cdb_atom_match = self.clientDbconn.atomMatch lookup_conflict_replacement = self._lookup_conflict_replacement lookup_library_breakages = self._lookup_library_breakages lookup_inverse_dependencies = self._lookup_inverse_dependencies get_unsatisfied_deps = self.get_unsatisfied_dependencies def my_dep_filter(x): if x in treecache: return False if tuple(x.split(":")) in keyslotcache: return False return True while mydep: dep_level, dep_atom = mydep # already analyzed in this call if dep_atom in treecache: mydep = mybuffer.pop() continue treecache.add(dep_atom) if dep_atom == None: # corrupted entry mydep = mybuffer.pop() continue # conflicts if dep_atom[0] == "!": c_idpackage, xst = cdb_atom_match(dep_atom[1:]) if c_idpackage != -1: myreplacement = lookup_conflict_replacement(dep_atom[1:], c_idpackage, deep_deps = deep_deps) if (myreplacement != None) and (myreplacement not in treecache): mybuffer.push((dep_level+1,myreplacement)) else: conflicts.add(c_idpackage) mydep = mybuffer.pop() continue # atom found? if virgin: virgin = False m_idpackage, m_repo = atomInfo dbconn = open_repo(m_repo) myidpackage, idreason = dbconn.idpackageValidator(m_idpackage) if myidpackage == -1: m_idpackage = -1 else: m_idpackage, m_repo = atom_match(dep_atom) if m_idpackage == -1: deps_not_found.add(dep_atom) mydep = mybuffer.pop() continue # check if atom has been already pulled in matchdb = open_repo(m_repo) matchatom = matchdb.retrieveAtom(m_idpackage) matchkey, matchslot = matchdb.retrieveKeySlot(m_idpackage) if (dep_atom != matchatom) and (matchatom in treecache): mydep = mybuffer.pop() continue treecache.add(matchatom) # check if key + slot has been already pulled in if (matchslot,matchkey) in keyslotcache: mydep = mybuffer.pop() continue else: keyslotcache.add((matchslot,matchkey)) match = (m_idpackage, m_repo,) # result already analyzed? if match in matchfilter: mydep = mybuffer.pop() continue # already analyzed by the calling function if match in matchfilter: mydep = mybuffer.pop() continue matchfilter.add(match) treedepth = dep_level+1 # all checks passed, well done matchfilter.add(match) deptree.add((dep_level,match)) # add match # extra hooks cm_idpackage, cm_result = cdb_atom_match(matchkey, matchSlot = matchslot) if cm_idpackage != -1: broken_atoms = lookup_library_breakages(match, (cm_idpackage, cm_result,), deep_deps = deep_deps) inverse_deps = lookup_inverse_dependencies(match, (cm_idpackage, cm_result,)) if inverse_deps: deptree.remove((dep_level,match)) for ikey,islot in inverse_deps: iks_str = '%s:%s' % (ikey,islot,) if ((ikey,islot) not in keyslotcache) and (iks_str not in treecache): mybuffer.push((dep_level,iks_str)) keyslotcache.add((ikey,islot)) deptree.add((treedepth,match)) treedepth += 1 for x in broken_atoms: if (tuple(x.split(":")) not in keyslotcache) and (x not in treecache): mybuffer.push((treedepth,x)) myundeps = filter(my_dep_filter,matchdb.retrieveDependenciesList(m_idpackage)) if not empty_deps: myundeps = filter(my_dep_filter,get_unsatisfied_deps(myundeps, deep_deps, depcache = filter_unsat_cache)) # PDEPENDs support if myundeps: post_deps = [x for x in matchdb.retrievePostDependencies(m_idpackage) if x in myundeps] myundeps = [x for x in myundeps if x not in post_deps] for x in post_deps: mybuffer.push((-1,x)) # always after the package itself for x in myundeps: mybuffer.push((treedepth,x)) mydep = mybuffer.pop() if deps_not_found: return list(deps_not_found),-2 if flat: return [x[1] for x in deptree],0 newdeptree = {} for key,item in deptree: if key not in newdeptree: newdeptree[key] = set() newdeptree[key].add(item) # conflicts newdeptree[0] = conflicts return newdeptree,0 # note: newtree[0] contains possible conflicts def _lookup_system_mask_repository_deps(self): data = self.SystemSettings['repos_system_mask'] if not data: return [] mydata = [] cached_items = set() for atom in data: mymatch = self.atomMatch(atom) if mymatch[0] == -1: # ignore missing ones intentionally continue if mymatch in cached_items: continue if mymatch not in mydata: # check if not found myaction = self.get_package_action(mymatch) if myaction != 0: mydata.append(mymatch) # upgrades, installs and downgrades allowed cached_items.add(mymatch) return mydata def _lookup_conflict_replacement(self, conflict_atom, client_idpackage, deep_deps): if self.entropyTools.isjustname(conflict_atom): return None conflict_match = self.atomMatch(conflict_atom) mykey, myslot = self.clientDbconn.retrieveKeySlot(client_idpackage) new_match = self.atomMatch(mykey, matchSlot = myslot) if (conflict_match == new_match) or (new_match[1] == 1): return None action = self.get_package_action(new_match) if (action == 0) and (not deep_deps): return None return "%s:%s" % (mykey,myslot,) def _lookup_inverse_dependencies(self, match, clientmatch): cmpstat = self.get_package_action(match) if cmpstat == 0: return set() keyslots = set() mydepends = self.clientDbconn.retrieveDepends(clientmatch[0]) am = self.atomMatch cdb_rdeps = self.clientDbconn.retrieveDependencies cdb_rks = self.clientDbconn.retrieveKeySlot gpa = self.get_package_action keyslots_cache = set() match_cache = {} for idpackage in mydepends: try: key, slot = cdb_rks(idpackage) except TypeError: continue if (key,slot) in keyslots_cache: continue keyslots_cache.add((key,slot)) if (key,slot) in keyslots: continue # grab its deps mydeps = cdb_rdeps(idpackage) found = False for mydep in mydeps: mymatch = match_cache.get(mydep, 0) if mymatch == 0: mymatch = am(mydep) match_cache[mydep] = mymatch if mymatch == match: found = True break if not found: mymatch = am(key, matchSlot = slot) if mymatch[0] == -1: continue cmpstat = gpa(mymatch) if cmpstat == 0: continue keyslots.add((key,slot)) return keyslots def _lookup_library_breakages(self, match, clientmatch, deep_deps = False): # there is no need to update this cache when "match" will be installed, because at that point # clientmatch[0] will differ. c_hash = "%s|%s|%s" % (hash(tuple(match)),hash(deep_deps),hash(tuple(clientmatch)),) c_hash = "%s%s" % (etpCache['library_breakage'],hash(c_hash),) if self.xcache: cached = self.Cacher.pop(c_hash) if cached != None: return cached # these should be pulled in before repo_atoms = set() # these can be pulled in after client_atoms = set() matchdb = self.openRepositoryDatabase(match[1]) reponeeded = matchdb.retrieveNeeded(match[0], extended = True, format = True) clientneeded = self.clientDbconn.retrieveNeeded(clientmatch[0], extended = True, format = True) repo_split = [x.split(".so")[0] for x in reponeeded] client_split = [x.split(".so")[0] for x in clientneeded] client_side = [x for x in clientneeded if (x not in reponeeded) and (x.split(".so")[0] in repo_split)] repo_side = [x for x in reponeeded if (x not in clientneeded) and (x.split(".so")[0] in client_split)] del clientneeded,client_split,repo_split # all the packages in client_side should be pulled in and updated client_idpackages = set() for needed in client_side: client_idpackages |= self.clientDbconn.searchNeeded(needed) client_keyslots = set() def mymf(idpackage): if idpackage == clientmatch[0]: return 0 return self.clientDbconn.retrieveKeySlot(idpackage) client_keyslots = set([x for x in map(mymf,client_idpackages) if x != 0]) # all the packages in repo_side should be pulled in too repodata = {} for needed in repo_side: repodata[needed] = reponeeded[needed] del repo_side,reponeeded repo_dependencies = matchdb.retrieveDependencies(match[0]) matched_deps = set() matched_repos = set() for dependency in repo_dependencies: depmatch = self.atomMatch(dependency) if depmatch[0] == -1: continue matched_repos.add(depmatch[1]) matched_deps.add(depmatch) matched_repos = [x for x in etpRepositoriesOrder if x in matched_repos] found_matches = set() for needed in repodata: for myrepo in matched_repos: mydbc = self.openRepositoryDatabase(myrepo) solved_needed = mydbc.resolveNeeded(needed, elfclass = repodata[needed]) found = False for idpackage,myfile in solved_needed: x = (idpackage,myrepo) if x in matched_deps: found_matches.add(x) found = True break if found: break for idpackage,repo in found_matches: if not deep_deps: cmpstat = self.get_package_action((idpackage,repo)) if cmpstat == 0: continue mydbc = self.openRepositoryDatabase(repo) repo_atoms.add(mydbc.retrieveAtom(idpackage)) for key, slot in client_keyslots: idpackage, repo = self.atomMatch(key, matchSlot = slot) if idpackage == -1: continue if not deep_deps: cmpstat = self.get_package_action((idpackage, repo)) if cmpstat == 0: continue mydbc = self.openRepositoryDatabase(repo) client_atoms.add(mydbc.retrieveAtom(idpackage)) client_atoms |= repo_atoms if self.xcache: self.Cacher.push(c_hash,client_atoms.copy()) return client_atoms def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False, quiet = False): c_hash = "%s%s" % (etpCache['dep_tree'],hash("%s|%s|%s|%s" % ( hash(frozenset(sorted(matched_atoms))),hash(empty_deps), hash(deep_deps),self.clientDbconn.database_checksum(), )),) if self.xcache: cached = self.Cacher.pop(c_hash) if cached != None: return cached deptree = {} deptree[0] = set() atomlen = len(matched_atoms); count = 0 error_generated = 0 error_tree = set() # check if there are repositories needing some mandatory packages forced_matches = self._lookup_system_mask_repository_deps() if forced_matches: if isinstance(matched_atoms, list): matched_atoms = forced_matches + [x for x in matched_atoms if x not in forced_matches] elif isinstance(matched_atoms, set): # we cannot do anything about the order here matched_atoms |= set(forced_matches) sort_dep_text = _("Sorting dependencies") filter_unsat_cache = {} treecache = set() keyslotcache = set() matchfilter = set() for atomInfo in matched_atoms: if not quiet: count += 1 if (count%10 == 0) or (count == atomlen) or (count == 1): self.updateProgress(sort_dep_text, importance = 0, type = "info", back = True, header = ":: ", footer = " ::", percent = True, count = (count,atomlen)) if atomInfo in matchfilter: continue newtree, result = self.generate_dependency_tree( atomInfo, empty_deps, deep_deps, matchfilter = matchfilter, filter_unsat_cache = filter_unsat_cache, treecache = treecache, keyslotcache = keyslotcache ) if result == -2: # deps not found error_generated = -2 error_tree |= set(newtree) # it is a list, we convert it into set and update error_tree elif (result != 0): return newtree, result elif newtree: # add conflicts max_parent_key = max(deptree) deptree[0] |= newtree.pop(0) levelcount = 0 for mylevel in sorted(newtree.keys(), reverse = True): levelcount += 1 deptree[max_parent_key+levelcount] = newtree.get(mylevel) if error_generated != 0: return error_tree,error_generated if self.xcache: self.Cacher.push(c_hash,(deptree,0)[:]) return deptree,0 def _filter_depends_multimatched_atoms(self, idpackage, depends, monotree): remove_depends = set() for d_idpackage in depends: mydeps = self.clientDbconn.retrieveDependencies(d_idpackage) for mydep in mydeps: matches, rslt = self.clientDbconn.atomMatch(mydep, multiMatch = True) if rslt == 1: continue if idpackage in matches and len(matches) > 1: # are all in depends? for mymatch in matches: if mymatch not in depends and mymatch not in monotree: remove_depends.add(d_idpackage) break depends -= remove_depends return depends def generate_depends_tree(self, idpackages, deep = False): c_hash = "%s%s" % (etpCache['depends_tree'],hash("%s|%s" % (hash(tuple(sorted(idpackages))),hash(deep),),),) if self.xcache: cached = self.Cacher.pop(c_hash) if cached != None: return cached dependscache = set() treeview = set(idpackages) treelevel = set(idpackages) tree = {} treedepth = 0 # I start from level 1 because level 0 is idpackages itself tree[treedepth] = set(idpackages) monotree = set(idpackages) # monodimensional tree # check if dependstable is sane before beginning self.clientDbconn.retrieveDepends(idpackages[0]) count = 0 rem_dep_text = _("Calculating removable depends of") while 1: treedepth += 1 tree[treedepth] = set() for idpackage in treelevel: count += 1 p_atom = self.clientDbconn.retrieveAtom(idpackage) self.updateProgress( blue(rem_dep_text + " %s" % (red(p_atom),)), importance = 0, type = "info", back = True, header = '|/-\\'[count%4]+" " ) systempkg = not self.validatePackageRemoval(idpackage) if (idpackage in dependscache) or systempkg: if idpackage in treeview: treeview.remove(idpackage) continue # obtain its depends depends = self.clientDbconn.retrieveDepends(idpackage) # filter already satisfied ones depends = set([x for x in depends if x not in monotree]) depends = set([x for x in depends if self.validatePackageRemoval(x)]) if depends: depends = self._filter_depends_multimatched_atoms(idpackage, depends, monotree) if depends: # something depends on idpackage tree[treedepth] |= depends monotree |= depends treeview |= depends elif deep: # if deep, grab its dependencies and check mydeps = set() for x in self.clientDbconn.retrieveDependencies(idpackage): match = self.clientDbconn.atomMatch(x) if match[0] != -1: mydeps.add(match[0]) # now filter them mydeps = [x for x in mydeps if x not in monotree and not (self.clientDbconn.isSystemPackage(x) or self.is_installed_idpackage_in_system_mask(x) )] for x in mydeps: mydepends = self.clientDbconn.retrieveDepends(x) mydepends -= set([y for y in mydepends if y not in monotree]) if not mydepends: tree[treedepth].add(x) monotree.add(x) treeview.add(x) dependscache.add(idpackage) if idpackage in treeview: treeview.remove(idpackage) treelevel = treeview.copy() if not treelevel: if not tree[treedepth]: del tree[treedepth] # probably the last one is empty then break # now filter newtree for count in sorted(tree.keys(), reverse = True): x = 0 while x < count: tree[x] -= tree[count] x += 1 if self.xcache: self.Cacher.push(c_hash,(tree,0)[:]) return tree,0 # treeview is used to show deps while tree is used to run the dependency code. def list_repo_categories(self): categories = set() for repo in self.validRepositories: dbconn = self.openRepositoryDatabase(repo) catsdata = dbconn.listAllCategories() categories.update(set([x[1] for x in catsdata])) return categories def list_repo_packages_in_category(self, category): pkg_matches = [] for repo in self.validRepositories: dbconn = self.openRepositoryDatabase(repo) catsdata = dbconn.searchPackagesByCategory(category, branch = etpConst['branch']) pkg_matches.extend([(x[1],repo,) for x in catsdata if (x[1],repo,) not in pkg_matches]) return pkg_matches def get_category_description_data(self, category): data = {} for repo in self.validRepositories: try: dbconn = self.openRepositoryDatabase(repo) except exceptionTools.RepositoryError: continue try: data = dbconn.retrieveCategoryDescription(category) except (dbapi2.OperationalError, dbapi2.IntegrityError,): continue if data: break return data def list_installed_packages_in_category(self, category): pkg_matches = set([x[1] for x in self.clientDbconn.searchPackagesByCategory(category)]) return pkg_matches def all_repositories_checksum(self): sum_hashes = '' for repo in self.validRepositories: try: dbconn = self.openRepositoryDatabase(repo) except (exceptionTools.RepositoryError): continue # repo not available try: sum_hashes += dbconn.database_checksum() except dbapi2.OperationalError: pass return sum_hashes def get_available_packages_chash(self, branch): repo_digest = self.all_repositories_checksum() # client digest not needed, cache is kept updated c_hash = "%s%s%s" % (hash(repo_digest),hash(branch),hash(tuple(self.validRepositories)),) return str(hash(c_hash)) def get_available_packages_cache(self, branch = etpConst['branch'], myhash = None): if myhash == None: myhash = self.get_available_packages_chash(branch) disk_cache = self.Cacher.pop(etpCache['world_available']) try: if disk_cache['chash'] == myhash: return disk_cache['available'] except (KeyError,TypeError,): return None # this function searches all the not installed packages available in the repositories def calculate_available_packages(self, use_cache = True): c_hash = self.get_available_packages_chash(etpConst['branch']) if use_cache and self.xcache: cached = self.get_available_packages_cache(myhash = c_hash) if cached != None: return cached available = [] self.setTotalCycles(len(self.validRepositories)) avail_dep_text = _("Calculating available packages for") for repo in self.validRepositories: try: dbconn = self.openRepositoryDatabase(repo) dbconn.validateDatabase() except (exceptionTools.RepositoryError,exceptionTools.SystemDatabaseError): self.cycleDone() continue idpackages = [ x for x in dbconn.listAllIdpackages(branch = etpConst['branch'], branch_operator = "<=", order_by = 'atom') \ if dbconn.idpackageValidator(x)[0] != -1 ] count = 0 maxlen = len(idpackages) myavailable = [] do_break = False for idpackage in idpackages: if do_break: break count += 1 if (count % 10 == 0) or (count == 1) or (count == maxlen): self.updateProgress( avail_dep_text + " %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::" ) # get key + slot try: key, slot = dbconn.retrieveKeySlot(idpackage) matches = self.clientDbconn.searchKeySlot(key, slot) except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,): self.cycleDone() do_break = True continue if not matches: myavailable.append((idpackage,repo)) available += myavailable[:] self.cycleDone() if self.xcache: data = {} data['chash'] = c_hash data['available'] = available self.Cacher.push(etpCache['world_available'],data.copy()) return available def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None, ignore_spm_downgrades = False): if self.xcache: if db_digest == None: db_digest = self.all_repositories_checksum() c_hash = "%s%s" % (etpCache['world_update'],self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades),) disk_cache = self.Cacher.pop(c_hash) if disk_cache != None: try: return disk_cache['r'] except (KeyError, TypeError): return None def get_world_update_cache_hash(self, db_digest, empty_deps, branch, ignore_spm_downgrades): c_hash = "%s|%s|%s|%s|%s|%s" % ( db_digest,empty_deps,self.validRepositories, etpRepositoriesOrder, branch, ignore_spm_downgrades, ) return str(hash(c_hash)) def calculate_world_updates( self, empty_deps = False, branch = etpConst['branch'], ignore_spm_downgrades = etpConst['spm']['ignore-spm-downgrades'], use_cache = True ): db_digest = self.all_repositories_checksum() if use_cache and self.xcache: cached = self.get_world_update_cache(empty_deps = empty_deps, branch = branch, db_digest = db_digest, ignore_spm_downgrades = ignore_spm_downgrades) if cached != None: return cached update = [] remove = [] fine = [] # get all the installed packages idpackages = self.clientDbconn.listAllIdpackages(order_by = 'atom') maxlen = len(idpackages) count = 0 mytxt = _("Calculating world packages") for idpackage in idpackages: count += 1 if (count%10 == 0) or (count == maxlen) or (count == 1): self.updateProgress( mytxt, importance = 0, type = "info", back = True, header = ":: ", count = (count,maxlen), percent = True, footer = " ::" ) mystrictdata = self.clientDbconn.getStrictData(idpackage) # check against broken entries, or removed during iteration if mystrictdata == None: continue use_match_cache = True do_continue = False while 1: try: match = self.atomMatch( mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), extendedResults = True, useCache = use_match_cache ) except dbapi2.OperationalError: # ouch, but don't crash here do_continue = True break try: m_idpackage = match[0][0] except TypeError: if not use_match_cache: raise use_match_cache = False continue break if do_continue: continue # now compare # version: mystrictdata[2] # tag: mystrictdata[3] # revision: mystrictdata[4] if (m_idpackage != -1): repoid = match[1] version = match[0][1] tag = match[0][2] revision = match[0][3] if empty_deps: if (m_idpackage,repoid) not in update: update.append((m_idpackage,repoid)) continue elif (mystrictdata[2] != version): # different versions if (m_idpackage,repoid) not in update: update.append((m_idpackage,repoid)) continue elif (mystrictdata[3] != tag): # different tags if (m_idpackage,repoid) not in update: update.append((m_idpackage,repoid)) continue elif (mystrictdata[4] != revision): # different revision if mystrictdata[4] == 9999 and ignore_spm_downgrades: # no difference, we're ignoring revision 9999 fine.append(mystrictdata[5]) continue else: if (m_idpackage,repoid) not in update: update.append((m_idpackage,repoid)) continue else: # no difference fine.append(mystrictdata[5]) continue # don't take action if it's just masked maskedresults = self.atomMatch(mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), packagesFilter = False) if maskedresults[0] == -1: remove.append(idpackage) # look for packages that would match key with any slot (for eg: gcc, kernel updates) matchresults = self.atomMatch(mystrictdata[0], matchBranches = (branch,)) if matchresults[0] != -1: m_action = self.get_package_action(matchresults) if m_action > 0 and (matchresults not in update): update.append(matchresults) if self.xcache: c_hash = self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades) data = { 'r': (update, remove, fine,), 'empty_deps': empty_deps, } self.Cacher.push("%s%s" % (etpCache['world_update'],c_hash,), data, async = False) return update, remove, fine def get_match_conflicts(self, match): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) conflicts = dbconn.retrieveConflicts(m_id) found_conflicts = set() for conflict in conflicts: my_m_id, my_m_rc = self.clientDbconn.atomMatch(conflict) if my_m_id != -1: # check if the package shares the same slot match_data = dbconn.retrieveKeySlot(m_id) installed_match_data = self.clientDbconn.retrieveKeySlot(my_m_id) if match_data != installed_match_data: found_conflicts.add(my_m_id) return found_conflicts def is_match_masked(self, match, live_check = True): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) if idpackage != -1: return False return True def is_match_masked_by_user(self, match, live_check = True): # (query_status,masked?,) m_id, m_repo = match if m_repo not in self.validRepositories: return False dbconn = self.openRepositoryDatabase(m_repo) idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) if idpackage != -1: return False #,False myr = self.SystemSettings['pkg_masking_reference'] user_masks = [myr['user_package_mask'],myr['user_license_mask'],myr['user_live_mask']] if idreason in user_masks: return True #,True return False #,True def is_match_unmasked_by_user(self, match, live_check = True): # (query_status,unmasked?,) m_id, m_repo = match if m_repo not in self.validRepositories: return False dbconn = self.openRepositoryDatabase(m_repo) idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) if idpackage == -1: return False #,False myr = self.SystemSettings['pkg_masking_reference'] user_masks = [ myr['user_package_unmask'],myr['user_live_unmask'],myr['user_package_keywords'], myr['user_repo_package_keywords_all'], myr['user_repo_package_keywords'] ] if idreason in user_masks: return True #,True return False #,True def mask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): if self.is_match_masked(match, live_check = False): return True methods = { 'atom': self.mask_match_by_atom, 'keyslot': self.mask_match_by_keyslot, } rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) if dry_run: # inject if done "live" self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) self.SystemSettings['live_packagemasking']['mask_matches'].add(match) return rc def unmask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): if not self.is_match_masked(match, live_check = False): return True methods = { 'atom': self.unmask_match_by_atom, 'keyslot': self.unmask_match_by_keyslot, } rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) if dry_run: # inject if done "live" self.SystemSettings['live_packagemasking']['unmask_matches'].add(match) self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) return rc def _mask_unmask_match(self, match, method, methods_reference, dry_run = False, clean_all_cache = False): f = methods_reference.get(method) if not callable(f): raise exceptionTools.IncorrectParameter('IncorrectParameter: %s: %s' % (_("not a valid method"),method,) ) self.Cacher.sync(wait = True) done = f(match, dry_run) if done: self.SystemSettings.clear() # clear atomMatch cache anyway if clean_all_cache and not dry_run: self.clear_dump_cache(etpCache['world_available']) self.clear_dump_cache(etpCache['world_update']) self.clear_dump_cache(etpCache['check_package_update']) self.clear_dump_cache(etpCache['filter_satisfied_deps']) self.clear_dump_cache(self.atomMatchCacheKey) self.clear_dump_cache(etpCache['dep_tree']) self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],)) self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],)) self.package_match_validator_cache.clear() return done def unmask_match_by_atom(self, match, dry_run = False): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) atom = dbconn.retrieveAtom(m_id) return self.unmask_match_generic(match, atom, dry_run = dry_run) def unmask_match_by_keyslot(self, match, dry_run = False): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) return self.unmask_match_generic(match, keyslot, dry_run = dry_run) def mask_match_by_atom(self, match, dry_run = False): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) atom = dbconn.retrieveAtom(m_id) return self.mask_match_generic(match, atom, dry_run = dry_run) def mask_match_by_keyslot(self, match, dry_run = False): m_id, m_repo = match dbconn = self.openRepositoryDatabase(m_repo) keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) return self.mask_match_generic(match, keyslot, dry_run = dry_run) def unmask_match_generic(self, match, keyword, dry_run = False): self.clear_match_mask(match, dry_run) m_file = self.SystemSettings.etpSettingFiles['unmask'] return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) def mask_match_generic(self, match, keyword, dry_run = False): self.clear_match_mask(match, dry_run) m_file = self.SystemSettings.etpSettingFiles['mask'] return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) def _mask_unmask_match_generic(self, keyword, m_file, dry_run = False): exist = False if not os.path.isfile(m_file): if not os.access(os.path.dirname(m_file),os.W_OK): return False # cannot write elif not os.access(m_file, os.W_OK): return False elif not dry_run: exist = True if dry_run: return True content = [] if exist: f = open(m_file,"r") content = [x.strip() for x in f.readlines()] f.close() content.append(keyword) m_file_tmp = m_file+".tmp" f = open(m_file_tmp,"w") for line in content: f.write(line+"\n") f.flush() f.close() shutil.move(m_file_tmp,m_file) return True def clear_match_mask(self, match, dry_run = False): masking_list = [self.SystemSettings.etpSettingFiles['mask'],self.SystemSettings.etpSettingFiles['unmask']] return self._clear_match_generic(match, masking_list = masking_list, dry_run = dry_run) def _clear_match_generic(self, match, masking_list = [], dry_run = False): self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) if dry_run: return for mask_file in masking_list: if not (os.path.isfile(mask_file) and os.access(mask_file,os.W_OK)): continue f = open(mask_file,"r") newf = self.entropyTools.open_buffer() line = f.readline() while line: line = line.strip() if line.startswith("#"): newf.write(line+"\n") line = f.readline() continue elif not line: newf.write("\n") line = f.readline() continue mymatch = self.atomMatch(line, packagesFilter = False) if mymatch == match: line = f.readline() continue newf.write(line+"\n") line = f.readline() f.close() tmpfile = mask_file+".w_tmp" f = open(tmpfile,"w") f.write(newf.getvalue()) f.flush() f.close() newf.close() shutil.move(tmpfile,mask_file) def add_user_package_set(self, set_name, set_atoms): def _ensure_package_sets_dir(): sets_dir = etpConst['confsetsdir'] if not os.path.isdir(sets_dir): if os.path.lexists(sets_dir): os.remove(sets_dir) os.makedirs(sets_dir,0755) const_setup_perms(sets_dir, etpConst['entropygid']) try: set_name = str(set_name) except (UnicodeEncodeError,UnicodeDecodeError,): raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) if set_name.startswith(etpConst['packagesetprefix']): raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) set_match, rc = self.packageSetMatch(set_name) if rc: return -1,_("Name already taken") _ensure_package_sets_dir() set_file = os.path.join(etpConst['confsetsdir'],set_name) if os.path.isfile(set_file) and os.access(set_file,os.W_OK): try: os.remove(set_file) except OSError: return -2,_("Cannot remove the old element") if not os.access(os.path.dirname(set_file),os.W_OK): return -3,_("Cannot create the element") f = open(set_file,"w") for x in set_atoms: f.write("%s\n" % (x,)) f.flush() f.close() self.SystemSettings['system_package_sets'][set_name] = set(set_atoms) return 0,_("All fine") def remove_user_package_set(self, set_name): try: set_name = str(set_name) except (UnicodeEncodeError,UnicodeDecodeError,): raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) if set_name.startswith(etpConst['packagesetprefix']): raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) set_match, rc = self.packageSetMatch(set_name) if not rc: return -1,_("Already removed") set_id, set_x, set_y = set_match if set_id != etpConst['userpackagesetsid']: return -2,_("Not defined by user") set_file = os.path.join(etpConst['confsetsdir'],set_name) if os.path.isfile(set_file) and os.access(set_file,os.W_OK): os.remove(set_file) if set_name in self.SystemSettings['system_package_sets']: del self.SystemSettings['system_package_sets'][set_name] return 0,_("All fine") return -3,_("Set not found or unable to remove") # every tbz2 file that would be installed must pass from here def add_tbz2_to_repos(self, tbz2file): atoms_contained = [] basefile = os.path.basename(tbz2file) if os.path.isdir(etpConst['entropyunpackdir']+"/"+basefile[:-5]): shutil.rmtree(etpConst['entropyunpackdir']+"/"+basefile[:-5]) os.makedirs(etpConst['entropyunpackdir']+"/"+basefile[:-5]) dbfile = self.entropyTools.extractEdb(tbz2file, dbpath = etpConst['entropyunpackdir']+"/"+basefile[:-5]+"/packages.db") if dbfile == None: return -1,atoms_contained etpSys['dirstoclean'].add(os.path.dirname(dbfile)) # add dbfile repodata = {} repodata['repoid'] = basefile repodata['description'] = "Dynamic database from "+basefile repodata['packages'] = [] repodata['dbpath'] = os.path.dirname(dbfile) repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added repodata['smartpackage'] = False # extra info added mydbconn = self.openGenericDatabase(dbfile) # read all idpackages try: myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files except: return -2,atoms_contained if len(myidpackages) > 1: repodata[basefile]['smartpackage'] = True for myidpackage in myidpackages: compiled_arch = mydbconn.retrieveDownloadURL(myidpackage) if compiled_arch.find("/"+etpSys['arch']+"/") == -1: return -3,atoms_contained atoms_contained.append((int(myidpackage),basefile)) self.addRepository(repodata) self.validate_repositories() if basefile not in self.validRepositories: self.removeRepository(basefile) return -4,atoms_contained mydbconn.closeDB() del mydbconn return 0,atoms_contained # This is the function that should be used by third party applications # to retrieve a list of available updates, along with conflicts (removalQueue) and obsoletes # (removed) def retrieveWorldQueue(self, empty_deps = False, branch = etpConst['branch']): update, remove, fine = self.calculate_world_updates(empty_deps = empty_deps, branch = branch) del fine data = {} data['removed'] = list(remove) data['runQueue'] = [] data['removalQueue'] = [] status = -1 if update: # calculate install+removal queues install, removal, status = self.retrieveInstallQueue(update, empty_deps, deep_deps = False) # update data['removed'] data['removed'] = [x for x in data['removed'] if x not in removal] data['runQueue'] += install data['removalQueue'] += removal return data,status def validatePackageRemoval(self, idpackage): pkgatom = self.clientDbconn.retrieveAtom(idpackage) pkgkey = self.entropyTools.dep_getkey(pkgatom) if self.is_installed_idpackage_in_system_mask(idpackage): idpackages = self.SystemSettings['repos_system_mask_installed_keys'].get(pkgkey) if not idpackages: return False if len(idpackages) > 1: return True return False # sorry! # did we store the bastard in the db? system_pkg = self.clientDbconn.isSystemPackage(idpackage) if not system_pkg: return True # check if the package is slotted and exist more than one installed first sysresults = self.clientDbconn.atomMatch(pkgkey, multiMatch = True) if sysresults[1] == 0: if len(sysresults[0]) < 2: return False return True return False def retrieveRemovalQueue(self, idpackages, deep = False): queue = [] if not idpackages: return queue treeview, status = self.generate_depends_tree(idpackages, deep = deep) if status == 0: for x in range(len(treeview))[::-1]: queue.extend(treeview[x]) return queue def retrieveInstallQueue(self, matched_atoms, empty_deps, deep_deps, quiet = False): install = [] removal = [] treepackages, result = self.get_required_packages(matched_atoms, empty_deps, deep_deps, quiet = quiet) if result == -2: return treepackages,removal,result # format removal = treepackages.pop(0, set()) for x in sorted(treepackages.keys()): install.extend(treepackages[x]) # filter out packages that are in actionQueue comparing key + slot if install and removal: myremmatch = {} for x in removal: atom = self.clientDbconn.retrieveAtom(x) # XXX check if users removed idpackage while this whole instance is running if atom == None: continue myremmatch[(self.entropyTools.dep_getkey(atom),self.clientDbconn.retrieveSlot(x),)] = x for pkg_id, pkg_repo in install: dbconn = self.openRepositoryDatabase(pkg_repo) testtuple = (self.entropyTools.dep_getkey(dbconn.retrieveAtom(pkg_id)),dbconn.retrieveSlot(pkg_id)) removal.discard(myremmatch.get(testtuple)) return install, sorted(removal), 0 # this function searches into client database for a package matching provided key + slot # and returns its idpackage or -1 if none found def retrieveInstalledIdPackage(self, pkgkey, pkgslot): match = self.clientDbconn.atomMatch(pkgkey, matchSlot = pkgslot) if match[1] == 0: return match[0] return -1 ''' Package interface :: begin ''' def check_needed_package_download(self, filepath, checksum = None): # is the file available if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath): if checksum is None: return 0 else: # check digest md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum) if (md5res): return 0 else: return -2 else: return -1 def fetch_file(self, url, branch, digest = None, resume = True, fetch_file_abort_function = None, filepath = None): # remove old filename = os.path.basename(url) if not filepath: filepath = etpConst['packagesbindir']+"/"+branch+"/"+filename filepath_dir = os.path.dirname(filepath) if not os.path.isdir(filepath_dir): os.makedirs(filepath_dir,0755) # load class fetchConn = self.urlFetcher(url, filepath, resume = resume, abort_check_func = fetch_file_abort_function) fetchConn.progress = self.progress # start to download data_transfer = 0 resumed = False try: fetchChecksum = fetchConn.download() data_transfer = fetchConn.datatransfer resumed = fetchConn.resumed except KeyboardInterrupt: return -4, data_transfer, resumed except NameError: raise except: return -1, data_transfer, resumed if fetchChecksum == "-3": return -3, data_transfer, resumed del fetchConn if digest: if fetchChecksum != digest: # not properly downloaded return -2, data_transfer, resumed else: return 0, data_transfer, resumed return 0, data_transfer, resumed def add_failing_mirror(self, mirrorname,increment = 1): item = self.mirrorDownloadFailures.get(mirrorname) if item == None: self.mirrorDownloadFailures[mirrorname] = increment else: self.mirrorDownloadFailures[mirrorname] += increment # add a failure return self.mirrorDownloadFailures[mirrorname] def get_failing_mirror_status(self, mirrorname): item = self.mirrorDownloadFailures.get(mirrorname) if item == None: return 0 else: return item def fetch_file_on_mirrors(self, repository, branch, filename, digest = False, verified = False, fetch_abort_function = None): uris = etpRepositories[repository]['packages'][::-1] remaining = set(uris[:]) if verified: # file is already in place, match_checksum set infoDict['verified'] to True return 0 mirrorcount = 0 for uri in uris: if not remaining: # tried all the mirrors, quitting for error return 3 mirrorcount += 1 mirrorCountText = "( mirror #"+str(mirrorcount)+" ) " url = uri+"/"+filename # check if uri is sane if self.get_failing_mirror_status(uri) >= 30: # ohohoh! self.mirrorDownloadFailures[uri] = 30 # set to 30 for convenience mytxt = mirrorCountText mytxt += blue(" %s: ") % (_("Mirror"),) mytxt += red(self.entropyTools.spliturl(url)[1]) mytxt += " - %s." % (_("maximum failure threshold reached"),) self.updateProgress( mytxt, importance = 1, type = "warning", header = red(" ## ") ) if self.get_failing_mirror_status(uri) == 30: self.add_failing_mirror(uri,45) # put to 75 then decrement by 4 so we won't reach 30 anytime soon ahahaha else: # now decrement each time this point is reached, if will be back < 30, then equo will try to use it again if self.get_failing_mirror_status(uri) > 31: self.add_failing_mirror(uri,-4) else: # put to 0 - reenable mirror, welcome back uri! self.mirrorDownloadFailures[uri] = 0 if uri in remaining: remaining.remove(uri) continue do_resume = True while 1: try: mytxt = mirrorCountText mytxt += blue("%s: ") % (_("Downloading from"),) mytxt += red(self.entropyTools.spliturl(url)[1]) # now fetch the new one self.updateProgress( mytxt, importance = 1, type = "warning", header = red(" ## ") ) rc, data_transfer, resumed = self.fetch_file( url, branch, digest, do_resume, fetch_file_abort_function = fetch_abort_function ) if rc == 0: mytxt = mirrorCountText mytxt += blue("%s: ") % (_("Successfully downloaded from"),) mytxt += red(self.entropyTools.spliturl(url)[1]) mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),) self.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) return 0 elif resumed: do_resume = False continue else: error_message = mirrorCountText error_message += blue("%s: %s") % ( _("Error downloading from"), red(self.entropyTools.spliturl(url)[1]), ) # something bad happened if rc == -1: error_message += " - %s." % (_("file not available on this mirror"),) elif rc == -2: self.add_failing_mirror(uri,1) error_message += " - %s." % (_("wrong checksum"),) elif rc == -3: #self.add_failing_mirror(uri,2) error_message += " - not found." elif rc == -4: error_message += " - %s." % (_("discarded download"),) else: self.add_failing_mirror(uri, 5) error_message += " - %s." % (_("unknown reason"),) self.updateProgress( error_message, importance = 1, type = "warning", header = red(" ## ") ) if rc == -4: # user discarded fetch return 1 if uri in remaining: remaining.remove(uri) break except KeyboardInterrupt: break except: raise return 0 def quickpkg(self, atomstring, savedir = None): if savedir == None: savedir = etpConst['packagestmpdir'] if not os.path.isdir(etpConst['packagestmpdir']): os.makedirs(etpConst['packagestmpdir']) # match package match = self.clientDbconn.atomMatch(atomstring) if match[0] == -1: return -1,None,None atom = self.clientDbconn.atomMatch(match[0]) pkgdata = self.clientDbconn.getPackageData(match[0]) resultfile = self.quickpkg_handler(pkgdata = pkgdata, dirpath = savedir) if resultfile == None: return -1,atom,None else: return 0,atom,resultfile def quickpkg_handler( self, pkgdata, dirpath, edb = True, portdbPath = None, fake = False, compression = "bz2", shiftpath = "" ): import stat import tarfile if compression not in ("bz2","","gz"): compression = "bz2" # getting package info pkgtag = '' pkgrev = "~"+str(pkgdata['revision']) if pkgdata['versiontag']: pkgtag = "#"+pkgdata['versiontag'] pkgname = pkgdata['name']+"-"+pkgdata['version']+pkgrev+pkgtag # + version + tag pkgcat = pkgdata['category'] #pkgfile = pkgname+etpConst['packagesext'] dirpath += "/"+pkgname+etpConst['packagesext'] if os.path.isfile(dirpath): os.remove(dirpath) tar = tarfile.open(dirpath,"w:"+compression) if not fake: contents = sorted([x for x in pkgdata['content']]) id_strings = {} # collect files for path in contents: # convert back to filesystem str encoded_path = path path = path.encode('raw_unicode_escape') path = shiftpath+path try: exist = os.lstat(path) except OSError: continue # skip file arcname = path[len(shiftpath):] # remove shiftpath if arcname.startswith("/"): arcname = arcname[1:] # remove trailing / ftype = pkgdata['content'][encoded_path] if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype if 'dir' == ftype and \ not stat.S_ISDIR(exist.st_mode) and \ os.path.isdir(path): # workaround for directory symlink issues path = os.path.realpath(path) tarinfo = tar.gettarinfo(path, arcname) tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) if stat.S_ISREG(exist.st_mode): tarinfo.type = tarfile.REGTYPE f = open(path) try: tar.addfile(tarinfo, f) finally: f.close() else: tar.addfile(tarinfo) tar.close() # appending xpak metadata if etpConst['gentoo-compat']: import etpXpak Spm = self.Spm() gentoo_name = self.entropyTools.remove_tag(pkgname) gentoo_name = self.entropyTools.remove_entropy_revision(gentoo_name) if portdbPath == None: dbdir = Spm.get_vdb_path()+"/"+pkgcat+"/"+gentoo_name+"/" else: dbdir = portdbPath+"/"+pkgcat+"/"+gentoo_name+"/" if os.path.isdir(dbdir): tbz2 = etpXpak.tbz2(dirpath) tbz2.recompose(dbdir) if edb: self.inject_entropy_database_into_package(dirpath, pkgdata) if os.path.isfile(dirpath): return dirpath return None def inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None): dbpath = self.get_tmp_dbpath() dbconn = self.openGenericDatabase(dbpath) dbconn.initializeDatabase() dbconn.addPackage(data, revision = data['revision']) if treeupdates_actions != None: dbconn.bumpTreeUpdatesActions(treeupdates_actions) dbconn.commitChanges() dbconn.closeDB() self.entropyTools.aggregateEdb(tbz2file = package_filename, dbfile = dbpath) return dbpath def get_tmp_dbpath(self): dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) while os.path.isfile(dbpath): dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) return dbpath def Package(self): conn = PackageInterface(EquoInstance = self) return conn ''' Package interface :: end ''' ''' Source Package Manager Interface :: begin ''' def Spm(self): myroot = etpConst['systemroot'] cached = self.spmCache.get(myroot) if cached != None: return cached conn = SpmInterface(self) self.spmCache[myroot] = conn.intf return conn.intf def _extract_pkg_metadata_generate_extraction_dict(self): data = { 'chost': { 'path': etpConst['spm']['xpak_entries']['chost'], 'critical': True, }, 'description': { 'path': etpConst['spm']['xpak_entries']['description'], 'critical': False, }, 'homepage': { 'path': etpConst['spm']['xpak_entries']['homepage'], 'critical': False, }, 'slot': { 'path': etpConst['spm']['xpak_entries']['slot'], 'critical': False, }, 'cflags': { 'path': etpConst['spm']['xpak_entries']['cflags'], 'critical': False, }, 'cxxflags': { 'path': etpConst['spm']['xpak_entries']['cxxflags'], 'critical': False, }, 'category': { 'path': etpConst['spm']['xpak_entries']['category'], 'critical': True, }, 'rdepend': { 'path': etpConst['spm']['xpak_entries']['rdepend'], 'critical': False, }, 'pdepend': { 'path': etpConst['spm']['xpak_entries']['pdepend'], 'critical': False, }, 'depend': { 'path': etpConst['spm']['xpak_entries']['depend'], 'critical': False, }, 'use': { 'path': etpConst['spm']['xpak_entries']['use'], 'critical': False, }, 'iuse': { 'path': etpConst['spm']['xpak_entries']['iuse'], 'critical': False, }, 'license': { 'path': etpConst['spm']['xpak_entries']['license'], 'critical': False, }, 'provide': { 'path': etpConst['spm']['xpak_entries']['provide'], 'critical': False, }, 'sources': { 'path': etpConst['spm']['xpak_entries']['src_uri'], 'critical': False, }, 'eclasses': { 'path': etpConst['spm']['xpak_entries']['inherited'], 'critical': False, }, 'counter': { 'path': etpConst['spm']['xpak_entries']['counter'], 'critical': False, }, 'keywords': { 'path': etpConst['spm']['xpak_entries']['keywords'], 'critical': False, }, } return data def _extract_pkg_metadata_content(self, content_file, package_path): pkg_content = {} if os.path.isfile(content_file): f = open(content_file,"r") content = f.readlines() f.close() outcontent = set() for line in content: line = line.strip().split() try: datatype = line[0] datafile = line[1:] if datatype == 'obj': datafile = datafile[:-2] datafile = ' '.join(datafile) elif datatype == 'dir': datafile = ' '.join(datafile) elif datatype == 'sym': datafile = datafile[:-3] datafile = ' '.join(datafile) else: myexc = "InvalidData: %s %s. %s." % ( datafile, _("not supported"), _("Probably Portage API has changed"), ) raise exceptionTools.InvalidData(myexc) outcontent.add((datafile,datatype)) except: pass _outcontent = set() for i in outcontent: i = list(i) datatype = i[1] _outcontent.add((i[0],i[1])) outcontent = sorted(_outcontent) for i in outcontent: pkg_content[i[0]] = i[1] else: # CONTENTS is not generated when a package is emerged with portage and the option -B # we have to unpack the tbz2 and generate content dict mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject" if os.path.isdir(mytempdir): shutil.rmtree(mytempdir) if not os.path.isdir(mytempdir): os.makedirs(mytempdir) self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True) for currentdir, subdirs, files in os.walk(mytempdir): pkg_content[currentdir[len(mytempdir):]] = "dir" for item in files: item = currentdir+"/"+item if os.path.islink(item): pkg_content[item[len(mytempdir):]] = "sym" else: pkg_content[item[len(mytempdir):]] = "obj" # now remove shutil.rmtree(mytempdir,True) try: os.rmdir(mytempdir) except (OSError,): pass return pkg_content def _extract_pkg_metadata_needed(self, needed_file): pkg_needed = set() lines = [] try: f = open(needed_file,"r") lines = [x.strip() for x in f.readlines() if x.strip()] f.close() except IOError: return lines for line in lines: needed = line.split() if len(needed) == 2: ownlib = needed[0] ownelf = -1 if os.access(ownlib,os.R_OK): ownelf = self.entropyTools.read_elf_class(ownlib) for lib in needed[1].split(","): #if lib.find(".so") != -1: pkg_needed.add((lib,ownelf)) return list(pkg_needed) def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False): pkg_messages = [] if os.path.isdir(log_dir): elogfiles = os.listdir(log_dir) myelogfile = "%s:%s-%s" % (category, name, version,) foundfiles = [x for x in elogfiles if x.startswith(myelogfile)] if foundfiles: elogfile = foundfiles[0] if len(foundfiles) > 1: # get the latest mtimes = [] for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item)) mtimes = sorted(mtimes) elogfile = mtimes[-1][1] messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile)) for message in messages: message = message.replace("emerge","install") pkg_messages.append(message) elif not silent: mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),) self.updateProgress( red(log_dir)+mytxt, importance = 1, type = "warning", header = brown(" * ") ) return pkg_messages def _extract_pkg_metadata_license_data(self, licenses_dir, license_string): pkg_licensedata = {} if licenses_dir and os.path.isdir(licenses_dir): licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())] for mylicense in licdata: licfile = os.path.join(licenses_dir,mylicense) if os.access(licfile,os.R_OK): if self.entropyTools.istextfile(licfile): f = open(licfile) pkg_licensedata[mylicense] = f.read() f.close() return pkg_licensedata def _extract_pkg_metadata_mirror_links(self, Spm, sources_list): # =mirror://openoffice|link1|link2|link3 pkg_links = [] for i in sources_list: if i.startswith("mirror://"): # parse what mirror I need mirrorURI = i.split("/")[2] mirrorlist = Spm.get_third_party_mirrors(mirrorURI) pkg_links.append([mirrorURI,mirrorlist]) # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] return pkg_links def _extract_pkg_metadata_ebuild_entropy_tag(self, ebuild): search_tag = etpConst['spm']['ebuild_pkg_tag_var'] ebuild_tag = '' f = open(ebuild,"r") tags = [x.strip() for x in f.readlines() if x.strip() and x.strip().startswith(search_tag)] f.close() if not tags: return ebuild_tag tag = tags[-1] tag = tag.split("=")[-1].strip('"').strip("'").strip() return tag # This function extracts all the info from a .tbz2 file and returns them def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False): data = {} info_package = bold(os.path.basename(package))+": " if not silent: self.updateProgress( red(info_package+_("Extracting package metadata")+" ..."), importance = 0, type = "info", header = brown(" * "), back = True ) filepath = package tbz2File = package package = package.split(etpConst['packagesext'])[0] package = self.entropyTools.remove_entropy_revision(package) package = self.entropyTools.remove_tag(package) # remove entropy category if package.find(":") != -1: package = ':'.join(package.split(":")[1:]) # pkgcat is always == "null" here pkgcat, pkgname, pkgver, pkgrev = self.entropyTools.catpkgsplit(os.path.basename(package)) if pkgrev != "r0": pkgver += "-%s" % (pkgrev,) # Fill Package name and version data['name'] = pkgname data['version'] = pkgver data['digest'] = self.entropyTools.md5sum(tbz2File) data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File)) data['size'] = str(self.entropyTools.get_file_size(tbz2File)) tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/" if not os.path.isdir(tbz2TmpDir): if os.path.lexists(tbz2TmpDir): os.remove(tbz2TmpDir) os.makedirs(tbz2TmpDir) self.entropyTools.extractXpak(tbz2File,tbz2TmpDir) data['injected'] = False if inject: data['injected'] = True data['branch'] = etpBranch portage_entries = self._extract_pkg_metadata_generate_extraction_dict() for item in portage_entries: value = '' try: f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r") value = f.readline().strip() f.close() except IOError: if portage_entries[item]['critical']: raise data[item] = value # setup vars data['eclasses'] = data['eclasses'].split() try: data['counter'] = int(data['counter']) except ValueError: data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()] if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed']) data['needed'] = self._extract_pkg_metadata_needed(needed_file) content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents']) data['content'] = self._extract_pkg_metadata_content(content_file, filepath) data['disksize'] = self.entropyTools.sum_file_sizes(data['content']) # [][][] Kernel dependent packages hook [][][] data['versiontag'] = '' kernelstuff = False kernelstuff_kernel = False for item in data['content']: if item.startswith("/lib/modules/"): kernelstuff = True # get the version of the modules kmodver = item.split("/lib/modules/")[1] kmodver = kmodver.split("/")[0] lp = kmodver.split("-")[-1] if lp.startswith("r"): kname = kmodver.split("-")[-2] kver = kmodver.split("-")[0]+"-"+kmodver.split("-")[-1] else: kname = kmodver.split("-")[-1] kver = kmodver.split("-")[0] break # validate the results above if kernelstuff: matchatom = "linux-%s-%s" % (kname,kver,) if (matchatom == data['name']+"-"+data['version']): kernelstuff_kernel = True data['versiontag'] = kmodver if not kernelstuff_kernel: data['slot'] = kmodver # if you change this behaviour, # you must change "reagent update" # and "equo database gentoosync" consequentially file_ext = etpConst['spm']['ebuild_file_extension'] ebuilds_in_path = [x for x in os.listdir(tbz2TmpDir) if x.endswith(".%s" % (file_ext,))] if not data['versiontag'] and ebuilds_in_path: # has the user specified a custom package tag inside the ebuild ebuild_path = os.path.join(tbz2TmpDir,ebuilds_in_path[0]) data['versiontag'] = self._extract_pkg_metadata_ebuild_entropy_tag(ebuild_path) data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/" data['download'] += self.entropyTools.create_package_filename(data['category'], data['name'], data['version'], data['versiontag']) data['trigger'] = "" if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']): f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb") data['trigger'] = f.read() f.close() Spm = self.Spm() # Get Spm ChangeLog pkgatom = "%s/%s-%s" % (data['category'],data['name'],data['version'],) try: data['changelog'] = Spm.get_package_changelog(pkgatom) except: data['changelog'] = None portage_metadata = Spm.calculate_dependencies( data['iuse'], data['use'], data['license'], data['depend'], data['rdepend'], data['pdepend'], data['provide'], data['sources'] ) data['provide'] = portage_metadata['PROVIDE'].split() data['license'] = portage_metadata['LICENSE'] data['useflags'] = [] for x in data['use'].split(): if x.startswith("+"): x = x[1:] elif x.startswith("-"): x = x[1:] if (x in portage_metadata['USE']) or (x in portage_metadata['USE_MASK']): data['useflags'].append(x) else: data['useflags'].append("-"+x) data['sources'] = portage_metadata['SRC_URI'].split() data['dependencies'] = {} for x in portage_metadata['RDEPEND'].split(): if x.startswith("!") or (x in ("(","||",")","")): continue data['dependencies'][x] = 0 for x in portage_metadata['PDEPEND'].split(): if x.startswith("!") or (x in ("(","||",")","")): continue data['dependencies'][x] = etpConst['spm']['pdepend_id'] data['conflicts'] = [x[1:] for x in portage_metadata['RDEPEND'].split()+portage_metadata['PDEPEND'].split() if x.startswith("!") and not x in ("(","||",")","")] if (kernelstuff) and (not kernelstuff_kernel): # add kname to the dependency data['dependencies']["=sys-kernel/linux-"+kname+"-"+kver] = 0 # Conflicting tagged packages support key = data['category']+"/"+data['name'] confl_data = self.SystemSettings['conflicting_tagged_packages'].get(key) if confl_data != None: for conflict in confl_data: data['conflicts'].append(conflict) # Get License text if possible licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses') data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license']) data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources']) # write only if it's a systempackage data['systempackage'] = False system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()] if data['category']+"/"+data['name'] in system_packages: data['systempackage'] = True # write only if it's a systempackage protect, mask = Spm.get_config_protect_and_mask() data['config_protect'] = protect data['config_protect_mask'] = mask log_dir = etpConst['logdir']+"/elog" if not os.path.isdir(log_dir): os.makedirs(log_dir) data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent) data['etpapi'] = etpConst['etpapi'] # removing temporary directory shutil.rmtree(tbz2TmpDir,True) if os.path.isdir(tbz2TmpDir): try: os.remove(tbz2TmpDir) except OSError: pass if not silent: self.updateProgress( red(info_package+_("Package extraction complete")), importance = 0, type = "info", header = brown(" * "), back = True ) return data ''' Source Package Manager Interface :: end ''' def Triggers(self, *args, **kwargs): conn = TriggerInterface(self, *args, **kwargs) return conn def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True): conn = RepoInterface(EquoInstance = self, reponames = reponames, forceUpdate = forceUpdate, noEquoCheck = noEquoCheck, fetchSecurity = fetchSecurity) return conn def FileUpdatesInterfaceLoader(self): conn = FileUpdatesInterface(EquoInstance = self) return conn ''' Real package actions (install/remove) interface ''' class PackageInterface: import entropyTools def __init__(self, EquoInstance): if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid Equo instance or subclass is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.Entropy = EquoInstance self.infoDict = {} self.prepared = False self.matched_atom = () self.valid_actions = ("source","fetch","remove","remove_conflict","install","config") self.action = None self.fetch_abort_function = None self.xterm_title = '' def kill(self): self.infoDict.clear() self.matched_atom = () self.valid_actions = () self.action = None self.prepared = False self.fetch_abort_function = None def error_on_prepared(self): if self.prepared: mytxt = _("Already prepared") raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,)) def error_on_not_prepared(self): if not self.prepared: mytxt = _("Not yet prepared") raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,)) def check_action_validity(self, action): if action not in self.valid_actions: mytxt = _("Action must be in") raise exceptionTools.InvalidData("InvalidData: %s %s" % (mytxt,self.valid_actions,)) def match_checksum(self): self.error_on_not_prepared() dlcount = 0 match = False while dlcount <= 5: self.Entropy.updateProgress( blue(_("Checking package checksum...")), importance = 0, type = "info", header = red(" ## "), back = True ) dlcheck = self.Entropy.check_needed_package_download(self.infoDict['download'], checksum = self.infoDict['checksum']) if dlcheck == 0: self.Entropy.updateProgress( blue(_("Package checksum matches.")), importance = 0, type = "info", header = red(" ## ") ) self.infoDict['verified'] = True match = True break # file downloaded successfully else: dlcount += 1 self.Entropy.updateProgress( blue(_("Package checksum does not match. Redownloading... attempt #%s") % (dlcount,)), importance = 0, type = "info", header = red(" ## "), back = True ) fetch = self.Entropy.fetch_file_on_mirrors( self.infoDict['repository'], self.Entropy.get_branch_from_download_relative_uri(self.infoDict['download']), self.infoDict['download'], self.infoDict['checksum'], fetch_abort_function = self.fetch_abort_function ) if fetch != 0: self.Entropy.updateProgress( blue(_("Cannot properly fetch package! Quitting.")), importance = 0, type = "info", header = red(" ## ") ) return fetch self.infoDict['verified'] = True match = True break if (not match): mytxt = _("Cannot properly fetch package or checksum does not match. Try download latest repositories.") self.Entropy.updateProgress( blue(mytxt), importance = 0, type = "info", header = red(" ## ") ) return 1 return 0 ''' @description: unpack the given package file into the unpack dir @input infoDict: dictionary containing package information @output: 0 = all fine, >0 = error! ''' def __unpack_package(self): if not self.infoDict['merge_from']: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Unpacking package: "+str(self.infoDict['atom'])) else: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Merging package: "+str(self.infoDict['atom'])) if os.path.isdir(self.infoDict['unpackdir']): shutil.rmtree(self.infoDict['unpackdir'].encode('raw_unicode_escape')) elif os.path.isfile(self.infoDict['unpackdir']): os.remove(self.infoDict['unpackdir'].encode('raw_unicode_escape')) os.makedirs(self.infoDict['imagedir']) if not os.path.isfile(self.infoDict['pkgpath']) and not self.infoDict['merge_from']: if os.path.isdir(self.infoDict['pkgpath']): shutil.rmtree(self.infoDict['pkgpath']) if os.path.islink(self.infoDict['pkgpath']): os.remove(self.infoDict['pkgpath']) self.infoDict['verified'] = False rc = self.fetch_step() if rc != 0: return rc if not self.infoDict['merge_from']: unpack_tries = 3 while 1: unpack_tries -= 1 try: rc = self.entropyTools.spawnFunction( self.entropyTools.uncompressTarBz2, self.infoDict['pkgpath'], self.infoDict['imagedir'], catchEmpty = True ) except EOFError: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"EOFError on "+self.infoDict['pkgpath']) rc = 1 except (UnicodeEncodeError,UnicodeDecodeError,): # this will make devs to actually catch the right exception and prepare a fix self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Raising Unicode Error for "+self.infoDict['pkgpath']) rc = self.entropyTools.uncompressTarBz2( self.infoDict['pkgpath'],self.infoDict['imagedir'], catchEmpty = True ) if rc == 0: break if unpack_tries <= 0: return rc # otherwise, try to download it again self.infoDict['verified'] = False f_rc = self.fetch_step() if f_rc != 0: return f_rc else: pid = os.fork() if pid > 0: os.waitpid(pid, 0) else: self.__fill_image_dir(self.infoDict['merge_from'],self.infoDict['imagedir']) os._exit(0) # unpack xpak ? if etpConst['gentoo-compat']: if os.path.isdir(self.infoDict['xpakpath']): shutil.rmtree(self.infoDict['xpakpath']) try: os.rmdir(self.infoDict['xpakpath']) except OSError: pass # create data dir where we'll unpack the xpak os.makedirs(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'],0755) #os.mkdir(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']) xpakPath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakfilename'] if not self.infoDict['merge_from']: if (self.infoDict['smartpackage']): # we need to get the .xpak from database xdbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository']) xpakdata = xdbconn.retrieveXpakMetadata(self.infoDict['idpackage']) if xpakdata: # save into a file f = open(xpakPath,"wb") f.write(xpakdata) f.flush() f.close() self.infoDict['xpakstatus'] = self.entropyTools.unpackXpak( xpakPath, self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] ) else: self.infoDict['xpakstatus'] = None del xpakdata else: self.infoDict['xpakstatus'] = self.entropyTools.extractXpak( self.infoDict['pkgpath'], self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] ) else: # link xpakdir to self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] tolink_dir = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] if os.path.isdir(tolink_dir): shutil.rmtree(tolink_dir,True) # now link os.symlink(self.infoDict['xpakdir'],tolink_dir) # create fake portage ${D} linking it to imagedir portage_db_fakedir = os.path.join( self.infoDict['unpackdir'], "portage/"+self.infoDict['category'] + "/" + self.infoDict['name'] + "-" + self.infoDict['version'] ) os.makedirs(portage_db_fakedir,0755) # now link it to self.infoDict['imagedir'] os.symlink(self.infoDict['imagedir'],os.path.join(portage_db_fakedir,"image")) return 0 def __configure_package(self): try: Spm = self.Entropy.Spm() except: return 1 spm_atom = self.infoDict['key']+"-"+self.infoDict['version'] myebuild = Spm.get_vdb_path()+spm_atom+"/"+self.infoDict['key'].split("/")[1]+"-"+self.infoDict['version']+etpConst['spm']['source_build_ext'] if not os.path.isfile(myebuild): return 2 self.Entropy.updateProgress( brown(" Ebuild: pkg_config()"), importance = 0, header = red(" ##") ) try: rc = Spm.spm_doebuild( myebuild, mydo = "config", tree = "bintree", cpv = spm_atom ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot properly run Spm pkg_config() for " + \ str(spm_atom)+". Something bad happened." ) return 3 except Exception, e: self.entropyTools.printTraceback() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot run Spm pkg_config() for "+spm_atom+"!! "+str(type(Exception))+": "+str(e) ) mytxt = "%s: %s %s. %s. %s: %s, %s" % ( bold(_("QA")), brown(_("Cannot run Spm pkg_config() for")), bold(str(spm_atom)), brown(_("Please report it")), bold(_("Error")), type(Exception), e, ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return 1 return 0 def __remove_package(self): # clear on-disk cache self.__clear_cache() self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: %s" % (self.infoDict['removeatom'],)) # remove from database if self.infoDict['removeidpackage'] != -1: mytxt = "%s: " % (_("Removing from Entropy"),) self.Entropy.updateProgress( blue(mytxt) + red(self.infoDict['removeatom']), importance = 1, type = "info", header = red(" ## ") ) self.__remove_package_from_database() # Handle gentoo database if (etpConst['gentoo-compat']): gentooAtom = self.entropyTools.remove_tag(self.infoDict['removeatom']) self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing from Portage: "+str(gentooAtom)) self.__remove_package_from_gentoo_database(gentooAtom) del gentooAtom self.__remove_content_from_system() return 0 def __remove_content_from_system(self): # load CONFIG_PROTECT and its mask # client database at this point has been surely opened, # so our dicts are already filled protect = etpConst['dbconfigprotect'] mask = etpConst['dbconfigprotectmask'] sys_root = etpConst['systemroot'] col_protect = etpConst['collisionprotect'] # remove files from system directories = set() for item in self.infoDict['removecontent']: # collision check if col_protect > 0: if self.Entropy.clientDbconn.isFileAvailable(item) and os.path.isfile(sys_root+item): # in this way we filter out directories mytxt = red(_("Collision found during removal of")) + " " + sys_root+item + " - " mytxt += red(_("cannot overwrite")) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = red(" ## ") ) self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Collision found during remove of "+sys_root+item+" - cannot overwrite" ) continue protected = False if (not self.infoDict['removeconfig']) and (not self.infoDict['diffremoval']): protected_item_test = sys_root+item if isinstance(protected_item_test,unicode): protected_item_test = protected_item_test.encode('utf-8') protected, x, do_continue = self._handle_config_protect(protect, mask, None, protected_item_test, do_allocation_check = False) if do_continue: protected = True if protected: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_VERBOSE, "[remove] Protecting config file: "+sys_root+item ) mytxt = "[%s] %s: %s" % ( red(_("remove")), brown(_("Protecting config file")), sys_root+item, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = red(" ## ") ) else: try: os.lstat(sys_root+item) except OSError: continue # skip file, does not exist except UnicodeEncodeError: mytxt = brown(_("This package contains a badly encoded file !!!")) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) continue # file has a really bad encoding if os.path.isdir(sys_root+item) and os.path.islink(sys_root+item): # S_ISDIR returns False for directory symlinks, so using os.path.isdir # valid directory symlink directories.add((sys_root+item,"link")) elif os.path.isdir(sys_root+item): # plain directory directories.add((sys_root+item,"dir")) else: # files, symlinks or not # just a file or symlink or broken directory symlink (remove now) try: os.remove(sys_root+item) # add its parent directory dirfile = os.path.dirname(sys_root+item) if os.path.isdir(dirfile) and os.path.islink(dirfile): directories.add((dirfile,"link")) elif os.path.isdir(dirfile): directories.add((dirfile,"dir")) except OSError: pass # now handle directories directories = sorted(list(directories), reverse = True) while 1: taint = False for directory, dirtype in directories: mydir = "%s%s" % (sys_root,directory,) if dirtype == "link": try: mylist = os.listdir(mydir) if not mylist: try: os.remove(mydir) taint = True except OSError: pass except OSError: pass elif dirtype == "dir": try: mylist = os.listdir(mydir) if not mylist: try: os.rmdir(mydir) taint = True except OSError: pass except OSError: pass if not taint: break del directories ''' @description: remove package entry from Gentoo database @input gentoo package atom (cat/name+ver): @output: 0 = all fine, <0 = error! ''' def __remove_package_from_gentoo_database(self, atom): # handle gentoo-compat try: Spm = self.Entropy.Spm() except: return -1 # no Spm support ?? portDbDir = Spm.get_vdb_path() removePath = portDbDir+atom key = self.entropyTools.dep_getkey(atom) others_installed = Spm.search_keys(key) slot = self.infoDict['slot'] tag = self.infoDict['versiontag'] if (tag == slot) and tag: slot = "0" if os.path.isdir(removePath): shutil.rmtree(removePath,True) elif others_installed: for myatom in others_installed: myslot = Spm.get_installed_package_slot(myatom) if myslot != slot: continue shutil.rmtree(portDbDir+myatom,True) if not others_installed: world_file = Spm.get_world_file() world_file_tmp = world_file+".entropy.tmp" if os.access(world_file,os.W_OK) and os.path.isfile(world_file): new = open(world_file_tmp,"w") old = open(world_file,"r") line = old.readline() while line: if line.find(key) != -1: line = old.readline() continue if line.find(key+":"+slot) != -1: line = old.readline() continue new.write(line) line = old.readline() new.flush() new.close() old.close() shutil.move(world_file_tmp,world_file) return 0 ''' @description: function that runs at the end of the package installation process, just removes data left by other steps @output: 0 = all fine, >0 = error! ''' def _cleanup_package(self, unpack_dir): # remove unpack dir shutil.rmtree(unpack_dir,True) try: os.rmdir(unpack_dir) except OSError: pass return 0 def __remove_package_from_database(self): self.error_on_not_prepared() self.Entropy.clientDbconn.removePackage(self.infoDict['removeidpackage']) return 0 def __clear_cache(self): self.Entropy.clear_dump_cache(etpCache['advisories']) self.Entropy.clear_dump_cache(etpCache['filter_satisfied_deps']) self.Entropy.clear_dump_cache(etpCache['depends_tree']) self.Entropy.clear_dump_cache(etpCache['check_package_update']) self.Entropy.clear_dump_cache(etpCache['dep_tree']) self.Entropy.clear_dump_cache(etpCache['dbMatch']+etpConst['clientdbid']+"/") self.Entropy.clear_dump_cache(etpCache['dbSearch']+etpConst['clientdbid']+"/") self.__update_available_cache() try: self.__update_world_cache() except: self.Entropy.clear_dump_cache(etpCache['world_update']) def __update_world_cache(self): if self.Entropy.xcache and (self.action in ("install","remove",)): wc_dir = os.path.dirname(os.path.join(etpConst['dumpstoragedir'],etpCache['world_update'])) wc_filename = os.path.basename(etpCache['world_update']) wc_cache_files = [os.path.join(wc_dir,x) for x in os.listdir(wc_dir) if x.startswith(wc_filename)] for cache_file in wc_cache_files: try: data = self.Entropy.dumpTools.loadobj(cache_file, completePath = True) (update, remove, fine) = data['r'] empty_deps = data['empty_deps'] except: self.Entropy.clear_dump_cache(etpCache['world_update']) return if empty_deps: continue if self.action == "install": if self.matched_atom in update: update.remove(self.matched_atom) self.Entropy.dumpTools.dumpobj( cache_file, {'r':(update, remove, fine),'empty_deps': empty_deps}, completePath = True ) else: key, slot = self.Entropy.clientDbconn.retrieveKeySlot(self.infoDict['removeidpackage']) matches = self.Entropy.atomMatch(key, matchSlot = slot, multiMatch = True, multiRepo = True) if matches[1] != 0: # hell why! better to rip all off self.Entropy.clear_dump_cache(etpCache['world_update']) return taint = False for match in matches[0]: if match in update: taint = True update.remove(match) if match in remove: taint = True remove.remove(match) if taint: self.Entropy.dumpTools.dumpobj( cache_file, {'r':(update, remove, fine),'empty_deps': empty_deps}, completePath = True ) elif (not self.Entropy.xcache) or (self.action in ("install",)): self.Entropy.clear_dump_cache(etpCache['world_update']) def __update_available_cache(self): # update world available cache if self.Entropy.xcache and (self.action in ("remove","install")): disk_cache = self.Entropy.Cacher.pop(etpCache['world_available']) if disk_cache != None: c_hash = self.Entropy.get_available_packages_chash(etpConst['branch']) try: if disk_cache['chash'] == c_hash: # remove and old install if self.infoDict['removeidpackage'] != -1: taint = False key = self.entropyTools.dep_getkey(self.infoDict['removeatom']) slot = self.infoDict['slot'] matches = self.Entropy.atomMatch(key, matchSlot = slot, multiRepo = True, multiMatch = True) if matches[1] == 0: for mymatch in matches[0]: if mymatch not in disk_cache['available']: disk_cache['available'].append(mymatch) taint = True if taint: mydata = {} mylist = [] for myidpackage,myrepo in disk_cache['available']: mydbc = self.Entropy.openRepositoryDatabase(myrepo) mydata[mydbc.retrieveAtom(myidpackage)] = (myidpackage,myrepo) mykeys = sorted(mydata.keys()) for mykey in mykeys: mylist.append(mydata[mykey]) disk_cache['available'] = mylist # install, doing here because matches[0] could contain self.matched_atoms if self.matched_atom in disk_cache['available']: disk_cache['available'].remove(self.matched_atom) self.Entropy.Cacher.push(etpCache['world_available'],disk_cache.copy()) except KeyError: self.Entropy.Cacher.push(etpCache['world_available'],{}) elif not self.Entropy.xcache: self.Entropy.clear_dump_cache(etpCache['world_available']) ''' @description: install unpacked files, update database and also update gentoo db if requested @output: 0 = all fine, >0 = error! ''' def __install_package(self): # clear on-disk cache self.__clear_cache() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Installing package: %s" % (self.infoDict['atom'],) ) # copy files over - install # use fork? (in this case all the changed structures need to be pushed back) rc = self.__move_image_to_system() if rc != 0: return rc # inject into database mytxt = "%s: %s" % (blue(_("Updating database")),red(self.infoDict['atom']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) newidpackage = self._install_package_into_database() # remove old files and gentoo stuff if (self.infoDict['removeidpackage'] != -1): # doing a diff removal self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Remove old package: %s" % (self.infoDict['removeatom'],) ) self.infoDict['removeidpackage'] = -1 # disabling database removal if etpConst['gentoo-compat']: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Removing Entropy and Gentoo database entry for %s" % (self.infoDict['removeatom'],) ) else: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Removing Entropy (only) database entry for %s" % (self.infoDict['removeatom'],) ) self.Entropy.updateProgress( blue(_("Cleaning old package files...")), importance = 1, type = "info", header = red(" ## ") ) self.__remove_package() rc = 0 if etpConst['gentoo-compat']: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Installing new Gentoo database entry: %s" % (self.infoDict['atom'],) ) rc = self._install_package_into_gentoo_database(newidpackage) return rc ''' @description: inject the database information into the Gentoo database @output: 0 = all fine, !=0 = error! ''' def _install_package_into_gentoo_database(self, newidpackage): # handle gentoo-compat try: Spm = self.Entropy.Spm() except: return -1 # no Portage support portDbDir = Spm.get_vdb_path() if os.path.isdir(portDbDir): # extract xpak from unpackDir+etpConst['packagecontentdir']+"/"+package key = self.infoDict['category']+"/"+self.infoDict['name'] atomsfound = set() dbdirs = os.listdir(portDbDir) if self.infoDict['category'] in dbdirs: catdirs = os.listdir(portDbDir+"/"+self.infoDict['category']) dirsfound = set([self.infoDict['category']+"/"+x for x in catdirs if key == self.entropyTools.dep_getkey(self.infoDict['category']+"/"+x)]) atomsfound.update(dirsfound) ### REMOVE # parse slot and match and remove if atomsfound: pkgToRemove = '' for atom in atomsfound: atomslot = Spm.get_installed_package_slot(atom) # get slot from gentoo db if atomslot == self.infoDict['slot']: pkgToRemove = atom break if (pkgToRemove): removePath = portDbDir+pkgToRemove shutil.rmtree(removePath,True) try: os.rmdir(removePath) except OSError: pass del atomsfound # we now install it if ((self.infoDict['xpakstatus'] != None) and \ os.path.isdir( self.infoDict['xpakpath'] + "/" + etpConst['entropyxpakdatarelativepath'])) or \ self.infoDict['merge_from']: if self.infoDict['merge_from']: copypath = self.infoDict['xpakdir'] if not os.path.isdir(copypath): return 0 else: copypath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] if not os.path.isdir(portDbDir+self.infoDict['category']): os.makedirs(portDbDir+self.infoDict['category'],0755) destination = portDbDir+self.infoDict['category']+"/"+self.infoDict['name']+"-"+self.infoDict['version'] if os.path.isdir(destination): shutil.rmtree(destination) try: shutil.copytree(copypath,destination) except (IOError,), e: mytxt = "%s: %s: %s: %s" % (red(_("QA")),brown(_("Cannot update Portage database to destination")),purple(destination),e,) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) # test if /var/cache/edb/counter is fine if os.path.isfile(etpConst['edbcounter']): try: f = open(etpConst['edbcounter'],"r") counter = int(f.readline().strip()) f.close() except: # need file recreation, parse gentoo tree counter = Spm.refill_counter() else: counter = Spm.refill_counter() # write new counter to file if os.path.isdir(destination): counter += 1 f = open(destination+"/"+etpConst['spm']['xpak_entries']['counter'],"w") f.write(str(counter)) f.flush() f.close() f = open(etpConst['edbcounter'],"w") f.write(str(counter)) f.flush() f.close() # update counter inside clientDatabase self.Entropy.clientDbconn.insertCounter(newidpackage,counter) else: mytxt = brown(_("Cannot update Portage counter, destination %s does not exist.") % (destination,)) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) # add to Portage world # key: key # slot: self.infoDict['slot'] myslot = self.infoDict['slot'] if (self.infoDict['versiontag'] == self.infoDict['slot']) and self.infoDict['versiontag']: # usually kernel packages myslot = "0" keyslot = key+":"+myslot world_file = Spm.get_world_file() world_atoms = set() if os.access(world_file,os.R_OK) and os.path.isfile(world_file): f = open(world_file,"r") world_atoms = set([x.strip() for x in f.readlines() if x.strip()]) f.close() else: mytxt = brown(_("Cannot update Portage world file, destination %s does not exist.") % (world_file,)) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) return 0 try: if keyslot not in world_atoms and \ os.access(os.path.dirname(world_file),os.W_OK) and \ self.entropyTools.istextfile(world_file): world_atoms.discard(key) world_atoms.add(keyslot) world_atoms = sorted(list(world_atoms)) world_file_tmp = world_file+".entropy_inst" f = open(world_file_tmp,"w") for item in world_atoms: f.write(item+"\n") f.flush() f.close() shutil.move(world_file_tmp,world_file) except (UnicodeDecodeError,UnicodeEncodeError), e: self.entropyTools.printTraceback(f = self.Entropy.clientLog) mytxt = brown(_("Cannot update Portage world file, destination %s is corrupted.") % (world_file,)) self.Entropy.updateProgress( red("QA: ")+mytxt+": "+unicode(e), importance = 1, type = "warning", header = darkred(" ## ") ) return 0 ''' @description: injects package info into the installed packages database @output: 0 = all fine, >0 = error! ''' def _install_package_into_database(self): # fetch info dbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository']) data = dbconn.getPackageData(self.infoDict['idpackage'], content_insert_formatted = True) # open client db # always set data['injected'] to False # installed packages database SHOULD never have more than one package for scope (key+slot) data['injected'] = False data['counter'] = -1 # gentoo counter will be set in self._install_package_into_gentoo_database() idpackage, rev, x = self.Entropy.clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision'], formattedContent = True) # update datecreation ctime = self.entropyTools.getCurrentUnixTime() self.Entropy.clientDbconn.setDateCreation(idpackage, str(ctime)) # add idpk to the installedtable self.Entropy.clientDbconn.removePackageFromInstalledTable(idpackage) self.Entropy.clientDbconn.addPackageToInstalledTable(idpackage,self.infoDict['repository']) # clear depends table, this will make clientdb dependstable to be regenerated during the next request (retrieveDepends) self.Entropy.clientDbconn.clearDependsTable() return idpackage def __fill_image_dir(self, mergeFrom, imageDir): dbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository']) package_content = dbconn.retrieveContent(self.infoDict['idpackage'], extended = True, formatted = True) contents = sorted(package_content) # collect files for path in contents: # convert back to filesystem str encoded_path = path path = os.path.join(mergeFrom,encoded_path[1:]) topath = os.path.join(imageDir,encoded_path[1:]) path = path.encode('raw_unicode_escape') topath = topath.encode('raw_unicode_escape') try: exist = os.lstat(path) except OSError: continue # skip file ftype = package_content[encoded_path] if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype if 'dir' == ftype and \ not stat.S_ISDIR(exist.st_mode) and \ os.path.isdir(path): # workaround for directory symlink issues path = os.path.realpath(path) copystat = False # if our directory is a symlink instead, then copy the symlink if os.path.islink(path): tolink = os.readlink(path) if os.path.islink(topath): os.remove(topath) os.symlink(tolink,topath) elif os.path.isdir(path): if not os.path.isdir(topath): os.makedirs(topath) copystat = True elif os.path.isfile(path): if os.path.isfile(topath): os.remove(topath) # should never happen shutil.copy2(path,topath) copystat = True if copystat: user = os.stat(path)[stat.ST_UID] group = os.stat(path)[stat.ST_GID] os.chown(topath,user,group) shutil.copystat(path,topath) def __move_image_to_system(self): # load CONFIG_PROTECT and its mask protect = etpRepositories[self.infoDict['repository']]['configprotect'] mask = etpRepositories[self.infoDict['repository']]['configprotectmask'] sys_root = etpConst['systemroot'] col_protect = etpConst['collisionprotect'] items_installed = set() # setup imageDir properly imageDir = self.infoDict['imagedir'] encoded_imageDir = imageDir.encode('utf-8') # merge data into system for currentdir,subdirs,files in os.walk(encoded_imageDir): # create subdirs for subdir in subdirs: imagepathDir = "%s/%s" % (currentdir,subdir,) rootdir = "%s%s" % (sys_root,imagepathDir[len(imageDir):],) # handle broken symlinks if os.path.islink(rootdir) and not os.path.exists(rootdir):# broken symlink os.remove(rootdir) # if our directory is a file on the live system elif os.path.isfile(rootdir): # really weird...! self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "WARNING!!! %s is a file when it should be a directory !! Removing in 20 seconds..." % (rootdir,) ) mytxt = darkred(_("%s is a file when should be a directory !! Removing in 20 seconds...") % (rootdir,)) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = red(" !!! ") ) self.entropyTools.ebeep(20) os.remove(rootdir) # if our directory is a symlink instead, then copy the symlink if os.path.islink(imagepathDir) and not os.path.isdir(rootdir): # for security we skip live items that are dirs tolink = os.readlink(imagepathDir) if os.path.islink(rootdir): os.remove(rootdir) os.symlink(tolink,rootdir) elif (not os.path.isdir(rootdir)) and (not os.access(rootdir,os.R_OK)): try: # we should really force a simple mkdir first of all os.mkdir(rootdir) except OSError: os.makedirs(rootdir) if not os.path.islink(rootdir) and os.access(rootdir,os.W_OK): # symlink don't need permissions, also until os.walk ends they might be broken # XXX also, added os.access() check because there might be directories/files unwriteable # what to do otherwise? user = os.stat(imagepathDir)[stat.ST_UID] group = os.stat(imagepathDir)[stat.ST_GID] os.chown(rootdir,user,group) shutil.copystat(imagepathDir,rootdir) items_installed.add(os.path.join(os.path.realpath(os.path.dirname(rootdir)),os.path.basename(rootdir))) for item in files: fromfile = "%s/%s" % (currentdir,item,) tofile = "%s%s" % (sys_root,fromfile[len(imageDir):],) if col_protect > 1: todbfile = fromfile[len(imageDir):] myrc = self._handle_install_collision_protect(tofile, todbfile) if not myrc: continue protected, tofile, do_continue = self._handle_config_protect(protect, mask, fromfile, tofile) if do_continue: continue try: if os.path.realpath(fromfile) == os.path.realpath(tofile) and os.path.islink(tofile): # there is a serious issue here, better removing tofile, happened to someone: try: # try to cope... os.remove(tofile) except OSError: pass # if our file is a dir on the live system if os.path.isdir(tofile) and not os.path.islink(tofile): # really weird...! self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "WARNING!!! %s is a directory when it should be a file !! Removing in 20 seconds..." % (tofile,) ) mytxt = _("%s is a directory when it should be a file !! Removing in 20 seconds...") % (tofile,) self.Entropy.updateProgress( red("QA: ")+darkred(mytxt), importance = 1, type = "warning", header = red(" !!! ") ) self.entropyTools.ebeep(10) time.sleep(20) try: shutil.rmtree(tofile, True) os.rmdir(tofile) except: pass try: # if it was a link os.remove(tofile) except OSError: pass # XXX # XXX moving file using the raw format like portage does # XXX done = self.entropyTools.movefile(fromfile, tofile, src_basedir = encoded_imageDir) if not done: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "WARNING!!! Error during file move to system: %s => %s" % (fromfile,tofile,) ) mytxt = "%s: %s => %s, %s" % (_("File move error"),fromfile,tofile,_("please report"),) self.Entropy.updateProgress( red("QA: ")+darkred(mytxt), importance = 1, type = "warning", header = red(" !!! ") ) return 4 except IOError, e: # try to move forward, sometimes packages might be # fucked up and contain broken things if e.errno != 2: raise items_installed.add(os.path.join(os.path.realpath(os.path.dirname(tofile)),os.path.basename(tofile))) if protected: # add to disk cache self.Entropy.FileUpdates.add_to_cache(tofile, quiet = True) # this is useful to avoid the removal of installed files by __remove_package just because # there's a difference in the directory path, perhaps, which is not handled correctly by # EntropyDatabaseInterface.contentDiff for obvious reasons (think about stuff in /usr/lib and /usr/lib64, # where the latter is just a symlink to the former) if self.infoDict.get('removecontent'): my_remove_content = set([x for x in self.infoDict['removecontent'] \ if os.path.join(os.path.realpath( os.path.dirname("%s%s" % (sys_root,x,))),os.path.basename(x) ) in items_installed]) self.infoDict['removecontent'] -= my_remove_content return 0 def _handle_config_protect(self, protect, mask, fromfile, tofile, do_allocation_check = True): protected = False tofile_before_protect = tofile do_continue = False try: encoded_protect = [x.encode('raw_unicode_escape') for x in protect] if tofile in encoded_protect: protected = True elif os.path.dirname(tofile) in encoded_protect: protected = True else: tofile_testdir = os.path.dirname(tofile) old_tofile_testdir = None while tofile_testdir != old_tofile_testdir: if tofile_testdir in encoded_protect: protected = True break old_tofile_testdir = tofile_testdir tofile_testdir = os.path.dirname(tofile_testdir) if protected: # check if perhaps, file is masked, so unprotected newmask = [x.encode('raw_unicode_escape') for x in mask] if tofile in newmask: protected = False elif os.path.dirname(tofile) in newmask: protected = False if not os.path.lexists(tofile): protected = False # file doesn't exist # check if it's a text file if (protected) and os.path.isfile(tofile): protected = self.entropyTools.istextfile(tofile) else: protected = False # it's not a file # request new tofile then if protected: if tofile not in etpConst['configprotectskip']: prot_status = True if do_allocation_check: tofile, prot_status = self.entropyTools.allocateMaskedFile(tofile, fromfile) if not prot_status: protected = False else: oldtofile = tofile if oldtofile.find("._cfg") != -1: oldtofile = os.path.dirname(oldtofile)+"/"+os.path.basename(oldtofile)[10:] self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Protecting config file: %s" % (oldtofile,) ) mytxt = red("%s: %s") % (_("Protecting config file"),oldtofile,) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) else: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Skipping config file installation/removal, as stated in equo.conf: %s" % (tofile,) ) mytxt = "%s: %s" % (_("Skipping file installation/removal"),tofile,) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) do_continue = True except Exception, e: self.entropyTools.printTraceback() protected = False # safely revert to false tofile = tofile_before_protect mytxt = darkred("%s: %s") % (_("Cannot check CONFIG PROTECTION. Error"),e,) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) return protected, tofile, do_continue def _handle_install_collision_protect(self, tofile, todbfile): avail = self.Entropy.clientDbconn.isFileAvailable(todbfile, get_id = True) if (self.infoDict['removeidpackage'] not in avail) and avail: mytxt = darkred(_("Collision found during install for")) mytxt += " %s - %s" % (blue(tofile),darkred(_("cannot overwrite")),) self.Entropy.updateProgress( red("QA: ")+mytxt, importance = 1, type = "warning", header = darkred(" ## ") ) self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "WARNING!!! Collision found during install for %s - cannot overwrite" % (tofile,) ) return False return True def sources_fetch_step(self): self.error_on_not_prepared() down_data = self.infoDict['download'] rc = 0 for key in sorted(down_data.keys()): # first fine wins for url in down_data[key]: dest_file = os.path.join(self.infoDict['unpackdir'],os.path.basename(url)) rc = self._fetch_source(url, dest_file) if not rc: break if rc: break return rc def _fetch_source(self, url, dest_file): try: mytxt = "%s: %s" % (blue(_("Downloading")),brown(url),) # now fetch the new one self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc, data_transfer, resumed = self.Entropy.fetch_file( url, None, None, False, fetch_file_abort_function = self.fetch_abort_function, filepath = dest_file ) if rc == 0: mytxt = blue("%s: ") % (_("Successfully downloaded from"),) mytxt += red(self.entropyTools.spliturl(url)[1]) mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) self.Entropy.updateProgress( "%s: %s" % (blue(_("Local path")),brown(dest_file),), importance = 1, type = "info", header = red(" # ") ) return 0 else: error_message = blue("%s: %s") % ( _("Error downloading from"), red(self.entropyTools.spliturl(url)[1]), ) # something bad happened if rc == -1: error_message += " - %s." % (_("file not available on this mirror"),) elif rc == -3: error_message += " - not found." elif rc == -4: error_message += " - %s." % (_("discarded download"),) else: error_message += " - %s: %s" % (_("unknown reason"),rc,) self.Entropy.updateProgress( error_message, importance = 1, type = "warning", header = red(" ## ") ) if rc == -4: # user discarded fetch return 1 except KeyboardInterrupt: return 1 def fetch_step(self): self.error_on_not_prepared() mytxt = "%s: %s" % (blue(_("Downloading archive")),red(os.path.basename(self.infoDict['download'])),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc = self.Entropy.fetch_file_on_mirrors( self.infoDict['repository'], self.Entropy.get_branch_from_download_relative_uri(self.infoDict['download']), self.infoDict['download'], self.infoDict['checksum'], self.infoDict['verified'], fetch_abort_function = self.fetch_abort_function ) if rc != 0: mytxt = "%s. %s: %s" % ( red(_("Package cannot be fetched. Try to update repositories and retry")), blue(_("Error")), rc, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = darkred(" ## ") ) return rc return 0 def fetch_not_available_step(self): self.Entropy.updateProgress( blue(_("Fetch for the chosen package is not available, unknown error.")), importance = 1, type = "info", header = red(" ## ") ) return 0 def vanished_step(self): self.Entropy.updateProgress( blue(_("Installed package in queue vanished, skipping.")), importance = 1, type = "info", header = red(" ## ") ) return 0 def checksum_step(self): self.error_on_not_prepared() rc = self.match_checksum() return rc def unpack_step(self): self.error_on_not_prepared() if not self.infoDict['merge_from']: mytxt = "%s: %s" % (blue(_("Unpacking package")),red(os.path.basename(self.infoDict['download'])),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) else: mytxt = "%s: %s" % (blue(_("Merging package")),red(os.path.basename(self.infoDict['atom'])),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc = self.__unpack_package() if rc != 0: if rc == 512: errormsg = "%s. %s. %s: 512" % ( red(_("You are running out of disk space")), red(_("I bet, you're probably Michele")), blue(_("Error")), ) else: errormsg = "%s. %s. %s: %s" % ( red(_("An error occured while trying to unpack the package")), red(_("Check if your system is healthy")), blue(_("Error")), rc, ) self.Entropy.updateProgress( errormsg, importance = 1, type = "error", header = red(" ## ") ) return rc def install_step(self): self.error_on_not_prepared() mytxt = "%s: %s" % (blue(_("Installing package")),red(self.infoDict['atom']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc = self.__install_package() if rc != 0: mytxt = "%s. %s. %s: %s" % ( red(_("An error occured while trying to install the package")), red(_("Check if your system is healthy")), blue(_("Error")), rc, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = red(" ## ") ) return rc def remove_step(self): self.error_on_not_prepared() mytxt = "%s: %s" % (blue(_("Removing data")),red(self.infoDict['removeatom']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc = self.__remove_package() if rc != 0: mytxt = "%s. %s. %s: %s" % ( red(_("An error occured while trying to remove the package")), red(_("Check if you have enough disk space on your hard disk")), blue(_("Error")), rc, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = red(" ## ") ) return rc def cleanup_step(self): self.error_on_not_prepared() mytxt = "%s: %s" % (blue(_("Cleaning")),red(self.infoDict['atom']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) self._cleanup_package(self.infoDict['unpackdir']) # we don't care if cleanupPackage fails since it's not critical return 0 def logmessages_step(self): for msg in self.infoDict['messages']: self.Entropy.clientLog.write(">>> "+msg) return 0 def messages_step(self): self.error_on_not_prepared() # get messages if self.infoDict['messages']: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "Message from %s:" % (self.infoDict['atom'],) ) mytxt = "%s:" % (darkgreen(_("Compilation messages")),) self.Entropy.updateProgress( mytxt, importance = 0, type = "warning", header = brown(" ## ") ) for msg in self.infoDict['messages']: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,msg) self.Entropy.updateProgress( msg, importance = 0, type = "warning", header = brown(" ## ") ) if self.infoDict['messages']: self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"End message.") def postinstall_step(self): self.error_on_not_prepared() pkgdata = self.infoDict['triggers'].get('install') if pkgdata: Trigger = self.Entropy.Triggers('postinstall',pkgdata, self.action) Trigger.prepare() Trigger.run() Trigger.kill() del Trigger del pkgdata return 0 def preinstall_step(self): self.error_on_not_prepared() pkgdata = self.infoDict['triggers'].get('install') if pkgdata: Trigger = self.Entropy.Triggers('preinstall',pkgdata, self.action) Trigger.prepare() if self.infoDict.get("diffremoval"): # diffremoval is true only when the # removal is triggered by a package install remdata = self.infoDict['triggers'].get('remove') if remdata: rTrigger = self.Entropy.Triggers('preremove',remdata, self.action) rTrigger.prepare() Trigger.triggers = [x for x in Trigger.triggers if x not in rTrigger.triggers] rTrigger.kill() del rTrigger del remdata Trigger.run() Trigger.kill() del Trigger del pkgdata return 0 def preremove_step(self): self.error_on_not_prepared() remdata = self.infoDict['triggers'].get('remove') if remdata: Trigger = self.Entropy.Triggers('preremove',remdata, self.action) Trigger.prepare() Trigger.run() Trigger.kill() del Trigger del remdata return 0 def postremove_step(self): self.error_on_not_prepared() remdata = self.infoDict['triggers'].get('remove') if remdata: Trigger = self.Entropy.Triggers('postremove',remdata, self.action) Trigger.prepare() if self.infoDict['diffremoval'] and (self.infoDict.get("atom") != None): # diffremoval is true only when the remove action is triggered by installPackages() pkgdata = self.infoDict['triggers'].get('install') if pkgdata: iTrigger = self.Entropy.Triggers('postinstall',pkgdata, self.action) iTrigger.prepare() Trigger.triggers = [x for x in Trigger.triggers if x not in iTrigger.triggers] iTrigger.kill() del iTrigger del pkgdata Trigger.run() Trigger.kill() del Trigger del remdata return 0 def removeconflict_step(self): self.error_on_not_prepared() for idpackage in self.infoDict['conflicts']: if not self.Entropy.clientDbconn.isIDPackageAvailable(idpackage): continue Package = self.Entropy.Package() Package.prepare((idpackage,),"remove_conflict", self.infoDict['remove_metaopts']) rc = Package.run(xterm_header = self.xterm_title) Package.kill() if rc != 0: return rc return 0 def config_step(self): self.error_on_not_prepared() mytxt = "%s: %s" % (blue(_("Configuring package")),red(self.infoDict['atom']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" ## ") ) rc = self.__configure_package() if rc == 1: mytxt = "%s. %s. %s: %s" % ( red(_("An error occured while trying to configure the package")), red(_("Make sure that your system is healthy")), blue(_("Error")), rc, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = red(" ## ") ) elif rc == 2: mytxt = "%s. %s. %s: %s" % ( red(_("An error occured while trying to configure the package")), red(_("It seems that the Source Package Manager entry is missing")), blue(_("Error")), rc, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = red(" ## ") ) return rc def run_stepper(self, xterm_header): if xterm_header == None: xterm_header = "" if self.infoDict.has_key('remove_installed_vanished'): self.xterm_title += ' Installed package vanished' self.Entropy.setTitle(self.xterm_title) rc = self.vanished_step() return rc if self.infoDict.has_key('fetch_not_available'): self.xterm_title += ' Fetch not available' self.Entropy.setTitle(self.xterm_title) rc = self.fetch_not_available_step() return rc def do_fetch(): self.xterm_title += ' %s: %s' % (_("Fetching"),os.path.basename(self.infoDict['download']),) self.Entropy.setTitle(self.xterm_title) return self.fetch_step() def do_sources_fetch(): self.xterm_title += ' %s: %s' % (_("Fetching sources"),os.path.basename(self.infoDict['atom']),) self.Entropy.setTitle(self.xterm_title) return self.sources_fetch_step() def do_checksum(): self.xterm_title += ' %s: %s' % (_("Verifying"),os.path.basename(self.infoDict['download']),) self.Entropy.setTitle(self.xterm_title) return self.checksum_step() def do_unpack(): if not self.infoDict['merge_from']: mytxt = _("Unpacking") self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['download']),) else: mytxt = _("Merging") self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['atom']),) self.Entropy.setTitle(self.xterm_title) return self.unpack_step() def do_remove_conflicts(): return self.removeconflict_step() def do_install(): self.xterm_title += ' %s: %s' % (_("Installing"),self.infoDict['atom'],) self.Entropy.setTitle(self.xterm_title) return self.install_step() def do_remove(): self.xterm_title += ' %s: %s' % (_("Removing"),self.infoDict['removeatom'],) self.Entropy.setTitle(self.xterm_title) return self.remove_step() def do_showmessages(): return self.messages_step() def do_logmessages(): return self.logmessages_step() def do_cleanup(): self.xterm_title += ' %s: %s' % (_("Cleaning"),self.infoDict['atom'],) self.Entropy.setTitle(self.xterm_title) return self.cleanup_step() def do_postinstall(): self.xterm_title += ' %s: %s' % (_("Postinstall"),self.infoDict['atom'],) self.Entropy.setTitle(self.xterm_title) return self.postinstall_step() def do_preinstall(): self.xterm_title += ' %s: %s' % (_("Preinstall"),self.infoDict['atom'],) self.Entropy.setTitle(self.xterm_title) return self.preinstall_step() def do_preremove(): self.xterm_title += ' %s: %s' % (_("Preremove"),self.infoDict['removeatom'],) self.Entropy.setTitle(self.xterm_title) return self.preremove_step() def do_postremove(): self.xterm_title += ' %s: %s' % (_("Postremove"),self.infoDict['removeatom'],) self.Entropy.setTitle(self.xterm_title) return self.postremove_step() def do_config(): self.xterm_title += ' %s: %s' % (_("Configuring"),self.infoDict['atom'],) self.Entropy.setTitle(self.xterm_title) return self.config_step() steps_data = { "fetch": do_fetch, "sources_fetch": do_sources_fetch, "checksum": do_checksum, "unpack": do_unpack, "remove_conflicts": do_remove_conflicts, "install": do_install, "remove": do_remove, "showmessages": do_showmessages, "logmessages": do_logmessages, "cleanup": do_cleanup, "postinstall": do_postinstall, "preinstall": do_preinstall, "postremove": do_postremove, "preremove": do_preremove, "config": do_config, } rc = 0 for step in self.infoDict['steps']: self.xterm_title = xterm_header rc = steps_data.get(step)() if rc != 0: break return rc ''' @description: execute the requested steps @input xterm_header: purely optional ''' def run(self, xterm_header = None): self.error_on_not_prepared() gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock) if gave_up: return 20 locked = self.Entropy.application_lock_check() if locked: self.Entropy._resources_run_remove_lock() return 21 # lock self.Entropy._resources_run_create_lock() try: rc = self.run_stepper(xterm_header) except: self.Entropy._resources_run_remove_lock() raise # remove lock self.Entropy._resources_run_remove_lock() if rc != 0: self.Entropy.updateProgress( blue(_("An error occured. Action aborted.")), importance = 2, type = "error", header = darkred(" ## ") ) return rc ''' Install/Removal process preparation function - will generate all the metadata needed to run the action steps, creating infoDict automatically @input matched_atom(tuple): is what is returned by EquoInstance.atomMatch: (idpackage,repoid): (2000,u'sabayonlinux.org') NOTE: in case of remove action, matched_atom must be: (idpackage,) @input action(string): is an action to take, which must be one in self.valid_actions ''' def prepare(self, matched_atom, action, metaopts = {}): self.error_on_prepared() self.check_action_validity(action) self.action = action self.matched_atom = matched_atom self.metaopts = metaopts # generate metadata dictionary self.generate_metadata() def generate_metadata(self): self.error_on_prepared() self.check_action_validity(self.action) if self.action == "fetch": self.__generate_fetch_metadata() elif self.action in ("remove","remove_conflict"): self.__generate_remove_metadata() elif self.action == "install": self.__generate_install_metadata() elif self.action == "source": self.__generate_fetch_metadata(sources = True) elif self.action == "config": self.__generate_config_metadata() self.prepared = True def __generate_remove_metadata(self): self.infoDict.clear() idpackage = self.matched_atom[0] if not self.Entropy.clientDbconn.isIDPackageAvailable(idpackage): self.infoDict['remove_installed_vanished'] = True return 0 self.infoDict['triggers'] = {} self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage) self.infoDict['slot'] = self.Entropy.clientDbconn.retrieveSlot(idpackage) self.infoDict['versiontag'] = self.Entropy.clientDbconn.retrieveVersionTag(idpackage) self.infoDict['removeidpackage'] = idpackage self.infoDict['diffremoval'] = False removeConfig = False if self.metaopts.has_key('removeconfig'): removeConfig = self.metaopts.get('removeconfig') self.infoDict['removeconfig'] = removeConfig self.infoDict['removecontent'] = self.Entropy.clientDbconn.retrieveContent(idpackage) self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getTriggerInfo(idpackage) self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent'] self.infoDict['steps'] = [] self.infoDict['steps'].append("preremove") self.infoDict['steps'].append("remove") self.infoDict['steps'].append("postremove") return 0 def __generate_config_metadata(self): self.infoDict.clear() idpackage = self.matched_atom[0] self.infoDict['atom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage) key, slot = self.Entropy.clientDbconn.retrieveKeySlot(idpackage) self.infoDict['key'], self.infoDict['slot'] = key, slot self.infoDict['version'] = self.Entropy.clientDbconn.retrieveVersion(idpackage) self.infoDict['steps'] = [] self.infoDict['steps'].append("config") return 0 def __generate_install_metadata(self): self.infoDict.clear() idpackage, repository = self.matched_atom self.infoDict['idpackage'] = idpackage self.infoDict['repository'] = repository # fetch abort function if self.metaopts.has_key('fetch_abort_function'): self.fetch_abort_function = self.metaopts.pop('fetch_abort_function') # get package atom dbconn = self.Entropy.openRepositoryDatabase(repository) self.infoDict['triggers'] = {} self.infoDict['atom'] = dbconn.retrieveAtom(idpackage) self.infoDict['slot'] = dbconn.retrieveSlot(idpackage) self.infoDict['version'], self.infoDict['versiontag'], self.infoDict['revision'] = dbconn.getVersioningData(idpackage) self.infoDict['category'] = dbconn.retrieveCategory(idpackage) self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage) self.infoDict['name'] = dbconn.retrieveName(idpackage) self.infoDict['messages'] = dbconn.retrieveMessages(idpackage) self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage) self.infoDict['accept_license'] = dbconn.retrieveLicensedataKeys(idpackage) self.infoDict['conflicts'] = self.Entropy.get_match_conflicts(self.matched_atom) # fill action queue self.infoDict['removeidpackage'] = -1 removeConfig = False if self.metaopts.has_key('removeconfig'): removeConfig = self.metaopts.get('removeconfig') self.infoDict['remove_metaopts'] = { 'removeconfig': True, } if self.metaopts.has_key('remove_metaopts'): self.infoDict['remove_metaopts'] = self.metaopts.get('remove_metaopts') self.infoDict['merge_from'] = None mf = self.metaopts.get('merge_from') if mf != None: self.infoDict['merge_from'] = unicode(mf) self.infoDict['removeconfig'] = removeConfig self.infoDict['removeidpackage'] = self.Entropy.retrieveInstalledIdPackage( self.entropyTools.dep_getkey(self.infoDict['atom']), self.infoDict['slot'] ) if self.infoDict['removeidpackage'] != -1: avail = self.Entropy.clientDbconn.isIDPackageAvailable(self.infoDict['removeidpackage']) if avail: self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage']) else: self.infoDict['removeidpackage'] = -1 # smartpackage ? self.infoDict['smartpackage'] = False # set unpack dir and image dir if self.infoDict['repository'].endswith(etpConst['packagesext']): # do arch check compiled_arch = dbconn.retrieveDownloadURL(idpackage) if compiled_arch.find("/"+etpSys['arch']+"/") == -1: self.infoDict.clear() self.prepared = False return -1 self.infoDict['smartpackage'] = etpRepositories[self.infoDict['repository']]['smartpackage'] self.infoDict['pkgpath'] = etpRepositories[self.infoDict['repository']]['pkgpath'] else: self.infoDict['pkgpath'] = etpConst['entropyworkdir']+"/"+self.infoDict['download'] self.infoDict['unpackdir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download'] self.infoDict['imagedir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyimagerelativepath'] # gentoo xpak data if etpConst['gentoo-compat']: self.infoDict['xpakpath'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyxpakrelativepath'] if not self.infoDict['merge_from']: self.infoDict['xpakstatus'] = None self.infoDict['xpakdir'] = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'] else: self.infoDict['xpakstatus'] = True portdbdir = 'var/db/pkg' # XXX hard coded ? portdbdir = os.path.join(self.infoDict['merge_from'],portdbdir) portdbdir = os.path.join(portdbdir,self.infoDict['category']) portdbdir = os.path.join(portdbdir,self.infoDict['name']+"-"+self.infoDict['version']) self.infoDict['xpakdir'] = portdbdir # compare both versions and if they match, disable removeidpackage if self.infoDict['removeidpackage'] != -1: installedVer, installedTag, installedRev = self.Entropy.clientDbconn.getVersioningData(self.infoDict['removeidpackage']) pkgcmp = self.entropyTools.entropyCompareVersions( (self.infoDict['version'], self.infoDict['versiontag'], self.infoDict['revision'],), (installedVer, installedTag, installedRev,) ) if pkgcmp == 0: self.infoDict['removeidpackage'] = -1 else: # differential remove list self.infoDict['diffremoval'] = True self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage']) self.infoDict['removecontent'] = self.Entropy.clientDbconn.contentDiff( self.infoDict['removeidpackage'], dbconn, idpackage ) self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getTriggerInfo( self.infoDict['removeidpackage'] ) self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent'] # set steps self.infoDict['steps'] = [] if self.infoDict['conflicts']: self.infoDict['steps'].append("remove_conflicts") # install self.infoDict['steps'].append("unpack") # preinstall placed before preremove in order # to respect Spm order self.infoDict['steps'].append("preinstall") if (self.infoDict['removeidpackage'] != -1): self.infoDict['steps'].append("preremove") self.infoDict['steps'].append("install") if (self.infoDict['removeidpackage'] != -1): self.infoDict['steps'].append("postremove") self.infoDict['steps'].append("postinstall") if not etpConst['gentoo-compat']: # otherwise gentoo triggers will show that self.infoDict['steps'].append("showmessages") else: self.infoDict['steps'].append("logmessages") self.infoDict['steps'].append("cleanup") self.infoDict['triggers']['install'] = dbconn.getTriggerInfo(idpackage) self.infoDict['triggers']['install']['accept_license'] = self.infoDict['accept_license'] self.infoDict['triggers']['install']['unpackdir'] = self.infoDict['unpackdir'] if etpConst['gentoo-compat']: #self.infoDict['triggers']['install']['xpakpath'] = self.infoDict['xpakpath'] self.infoDict['triggers']['install']['xpakdir'] = self.infoDict['xpakdir'] return 0 def __generate_fetch_metadata(self, sources = False): self.infoDict.clear() idpackage, repository = self.matched_atom dochecksum = True # fetch abort function if self.metaopts.has_key('fetch_abort_function'): self.fetch_abort_function = self.metaopts.pop('fetch_abort_function') if self.metaopts.has_key('dochecksum'): dochecksum = self.metaopts.get('dochecksum') self.infoDict['repository'] = repository self.infoDict['idpackage'] = idpackage dbconn = self.Entropy.openRepositoryDatabase(repository) self.infoDict['atom'] = dbconn.retrieveAtom(idpackage) if sources: self.infoDict['download'] = dbconn.retrieveSources(idpackage, extended = True) else: self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage) self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage) if not self.infoDict['download']: self.infoDict['fetch_not_available'] = True return 0 self.infoDict['verified'] = False self.infoDict['steps'] = [] if not repository.endswith(etpConst['packagesext']) and not sources: if self.Entropy.check_needed_package_download(self.infoDict['download'], None) < 0: self.infoDict['steps'].append("fetch") if dochecksum: self.infoDict['steps'].append("checksum") elif sources: self.infoDict['steps'].append("sources_fetch") if sources: # create sources destination directory unpack_dir = etpConst['entropyunpackdir']+"/sources/"+self.infoDict['atom'] self.infoDict['unpackdir'] = unpack_dir if os.path.lexists(unpack_dir): if os.path.isfile(unpack_dir): os.remove(unpack_dir) elif os.path.isdir(unpack_dir): shutil.rmtree(unpack_dir,True) if not os.path.lexists(unpack_dir): os.makedirs(unpack_dir,0775) const_setup_perms(unpack_dir,etpConst['entropygid']) else: # if file exists, first checksum then fetch if os.path.isfile(os.path.join(etpConst['entropyworkdir'],self.infoDict['download'])): # check size first repo_size = dbconn.retrieveSize(idpackage) f = open(os.path.join(etpConst['entropyworkdir'],self.infoDict['download']),"r") f.seek(0,2) disk_size = f.tell() f.close() if repo_size == disk_size: self.infoDict['steps'].reverse() return 0 class FileUpdatesInterface: def __init__(self, EquoInstance): if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid Equo instance or subclass is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.Entropy = EquoInstance self.scandata = None def merge_file(self, key): self.scanfs(dcache = True) self.do_backup(key) if os.access(etpConst['systemroot'] + self.scandata[key]['source'], os.R_OK): shutil.move( etpConst['systemroot'] + self.scandata[key]['source'], etpConst['systemroot'] + self.scandata[key]['destination'] ) self.remove_from_cache(key) def remove_file(self, key): self.scanfs(dcache = True) try: os.remove(etpConst['systemroot'] + self.scandata[key]['source']) except OSError: pass self.remove_from_cache(key) def do_backup(self, key): self.scanfs(dcache = True) if etpConst['filesbackup'] and os.path.isfile(etpConst['systemroot']+self.scandata[key]['destination']): bcount = 0 backupfile = etpConst['systemroot'] + \ os.path.dirname(self.scandata[key]['destination']) + \ "/._equo_backup." + unicode(bcount) + "_" + \ os.path.basename(self.scandata[key]['destination']) while os.path.lexists(backupfile): bcount += 1 backupfile = etpConst['systemroot'] + \ os.path.dirname(self.scandata[key]['destination']) + \ "/._equo_backup." + unicode(bcount) + "_" + \ os.path.basename(self.scandata[key]['destination']) try: shutil.copy2(etpConst['systemroot'] + self.scandata[key]['destination'],backupfile) except IOError: pass ''' @description: scan for files that need to be merged @output: dictionary using filename as key ''' def scanfs(self, dcache = True, quiet = False): if dcache: if self.scandata != None: return self.scandata # can we load cache? try: z = self.load_cache() if z != None: self.scandata = z return self.scandata except: pass # open client database to fill etpConst['dbconfigprotect'] scandata = {} counter = 0 name_cache = set() for path in etpConst['dbconfigprotect']: # it's a file? scanfile = False if os.path.isfile(path): # find inside basename path = os.path.dirname(path) scanfile = True for currentdir,subdirs,files in os.walk(path): for item in files: if scanfile: if path != item: continue filepath = os.path.join(currentdir,item) if item.startswith("._cfg"): # further check then number = item[5:9] try: int(number) except ValueError: continue # not a valid etc-update file if item[9] != "_": # no valid format provided continue if filepath in name_cache: continue # skip, already done name_cache.add(filepath) mydict = self.generate_dict(filepath) if mydict['automerge']: if not quiet: mytxt = _("Automerging file") self.Entropy.updateProgress( darkred("%s: %s") % ( mytxt, darkgreen(etpConst['systemroot'] + mydict['source']), ), importance = 0, type = "info" ) if os.path.isfile(etpConst['systemroot']+mydict['source']): try: shutil.move( etpConst['systemroot']+mydict['source'], etpConst['systemroot']+mydict['destination'] ) except IOError, e: if not quiet: mytxt = "%s :: %s: %s. %s: %s" % ( red(_("I/O Error")), red(_("Cannot automerge file")), brown(etpConst['systemroot'] + mydict['source']), blue("Error"), e, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning" ) continue else: counter += 1 scandata[counter] = mydict.copy() if not quiet: try: self.Entropy.updateProgress( "("+blue(str(counter))+") " + red(" file: ") + \ os.path.dirname(filepath) + "/" + os.path.basename(filepath)[10:], importance = 1, type = "info" ) except: pass # possible encoding issues # store data self.Entropy.Cacher.push(etpCache['configfiles'],scandata.copy()) self.scandata = scandata.copy() return scandata def load_cache(self): sd = self.Entropy.Cacher.pop(etpCache['configfiles']) if not isinstance(sd,dict): raise exceptionTools.CacheCorruptionError("CacheCorruptionError") # quick test if data is reliable try: name_cache = set() for x in sd: mysource = sd[x]['source'] # filter dupies if mysource in name_cache: sd.pop(x) continue if not os.path.isfile(etpConst['systemroot']+mysource): raise exceptionTools.CacheCorruptionError("CacheCorruptionError") name_cache.add(mysource) return sd except (KeyError,EOFError,IOError,): raise exceptionTools.CacheCorruptionError("CacheCorruptionError") ''' @description: prints information about config files that should be updated @attention: please be sure that filepath is properly formatted before using this function ''' def add_to_cache(self, filepath, quiet = False): self.scanfs(dcache = True, quiet = quiet) keys = self.scandata.keys() try: for key in keys: if self.scandata[key]['source'] == filepath[len(etpConst['systemroot']):]: del self.scandata[key] except: pass # get next counter if keys: keys = sorted(keys) index = keys[-1] else: index = 0 index += 1 mydata = self.generate_dict(filepath) self.scandata[index] = mydata.copy() self.Entropy.Cacher.push(etpCache['configfiles'],self.scandata.copy()) def remove_from_cache(self, key): self.scanfs(dcache = True) try: del self.scandata[key] except: pass self.Entropy.Cacher.push(etpCache['configfiles'],self.scandata.copy()) return self.scandata def generate_dict(self, filepath): item = os.path.basename(filepath) currentdir = os.path.dirname(filepath) tofile = item[10:] number = item[5:9] try: int(number) except: mytxt = _("Invalid config file number") raise exceptionTools.InvalidDataType("InvalidDataType: %s '0000->9999'." % (mytxt,)) tofilepath = currentdir+"/"+tofile mydict = {} mydict['revision'] = number mydict['destination'] = tofilepath[len(etpConst['systemroot']):] mydict['source'] = filepath[len(etpConst['systemroot']):] mydict['automerge'] = False if not os.path.isfile(tofilepath): mydict['automerge'] = True if (not mydict['automerge']): # is it trivial? try: if not os.path.lexists(filepath): # if file does not even exist return mydict if os.path.islink(filepath): # if it's broken, skip diff and automerge if not os.path.exists(filepath): return mydict result = commands.getoutput('diff -Nua "%s" "%s" | grep "^[+-][^+-]" | grep -v \'# .Header:.*\'' % (filepath,tofilepath,)) if not result: mydict['automerge'] = True except: pass # another test if (not mydict['automerge']): try: if not os.path.lexists(filepath): # if file does not even exist return mydict if os.path.islink(filepath): # if it's broken, skip diff and automerge if not os.path.exists(filepath): return mydict result = subprocess.call('diff -Bbua "%s" "%s" | egrep \'^[+-]\' | egrep -v \'^[+-][\t ]*#|^--- |^\+\+\+ \' | egrep -qv \'^[-+][\t ]*$\'' % (filepath,tofilepath,), shell = True) if result == 1: mydict['automerge'] = True except: pass return mydict # # repository control class, that's it # class RepoInterface: import dumpTools import entropyTools import socket def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True): self.LockScanner = None if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid Equo instance or subclass is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.supported_download_items = ( "db","rev","ck", "lock","mask","system_mask","dbdump", "conflicting_tagged", "dbdumpck","lic_whitelist","make.conf", "package.mask","package.unmask","package.keywords","profile.link", "package.use","server.cert","ca.cert", "notice_board" ) self.big_socket_timeout = 25 self.Entropy = EquoInstance self.dbapi2 = dbapi2 self.reponames = reponames self.forceUpdate = forceUpdate self.syncErrors = False self.dbupdated = False self.newEquo = False self.fetchSecurity = fetchSecurity self.noEquoCheck = noEquoCheck self.alreadyUpdated = 0 self.notAvailable = 0 self.valid_eapis = [1,2,3] self.reset_dbformat_eapi(None) self.current_repository_got_locked = False self.updated_repos = set() # check etpRepositories if not etpRepositories: mytxt = _("No repositories specified in %s") % (etpConst['repositoriesconf'],) raise exceptionTools.MissingParameter("MissingParameter: %s" % (mytxt,)) if not self.reponames: self.reponames.extend(etpRepositories.keys()[:]) def __del__(self): if self.LockScanner != None: self.LockScanner.kill() def get_eapi3_connection(self, repository): # get database url dburl = etpRepositories[repository]['plain_database'] if dburl.startswith("file://"): return None try: dburl = dburl.split("/")[2] except IndexError: return None port = etpRepositories[repository]['service_port'] try: eapi3_socket = SystemSocketClientInterface( self.Entropy, RepositorySocketClientCommands, output_header = "\t" ) eapi3_socket.socket_timeout = self.big_socket_timeout eapi3_socket.connect(dburl, port) return eapi3_socket except (exceptionTools.ConnectionError,self.socket.error,): return None def check_eapi3_availability(self, repository): conn = self.get_eapi3_connection(repository) if conn == None: return False try: conn.disconnect() except (self.socket.error,AttributeError,): return False return True def reset_dbformat_eapi(self, repository): self.dbformat_eapi = 2 if repository != None: eapi_avail = self.check_eapi3_availability(repository) if eapi_avail: self.dbformat_eapi = 3 # FIXME, find a way to do that without needing sqlite3 exec. if not os.access("/usr/bin/sqlite3",os.X_OK) or self.entropyTools.islive(): self.dbformat_eapi = 1 else: rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True) if rc != 0: self.dbformat_eapi = 1 eapi_env = os.getenv("FORCE_EAPI") if eapi_env != None: try: myeapi = int(eapi_env) except (ValueError,TypeError,): return if myeapi in self.valid_eapis: self.dbformat_eapi = myeapi def __validate_repository_id(self, repoid): if repoid not in self.reponames: mytxt = _("Repository is not listed in self.reponames") raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,)) def __validate_compression_method(self, repo): self.__validate_repository_id(repo) cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat']) if cmethod == None: mytxt = _("Wrong database compression method") raise exceptionTools.InvalidDataType("InvalidDataType: %s" % (mytxt,)) return cmethod def __ensure_repository_path(self, repo): self.__validate_repository_id(repo) # create dir if it doesn't exist if not os.path.isdir(etpRepositories[repo]['dbpath']): os.makedirs(etpRepositories[repo]['dbpath'],0775) const_setup_perms(etpConst['etpdatabaseclientdir'],etpConst['entropygid']) def _construct_paths(self, item, repo, cmethod): if item not in self.supported_download_items: mytxt = _("Supported items: %s") % (self.supported_download_items,) raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,)) if (item in ("db","dbdump", "dbdumpck",)) and (cmethod == None): mytxt = _("For %s, cmethod can't be None") % (item,) raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,)) if item == "db": url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[2]] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[2]] elif item == "dbdump": url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[3]] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[3]] elif item == "rev": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile'] elif item == "ck": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile'] elif item == "dbdumpck": url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[4]] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[4]] elif item == "mask": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasemaskfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasemaskfile'] elif item == "system_mask": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasesytemmaskfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasesytemmaskfile'] elif item == "conflicting_tagged": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaseconflictingtaggedfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaseconflictingtaggedfile'] elif item == "make.conf": myfile = os.path.basename(etpConst['spm']['global_make_conf']) url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "package.mask": myfile = os.path.basename(etpConst['spm']['global_package_mask']) url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "package.unmask": myfile = os.path.basename(etpConst['spm']['global_package_unmask']) url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "package.keywords": myfile = os.path.basename(etpConst['spm']['global_package_keywords']) url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "package.use": myfile = os.path.basename(etpConst['spm']['global_package_use']) url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "profile.link": myfile = etpConst['spm']['global_make_profile_link_name'] url = etpRepositories[repo]['database'] + "/" + myfile filepath = etpRepositories[repo]['dbpath'] + "/" + myfile elif item == "lic_whitelist": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaselicwhitelistfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaselicwhitelistfile'] elif item == "lock": url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile'] filepath = "/dev/null" elif item == "server.cert": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasecacertfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasecacertfile'] elif item == "ca.cert": url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaseservercertfile'] filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaseservercertfile'] elif item == "notice_board": url = etpRepositories[repo]['notice_board'] filepath = etpRepositories[repo]['dbpath'] + "/" + os.path.basename(etpRepositories[repo]['notice_board']) return url, filepath def __remove_repository_files(self, repo, cmethod): dbfilenameid = cmethod[2] self.__validate_repository_id(repo) if self.dbformat_eapi == 1: if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']): os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]): os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]) if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) elif self.dbformat_eapi == 2: if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+cmethod[4]): os.remove(etpRepositories[repo]['dbpath']+"/"+cmethod[4]) if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]): os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]) if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) else: mytxt = _("self.dbformat_eapi must be in (1,2)") raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,)) def __unpack_downloaded_database(self, repo, cmethod): self.__validate_repository_id(repo) rc = 0 path = None if self.dbformat_eapi == 1: myfile = etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]] try: path = eval("self.entropyTools."+cmethod[1])(myfile) except EOFError: rc = 1 if os.path.isfile(myfile): os.remove(myfile) elif self.dbformat_eapi == 2: myfile = etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]] try: path = eval("self.entropyTools."+cmethod[1])(myfile) except EOFError: rc = 1 if os.path.isfile(myfile): os.remove(myfile) else: mytxt = _("self.dbformat_eapi must be in (1,2)") raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,)) if rc == 0: self.Entropy.setup_default_file_perms(path) return rc def __verify_database_checksum(self, repo, cmethod = None): self.__validate_repository_id(repo) if self.dbformat_eapi == 1: dbfile = etpConst['etpdatabasefile'] try: f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") md5hash = f.readline().strip() md5hash = md5hash.split()[0] f.close() except: return -1 elif self.dbformat_eapi == 2: dbfile = etpConst[cmethod[3]] try: f = open(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[4]],"r") md5hash = f.readline().strip() md5hash = md5hash.split()[0] f.close() except: return -1 else: mytxt = _("self.dbformat_eapi must be in (1,2)") raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,)) rc = self.entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+dbfile,md5hash) return rc # @returns -1 if the file is not available # @returns int>0 if the revision has been retrieved def get_online_repository_revision(self, repo): self.__validate_repository_id(repo) url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile'] status = self.entropyTools.get_remote_data(url) if (status): status = status[0].strip() try: status = int(status) except ValueError: status = -1 return status else: return -1 def get_online_eapi3_lock(self, repo): self.__validate_repository_id(repo) url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaseeapi3lockfile'] data = self.entropyTools.get_remote_data(url) if not data: return False return True def is_repository_eapi3_locked(self, repo): self.__validate_repository_id(repo) return self.get_online_eapi3_lock(repo) def is_repository_updatable(self, repo): self.__validate_repository_id(repo) onlinestatus = self.get_online_repository_revision(repo) if (onlinestatus != -1): localstatus = self.Entropy.get_repository_revision(repo) if (localstatus == onlinestatus) and (not self.forceUpdate): return False return True def is_repository_unlocked(self, repo): self.__validate_repository_id(repo) rc = self.download_item("lock", repo, disallow_redirect = True) if rc: # cannot download database self.syncErrors = True return False return True def clear_repository_cache(self, repo): self.__validate_repository_id(repo) self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repo,)) self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repo,)) # this function can be reimplemented def download_item(self, item, repo, cmethod = None, lock_status_func = None, disallow_redirect = True): self.__validate_repository_id(repo) url, filepath = self._construct_paths(item, repo, cmethod) # to avoid having permissions issues # it's better to remove the file before, # otherwise new permissions won't be written if os.path.isfile(filepath): os.remove(filepath) fetchConn = self.Entropy.urlFetcher( url, filepath, resume = False, abort_check_func = lock_status_func, disallow_redirect = disallow_redirect ) fetchConn.progress = self.Entropy.progress rc = fetchConn.download() del fetchConn if rc in ("-1","-2","-3"): return False self.Entropy.setup_default_file_perms(filepath) return True def check_downloaded_database(self, repo, cmethod): dbfilename = etpConst['etpdatabasefile'] if self.dbformat_eapi == 2: dbfilename = etpConst[cmethod[3]] # verify checksum mytxt = "%s %s %s" % (red(_("Checking downloaded database")),darkgreen(dbfilename),red("...")) self.Entropy.updateProgress( mytxt, importance = 0, back = True, type = "info", header = "\t" ) db_status = self.__verify_database_checksum(repo, cmethod) if db_status == -1: mytxt = "%s. %s !" % (red(_("Cannot open digest")),red(_("Cannot verify database integrity")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) elif db_status: mytxt = "%s: %s" % (red(_("Downloaded database status")),bold(_("OK")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t" ) else: mytxt = "%s: %s" % (red(_("Downloaded database status")),darkred(_("ERROR")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = "\t" ) mytxt = "%s. %s" % (red(_("An error occured while checking database integrity")),red(_("Giving up")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "error", header = "\t" ) return 1 return 0 def show_repository_information(self, repo, count_info): self.Entropy.updateProgress( bold("%s") % ( etpRepositories[repo]['description'] ), importance = 2, type = "info", count = count_info, header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Database URL")),darkgreen(etpRepositories[repo]['database']),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Database local path")),darkgreen(etpRepositories[repo]['dbpath']),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Database EAPI")),darkgreen(str(self.dbformat_eapi)),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # ") ) def get_eapi3_local_database(self, repo): dbfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasefile']) mydbconn = None try: mydbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False) mydbconn.validateDatabase() except ( self.Entropy.dbapi2.OperationalError, self.Entropy.dbapi2.IntegrityError, exceptionTools.SystemDatabaseError, IOError, OSError,): mydbconn = None return mydbconn def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, session): data = eapi3_interface.CmdInterface.differential_packages_comparison( session, idpackages, repo, etpConst['currentarch'], etpConst['product'] ) if isinstance(data,bool): # then it's probably == False return False,False,False elif not isinstance(data,dict): return None,None,None elif not data.has_key('added') or \ not data.has_key('removed') or \ not data.has_key('checksum'): return None,None,None return data['added'],data['removed'],data['checksum'] def get_eapi3_database_treeupdates(self, eapi3_interface, repo, session): self.socket.setdefaulttimeout(self.big_socket_timeout) data = eapi3_interface.CmdInterface.get_repository_treeupdates( session, repo, etpConst['currentarch'], etpConst['product'] ) if not isinstance(data,dict): return None,None return data.get('digest'), data.get('actions') def get_eapi3_package_sets(self, eapi3_interface, repo, session): self.socket.setdefaulttimeout(self.big_socket_timeout) data = eapi3_interface.CmdInterface.get_package_sets( session, repo, etpConst['currentarch'], etpConst['product'] ) if not isinstance(data,dict): return {} return data def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12): def prepare_exit(mysock, session = None): try: if session != None: mysock.close_session(session) mysock.disconnect() except (self.socket.error,): pass eapi3_interface = self.get_eapi3_connection(repo) if eapi3_interface == None: return False session = eapi3_interface.open_session() # AttributeError because mydbconn can be == None try: mydbconn = self.get_eapi3_local_database(repo) myidpackages = mydbconn.listAllIdpackages() except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,AttributeError,): prepare_exit(eapi3_interface, session) return False added_ids, removed_ids, checksum = self.get_eapi3_database_differences( eapi3_interface, repo, myidpackages, session ) if (None in (added_ids,removed_ids,checksum)) or \ (not added_ids and not removed_ids and self.forceUpdate): mydbconn.closeDB() prepare_exit(eapi3_interface, session) return False elif not checksum: # {added_ids, removed_ids, checksum} == False mydbconn.closeDB() prepare_exit(eapi3_interface, session) mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("remote database suddenly locked")),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) return None # is it worth it? if len(added_ids) > threshold: mytxt = "%s: %s (%s: %s/%s)" % ( blue(_("EAPI3 Service")), darkred(_("skipping differential sync")), brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)), ) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) mydbconn.closeDB() prepare_exit(eapi3_interface, session) return False count = 0 added_segments = [] mytmp = set() for idpackage in added_ids: count += 1 mytmp.add(idpackage) if count % chunk_size == 0: added_segments.append(list(mytmp)) mytmp.clear() if mytmp: added_segments.append(list(mytmp)) del mytmp # fetch and store count = 0 maxcount = len(added_segments) for segment in added_segments: count += 1 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t", back = True, count = (count,maxcount,) ) fetch_count = 0 max_fetch_count = 5 error_caught = False error_rc = None while 1: # anti loop protection if fetch_count > max_fetch_count: mydbconn.closeDB() prepare_exit(eapi3_interface, session) error_caught = True error_rc = False break fetch_count += 1 pkgdata = eapi3_interface.CmdInterface.get_package_information( session, segment, repo, etpConst['currentarch'], etpConst['product'] ) if pkgdata == None: mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), darkred(str(segment)),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t", count = (count,maxcount,) ) continue elif not pkgdata: # pkgdata == False mytxt = "%s: %s" % ( blue(_("Service status")), darkred("remote database suddenly locked"), ) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t", count = (count,maxcount,) ) mydbconn.closeDB() prepare_exit(eapi3_interface, session) error_caught = True break elif isinstance(pkgdata,tuple): mytxt = "%s: %s, %s. %s" % ( blue(_("Service status")), pkgdata[0], pkgdata[1], darkred("Error processing the command"),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t", count = (count,maxcount,) ) mydbconn.closeDB() prepare_exit(eapi3_interface, session) error_caught = True break try: for idpackage in pkgdata: self.dumpTools.dumpobj( "%s%s" % (etpCache['eapi3_fetch'],idpackage,), pkgdata[idpackage], ignoreExceptions = False ) except (IOError,EOFError,OSError,), e: mytxt = "%s: %s: %s." % ( blue(_("Local status")), darkred("Error storing data"), e,) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t", count = (count,maxcount,) ) mydbconn.closeDB() prepare_exit(eapi3_interface, session) error_caught = True break break if error_caught: return error_rc del added_segments # get treeupdates stuff dbdigest, treeupdates_actions = self.get_eapi3_database_treeupdates(eapi3_interface, repo, session) if dbdigest == None: mydbconn.closeDB() prepare_exit(eapi3_interface, session) mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("treeupdates data not available")),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) return None try: mydbconn.setRepositoryUpdatesDigest(repo, dbdigest) mydbconn.bumpTreeUpdatesActions(treeupdates_actions) except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,): mydbconn.closeDB() prepare_exit(eapi3_interface, session) mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update treeupdates data")),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) return None # get updated package sets repo_sets = self.get_eapi3_package_sets(eapi3_interface, repo, session) try: mydbconn.clearPackageSets() mydbconn.insertPackageSets(repo_sets) except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,): mydbconn.closeDB() prepare_exit(eapi3_interface, session) mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update package sets data")),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) return None # I don't need you anymore # disconnect socket prepare_exit(eapi3_interface, session) # now that we have all stored, add count = 0 maxcount = len(added_ids) for idpackage in added_ids: count += 1 mydata = self.Entropy.Cacher.pop("%s%s" % (etpCache['eapi3_fetch'],idpackage,)) if mydata == None: mytxt = "%s: %s" % ( blue(_("Fetch error on segment while adding")), darkred(str(segment)), ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t", count = (count,maxcount,) ) mydbconn.closeDB() return False mytxt = "%s %s" % (blue(_("Injecting package")), darkgreen(mydata['atom']),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t", back = True, count = (count,maxcount,) ) mydbconn.addPackage( mydata, revision = mydata['revision'], idpackage = idpackage, do_remove = False, do_commit = False, formatted_content = True ) self.Entropy.updateProgress( blue(_("Packages injection complete")), importance = 0, type = "info", header = "\t", ) # now remove maxcount = len(removed_ids) count = 0 for idpackage in removed_ids: myatom = mydbconn.retrieveAtom(idpackage) count += 1 mytxt = "%s: %s" % (blue(_("Removing package")), darkred(str(myatom)),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t", back = True, count = (count,maxcount,) ) mydbconn.removePackage(idpackage, do_cleanup = False, do_commit = False) self.Entropy.updateProgress( blue(_("Packages removal complete")), importance = 0, type = "info", header = "\t", ) #mydbconn.doCleanups() mydbconn.commitChanges() # now verify if both checksums match result = False mychecksum = mydbconn.database_checksum(do_order = True, strict = False, strings = True) if checksum == mychecksum: result = True else: mytxt = "%s: %s: %s | %s: %s" % ( blue(_("Database checksum doesn't match remote.")), darkgreen(_("local")), mychecksum, darkred(_("remote")), checksum, ) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t", ) mydbconn.closeDB() return result def run_sync(self): self.dbupdated = False repocount = 0 repolength = len(self.reponames) for repo in self.reponames: repocount += 1 self.reset_dbformat_eapi(repo) self.show_repository_information(repo, (repocount,repolength)) if not self.forceUpdate: updated = self.handle_repository_update(repo) if updated: self.Entropy.cycleDone() self.alreadyUpdated += 1 continue locked = self.handle_repository_lock(repo) if locked: self.notAvailable += 1 self.Entropy.cycleDone() continue # clear database interface cache belonging to this repository self.clear_repository_cache(repo) self.__ensure_repository_path(repo) # dealing with EAPI # setting some vars do_skip = False skip_this_repo = False db_down_status = False do_db_update_transfer = False rc = 0 # some variables dumpfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasedump']) dbfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasefile']) dbfile_old = dbfile+".sync" while 1: if do_skip: break if self.dbformat_eapi < 3: cmethod = self.__validate_compression_method(repo) down_status = self.handle_database_download(repo, cmethod) if not down_status: self.Entropy.cycleDone() self.notAvailable += 1 do_skip = True skip_this_repo = True continue db_down_status = self.handle_database_checksum_download(repo, cmethod) break elif self.dbformat_eapi == 3 and not (os.path.isfile(dbfile) and os.access(dbfile,os.W_OK)): do_db_update_transfer = None self.dbformat_eapi -= 1 continue elif self.dbformat_eapi == 3: status = False try: status = self.handle_eapi3_database_sync(repo) except self.socket.error, e: mytxt = "%s: %s" % ( blue(_("EAPI3 Service error")), darkred(unicode(e)), ) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = blue(" # "), ) status = False if status == None: # remote db not available anymore ? time.sleep(5) locked = self.handle_repository_lock(repo) if locked: self.Entropy.cycleDone() self.notAvailable += 1 do_skip = True skip_this_repo = True else: # ah, well... dunno then... do_db_update_transfer = None self.dbformat_eapi -= 1 continue elif not status: # (status == False) # set to none and completely skip database alignment do_db_update_transfer = None self.dbformat_eapi -= 1 continue break if skip_this_repo: continue if self.dbformat_eapi in (1,2,): if self.dbformat_eapi == 2 and db_down_status: rc = self.check_downloaded_database(repo, cmethod) if rc != 0: # delete all self.__remove_repository_files(repo, cmethod) self.syncErrors = True self.Entropy.cycleDone() continue if isinstance(do_db_update_transfer,bool) and not do_db_update_transfer: if os.path.isfile(dbfile): try: shutil.move(dbfile,dbfile_old) do_db_update_transfer = True except: pass # unpack database unpack_status = self.handle_downloaded_database_unpack(repo, cmethod) if not unpack_status: # delete all self.__remove_repository_files(repo, cmethod) self.syncErrors = True self.Entropy.cycleDone() continue if self.dbformat_eapi == 1 and db_down_status: rc = self.check_downloaded_database(repo, cmethod) if rc != 0: # delete all self.__remove_repository_files(repo, cmethod) self.syncErrors = True self.Entropy.cycleDone() if os.path.isfile(dbfile_old): os.remove(dbfile_old) continue # re-validate if not os.path.isfile(dbfile): do_db_update_transfer = False elif os.path.isfile(dbfile) and not do_db_update_transfer and (self.dbformat_eapi != 1): os.remove(dbfile) if self.dbformat_eapi == 2: rc = self.do_eapi2_inject_downloaded_dump(dumpfile, dbfile, cmethod) if do_db_update_transfer: self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old) if self.dbformat_eapi == 2: # remove the dump os.remove(dumpfile) if rc != 0: # delete all self.__remove_repository_files(repo, cmethod) self.syncErrors = True self.Entropy.cycleDone() if os.path.isfile(dbfile_old): os.remove(dbfile_old) continue if os.path.isfile(dbfile) and os.access(dbfile,os.W_OK): try: self.Entropy.setup_default_file_perms(dbfile) except OSError: # notification applet pass # database is going to be updated self.dbupdated = True self.do_standard_items_download(repo) self.Entropy.update_repository_revision(repo) if self.Entropy.indexing: self.do_database_indexing(repo) if (repo == etpConst['officialrepositoryid']): try: self.run_config_files_updates(repo) except Exception, e: self.entropyTools.printTraceback() mytxt = "%s: %s" % ( blue(_("Configuration files update error, not critical, continuing")), darkred(unicode(e)), ) self.Entropy.updateProgress(mytxt, importance = 0, type = "info", header = blue(" # "),) self.updated_repos.add(repo) self.Entropy.cycleDone() # remove garbage if os.path.isfile(dbfile_old): os.remove(dbfile_old) # keep them closed self.Entropy.closeAllRepositoryDatabases() self.Entropy.validate_repositories() self.Entropy.closeAllRepositoryDatabases() # clean caches, fetch security if self.dbupdated: self.Entropy.generate_cache( depcache = self.Entropy.xcache, configcache = False, client_purge = False, install_queue = False ) if self.fetchSecurity: self.do_update_security_advisories() # do treeupdates if isinstance(self.Entropy.clientDbconn,EntropyDatabaseInterface): for repo in self.reponames: dbc = self.Entropy.openRepositoryDatabase(repo) dbc.clientUpdatePackagesData(self.Entropy.clientDbconn) self.Entropy.closeAllRepositoryDatabases() if self.syncErrors: self.Entropy.updateProgress( red(_("Something bad happened. Please have a look.")), importance = 1, type = "warning", header = darkred(" @@ ") ) self.syncErrors = True self.Entropy._resources_run_remove_lock() return 128 if not self.noEquoCheck: self.check_entropy_updates() return 0 def run_config_files_updates(self, repo): # are we root? if etpConst['uid'] != 0: self.Entropy.updateProgress( brown(_("Skipping configuration files update, you are not root.")), importance = 1, type = "info", header = blue(" @@ ") ) return # make.conf self._config_updates_make_conf(repo) self._config_updates_make_profile(repo) def _config_updates_make_conf(self, repo): ## WARNING: it doesn't handle multi-line variables, yet. remember this. url, repo_make_conf = self._construct_paths("make.conf", repo, None) system_make_conf = etpConst['spm']['global_make_conf'] make_conf_variables_check = ["CHOST"] if os.path.isfile(repo_make_conf) and os.access(repo_make_conf,os.R_OK): if not os.path.isfile(system_make_conf): self.Entropy.updateProgress( "%s %s. %s." % (red(system_make_conf),blue(_("does not exist")),blue(_("Overwriting")),), importance = 1, type = "info", header = blue(" @@ ") ) if os.path.lexists(system_make_conf): shutil.move( system_make_conf, "%s.backup_%s" % (system_make_conf,self.entropyTools.getRandomNumber(),) ) shutil.copy2(repo_make_conf,system_make_conf) elif os.access(system_make_conf,os.W_OK): repo_f = open(repo_make_conf,"r") sys_f = open(system_make_conf,"r") repo_make_c = [x.strip() for x in repo_f.readlines()] sys_make_c = [x.strip() for x in sys_f.readlines()] repo_f.close() sys_f.close() # read repository settings repo_data = {} for setting in make_conf_variables_check: for line in repo_make_c: if line.startswith(setting+"="): # there can't be bash vars with a space after its name on declaration repo_data[setting] = line # I don't break, because there might be other overlapping settings differences = {} # update make.conf data in memory for setting in repo_data: for idx in range(len(sys_make_c)): line = sys_make_c[idx] if line.startswith(setting+"=") and (line != repo_data[setting]): # there can't be bash vars with a space after its name on declaration self.Entropy.updateProgress( "%s: %s %s. %s." % ( red(system_make_conf), bold(unicode(setting)), blue(_("variable differs")), red(_("Updating")), ), importance = 1, type = "info", header = blue(" @@ ") ) differences[setting] = repo_data[setting] line = repo_data[setting] sys_make_c[idx] = line if differences: self.Entropy.updateProgress( "%s: %s." % (red(system_make_conf), blue(_("updating critical variables")),), importance = 1, type = "info", header = blue(" @@ ") ) # backup user make.conf shutil.copy2(system_make_conf,"%s.entropy_backup" % (system_make_conf,)) self.Entropy.updateProgress( "%s: %s." % ( red(system_make_conf), darkgreen("writing changes to disk"), ), importance = 1, type = "info", header = blue(" @@ ") ) # write to disk, safely tmp_make_conf = "%s.entropy_write" % (system_make_conf,) f = open(tmp_make_conf,"w") for line in sys_make_c: f.write(line+"\n") f.flush() f.close() shutil.move(tmp_make_conf,system_make_conf) # update environment for var in differences: try: myval = '='.join(differences[var].strip().split("=")[1:]) if myval: if myval[0] in ("'",'"',): myval = myval[1:] if myval[-1] in ("'",'"',): myval = myval[:-1] except IndexError: myval = '' os.environ[var] = myval def _config_updates_make_profile(self, repo): url, repo_make_profile = self._construct_paths("profile.link", repo, None) system_make_profile = etpConst['spm']['global_make_profile'] if not (os.path.isfile(repo_make_profile) and os.access(repo_make_profile,os.R_OK)): return f = open(repo_make_profile,"r") repo_profile_link_data = f.readline().strip() f.close() current_profile_link = '' if os.path.islink(system_make_profile) and os.access(system_make_profile,os.R_OK): current_profile_link = os.readlink(system_make_profile) if repo_profile_link_data != current_profile_link: self.Entropy.updateProgress( "%s: %s %s. %s." % ( red(system_make_profile), blue("link"), blue(_("differs")), red(_("Updating")), ), importance = 1, type = "info", header = blue(" @@ ") ) if os.path.lexists(repo_make_profile): shutil.move(system_make_profile,"%s.entropy_old" % (system_make_profile,)) # broken symlink? if os.path.islink(system_make_profile) and not os.path.isdir(system_make_profile): os.remove(system_make_profile) os.symlink(repo_profile_link_data,system_make_profile) if not self.entropyTools.is_valid_path(system_make_profile): if os.path.lexists(system_make_profile): os.remove(system_make_profile) # revert change, link does not exist yet self.Entropy.updateProgress( "%s: %s %s. %s." % ( red(system_make_profile), blue("new link"), blue(_("does not exist")), red(_("Reverting")), ), importance = 1, type = "info", header = blue(" @@ ") ) os.symlink(current_profile_link,system_make_profile) def check_entropy_updates(self): rc = False if not self.noEquoCheck: try: rc = self.Entropy.check_equo_updates() except: pass if rc: self.newEquo = True mytxt = "%s: %s. %s." % ( bold("Equo/Entropy"), blue(_("a new release is available")), darkred(_("Mind to install it before any other package")), ) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = bold(" !!! ") ) def handle_downloaded_database_unpack(self, repo, cmethod): file_to_unpack = etpConst['etpdatabasedump'] if self.dbformat_eapi == 1: file_to_unpack = etpConst['etpdatabasefile'] mytxt = "%s %s %s" % (red(_("Unpacking database to")),darkgreen(file_to_unpack),red("..."),) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t" ) myrc = self.__unpack_downloaded_database(repo, cmethod) if myrc != 0: mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),red(_("Skipping repository")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) return False return True def handle_database_checksum_download(self, repo, cmethod): hashfile = etpConst['etpdatabasehashfile'] downitem = 'ck' if self.dbformat_eapi == 2: # EAPI = 2 hashfile = etpConst[cmethod[4]] downitem = 'dbdumpck' mytxt = "%s %s %s" % (red(_("Downloading checksum")),darkgreen(hashfile),red("..."),) # download checksum self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t" ) db_down_status = self.download_item(downitem, repo, cmethod, disallow_redirect = True) if not db_down_status: mytxt = "%s %s !" % (red(_("Cannot fetch checksum")),red(_("Cannot verify database integrity")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) return db_down_status def load_background_repository_lock_check(self, repo): # kill previous self.current_repository_got_locked = False self.kill_previous_repository_lock_scanner() self.LockScanner = self.entropyTools.TimeScheduled( self.repository_lock_scanner, 2, {'repo': repo} ) self.LockScanner.setName("Lock_Scanner::"+str(abs(hash(os.urandom(20))))) self.LockScanner.start() def kill_previous_repository_lock_scanner(self): if self.LockScanner != None: self.LockScanner.kill() def repository_lock_scanner(self, data): repo = data['repo'] locked = self.handle_repository_lock(repo) if locked: self.current_repository_got_locked = True def repository_lock_scanner_status(self): # raise an exception if repo got suddenly locked if self.current_repository_got_locked: mytxt = _("Current repository got suddenly locked. Download aborted.") raise exceptionTools.RepositoryError('RepositoryError %s' % (mytxt,)) def handle_database_download(self, repo, cmethod): def show_repo_locked_message(): mytxt = "%s: %s." % (bold(_("Attention")),red(_("remote database got suddenly locked")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) # starting to download mytxt = "%s ..." % (red(_("Downloading repository database")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t" ) down_status = False if self.dbformat_eapi == 2: # start a check in background self.load_background_repository_lock_check(repo) down_status = self.download_item("dbdump", repo, cmethod, lock_status_func = self.repository_lock_scanner_status, disallow_redirect = True) if self.current_repository_got_locked: self.kill_previous_repository_lock_scanner() show_repo_locked_message() return False if not down_status: # fallback to old db # start a check in background self.load_background_repository_lock_check(repo) self.dbformat_eapi = 1 down_status = self.download_item("db", repo, cmethod, lock_status_func = self.repository_lock_scanner_status, disallow_redirect = True) if self.current_repository_got_locked: self.kill_previous_repository_lock_scanner() show_repo_locked_message() return False if not down_status: mytxt = "%s: %s." % (bold(_("Attention")),red(_("database does not exist online")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) self.kill_previous_repository_lock_scanner() return down_status def handle_repository_update(self, repo): # check if database is already updated to the latest revision update = self.is_repository_updatable(repo) if not update: mytxt = "%s: %s." % (bold(_("Attention")),red(_("database is already up to date")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t" ) return True # also check for eapi3 lock if self.dbformat_eapi == 3: locked = self.is_repository_eapi3_locked(repo) if locked: mytxt = "%s: %s." % (bold(_("Attention")),red(_("database will be ready soon")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t" ) return True return False def handle_repository_lock(self, repo): # get database lock unlocked = self.is_repository_unlocked(repo) if not unlocked: mytxt = "%s: %s. %s." % ( bold(_("Attention")), red(_("Repository is being updated")), red(_("Try again in a few minutes")), ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = "\t" ) return True return False def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old): dbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False) old_dbconn = self.Entropy.openGenericDatabase(dbfile_old, xcache = False, indexing_override = False) upd_rc = 0 try: upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") except (dbapi2.OperationalError,dbapi2.IntegrityError,): pass old_dbconn.closeDB() dbconn.closeDB() if upd_rc > 0: # -1 means no changes, == force used # 0 means too much hassle shutil.move(dbfile_old,dbfile) return upd_rc def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod): # load the dump into database mytxt = "%s %s, %s %s" % ( red(_("Injecting downloaded dump")), darkgreen(etpConst[cmethod[3]]), red(_("please wait")), red("..."), ) self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t" ) dbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False) rc = dbconn.doDatabaseImport(dumpfile, dbfile) dbconn.closeDB() return rc def do_update_security_advisories(self): # update Security Advisories try: securityConn = self.Entropy.Security() securityConn.fetch_advisories() except Exception, e: self.entropyTools.printTraceback(f = self.Entropy.clientLog) mytxt = "%s: %s" % (red(_("Advisories fetch error")),e,) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = darkred(" @@ ") ) def do_standard_items_download(self, repo): download_items = [ ( "ca.cert", etpConst['etpdatabasecacertfile'], True, "%s %s %s" % ( red(_("Downloading SSL CA certificate")), darkgreen(etpConst['etpdatabasecacertfile']), red("..."), ) ), ( "server.cert", etpConst['etpdatabaseservercertfile'], True, "%s %s %s" % ( red(_("Downloading SSL Server certificate")), darkgreen(etpConst['etpdatabaseservercertfile']), red("..."), ) ), ( "mask", etpConst['etpdatabasemaskfile'], True, "%s %s %s" % ( red(_("Downloading package mask")), darkgreen(etpConst['etpdatabasemaskfile']), red("..."), ) ), ( "system_mask", etpConst['etpdatabasesytemmaskfile'], True, "%s %s %s" % ( red(_("Downloading packages system mask")), darkgreen(etpConst['etpdatabasesytemmaskfile']), red("..."), ) ), ( "conflicting_tagged", etpConst['etpdatabaseconflictingtaggedfile'], True, "%s %s %s" % ( red(_("Downloading conflicting tagged packages file")), darkgreen(etpConst['etpdatabaseconflictingtaggedfile']), red("..."), ) ), ( "lic_whitelist", etpConst['etpdatabaselicwhitelistfile'], True, "%s %s %s" % ( red(_("Downloading license whitelist")), darkgreen(etpConst['etpdatabaselicwhitelistfile']), red("..."), ) ), ( "rev", etpConst['etpdatabasemaskfile'], False, "%s %s %s" % ( red(_("Downloading revision")), darkgreen(etpConst['etpdatabaserevisionfile']), red("..."), ) ), ( "make.conf", os.path.basename(etpConst['spm']['global_make_conf']), True, "%s %s %s" % ( red(_("Downloading SPM global configuration")), darkgreen(os.path.basename(etpConst['spm']['global_make_conf'])), red("..."), ) ), ( "package.unmask", os.path.basename(etpConst['spm']['global_package_unmask']), True, "%s %s %s" % ( red(_("Downloading SPM package unmasking configuration")), darkgreen(os.path.basename(etpConst['spm']['global_package_unmask'])), red("..."), ) ), ( "package.keywords", os.path.basename(etpConst['spm']['global_package_keywords']), True, "%s %s %s" % ( red(_("Downloading SPM package keywording configuration")), darkgreen(os.path.basename(etpConst['spm']['global_package_keywords'])), red("..."), ) ), ( "package.use", os.path.basename(etpConst['spm']['global_package_use']), True, "%s %s %s" % ( red(_("Downloading SPM package USE flags configuration")), darkgreen(os.path.basename(etpConst['spm']['global_package_use'])), red("..."), ) ), ( "profile.link", etpConst['spm']['global_make_profile_link_name'], True, "%s %s %s" % ( red(_("Downloading SPM Profile configuration")), darkgreen(etpConst['spm']['global_make_profile_link_name']), red("..."), ) ), ( "notice_board", os.path.basename(etpRepositories[repo]['local_notice_board']), True, "%s %s %s" % ( red(_("Downloading Notice Board")), darkgreen(os.path.basename(etpRepositories[repo]['local_notice_board'])), red("..."), ) ) ] for item, myfile, ignorable, mytxt in download_items: self.Entropy.updateProgress( mytxt, importance = 0, type = "info", header = "\t", back = True ) mystatus = self.download_item(item, repo, disallow_redirect = True) mytype = 'info' if not mystatus: if ignorable: message = "%s: %s." % (blue(myfile),red(_("not available, it's ok"))) else: mytype = 'warning' message = "%s: %s." % (blue(myfile),darkred(_("not available, not much ok!"))) else: message = "%s: %s." % (blue(myfile),darkgreen(_("available, w00t!"))) self.Entropy.updateProgress( message, importance = 0, type = mytype, header = "\t" ) mytxt = "%s: %s" % ( red(_("Repository revision")), bold(str(self.Entropy.get_repository_revision(repo))), ) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t" ) def do_database_indexing(self, repo): # renice a bit, to avoid eating resources old_prio = self.Entropy.set_priority(15) mytxt = red("%s ...") % (_("Indexing Repository metadata"),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = "\t", back = True ) dbconn = self.Entropy.openRepositoryDatabase(repo) dbconn.createAllIndexes() # get list of indexes repo_indexes = dbconn.listAllIndexes() if self.Entropy.clientDbconn != None: try: # client db can be absent client_indexes = self.Entropy.clientDbconn.listAllIndexes() if repo_indexes != client_indexes: self.Entropy.clientDbconn.createAllIndexes() except: pass self.Entropy.set_priority(old_prio) def sync(self): # close them self.Entropy.closeAllRepositoryDatabases() # let's dance! mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),) self.Entropy.updateProgress( mytxt, importance = 2, type = "info", header = darkred(" @@ ") ) gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock) if gave_up: return 3 locked = self.Entropy.application_lock_check() if locked: self.Entropy._resources_run_remove_lock() return 4 # lock self.Entropy._resources_run_create_lock() try: rc = self.run_sync() except: self.Entropy._resources_run_remove_lock() raise if rc: return rc # remove lock self.Entropy._resources_run_remove_lock() if (self.notAvailable >= len(self.reponames)): return 2 elif (self.notAvailable > 0): return 1 return 0 class QAInterface: import entropyTools def __init__(self, EntropyInterface): if not isinstance(EntropyInterface, (EquoInterface, ServerInterface)) and \ not issubclass(EntropyInterface, (EquoInterface, ServerInterface)): mytxt = _("A valid EquoInterface/ServerInterface based instance is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,)) self.Entropy = EntropyInterface def test_depends_linking(self, idpackages, dbconn, repo = etpConst['officialrepositoryid']): scan_msg = blue(_("Now searching for broken depends")) self.Entropy.updateProgress( "[repo:%s] %s..." % ( darkgreen(repo), scan_msg, ), importance = 1, type = "info", header = red(" @@ ") ) broken = False count = 0 maxcount = len(idpackages) for idpackage in idpackages: count += 1 atom = dbconn.retrieveAtom(idpackage) scan_msg = "%s, %s:" % (blue(_("scanning for broken depends")),darkgreen(atom),) self.Entropy.updateProgress( "[repo:%s] %s" % ( darkgreen(repo), scan_msg, ), importance = 1, type = "info", header = blue(" @@ "), back = True, count = (count,maxcount,) ) mydepends = dbconn.retrieveDepends(idpackage) if not mydepends: continue for mydepend in mydepends: myatom = dbconn.retrieveAtom(mydepend) self.Entropy.updateProgress( "[repo:%s] %s => %s" % ( darkgreen(repo), darkgreen(atom), darkred(myatom), ), importance = 0, type = "info", header = blue(" @@ "), back = True, count = (count,maxcount,) ) mycontent = dbconn.retrieveContent(mydepend) mybreakages = self.content_test(mycontent) if not mybreakages: continue broken = True self.Entropy.updateProgress( "[repo:%s] %s %s => %s" % ( darkgreen(repo), darkgreen(atom), darkred(myatom), bold(_("broken libraries detected")), ), importance = 1, type = "warning", header = purple(" @@ "), count = (count,maxcount,) ) for mylib in mybreakages: self.Entropy.updateProgress( "%s %s:" % ( darkgreen(mylib), red(_("needs")), ), importance = 1, type = "warning", header = brown(" ## ") ) for needed in mybreakages[mylib]: self.Entropy.updateProgress( "%s" % ( red(needed), ), importance = 1, type = "warning", header = purple(" # ") ) return broken def scan_missing_dependencies(self, idpackages, dbconn, ask = True, self_check = False, repo = etpConst['officialrepositoryid'], black_list = None, black_list_adder = None): if not isinstance(black_list,set): black_list = set() taint = False scan_msg = blue(_("Now searching for missing RDEPENDs")) self.Entropy.updateProgress( "[repo:%s] %s..." % ( darkgreen(repo), scan_msg, ), importance = 1, type = "info", header = red(" @@ ") ) scan_msg = blue(_("scanning for missing RDEPENDs")) count = 0 maxcount = len(idpackages) for idpackage in idpackages: count += 1 atom = dbconn.retrieveAtom(idpackage) if not atom: continue self.Entropy.updateProgress( "[repo:%s] %s: %s" % ( darkgreen(repo), scan_msg, darkgreen(atom), ), importance = 1, type = "info", header = blue(" @@ "), back = True, count = (count,maxcount,) ) missing_extended, missing = self.get_missing_rdepends(dbconn, idpackage, self_check = self_check) missing -= black_list for item in missing_extended.keys(): missing_extended[item] -= black_list if not missing_extended[item]: del missing_extended[item] if (not missing) or (not missing_extended): continue self.Entropy.updateProgress( "[repo:%s] %s: %s %s:" % ( darkgreen(repo), blue("package"), darkgreen(atom), blue(_("is missing the following dependencies")), ), importance = 1, type = "info", header = red(" @@ "), count = (count,maxcount,) ) for missing_data in missing_extended: self.Entropy.updateProgress( "%s:" % (brown(unicode(missing_data)),), importance = 0, type = "info", header = purple(" ## ") ) for dependency in missing_extended[missing_data]: self.Entropy.updateProgress( "%s" % (darkred(dependency),), importance = 0, type = "info", header = blue(" # ") ) if ask: rc = self.Entropy.askQuestion(_("Do you want to add them?")) if rc == "No": continue rc = self.Entropy.askQuestion(_("Selectively?")) if rc == "Yes": newmissing = set() new_blacklist = set() for dependency in missing: self.Entropy.updateProgress( "[repo:%s|%s] %s" % ( darkgreen(repo), brown(atom), blue(dependency), ), importance = 0, type = "info", header = blue(" @@ ") ) rc = self.Entropy.askQuestion(_("Want to add?")) if rc == "Yes": newmissing.add(dependency) else: rc = self.Entropy.askQuestion(_("Want to blacklist?")) if rc == "Yes": new_blacklist.add(dependency) if new_blacklist and (black_list_adder != None): black_list_adder(new_blacklist, repo = repo) missing = newmissing if missing: taint = True dbconn.insertDependencies(idpackage,missing) dbconn.commitChanges() self.Entropy.updateProgress( "[repo:%s] %s: %s" % ( darkgreen(repo), darkgreen(atom), blue(_("missing dependencies added")), ), importance = 1, type = "info", header = red(" @@ "), count = (count,maxcount,) ) return taint def content_test(self, mycontent): def is_contained(needed, content): for item in content: if os.path.basename(item) == needed: return True return False mylibs = {} for myfile in mycontent: myfile = myfile.encode('raw_unicode_escape') if not os.access(myfile,os.R_OK): continue if not os.path.isfile(myfile): continue if not self.entropyTools.is_elf_file(myfile): continue mylibs[myfile] = self.entropyTools.read_elf_dynamic_libraries(myfile) broken_libs = {} for mylib in mylibs: for myneeded in mylibs[mylib]: # is this inside myself ? if is_contained(myneeded, mycontent): continue found = self.resolve_dynamic_library(myneeded, mylib) if found: continue if not broken_libs.has_key(mylib): broken_libs[mylib] = set() broken_libs[mylib].add(myneeded) return broken_libs def resolve_dynamic_library(self, library, requiring_executable): def do_resolve(mypaths): found_path = None for mypath in mypaths: mypath = os.path.join(etpConst['systemroot']+mypath,library) if not os.access(mypath,os.R_OK): continue if os.path.isdir(mypath): continue if not self.entropyTools.is_elf_file(mypath): continue found_path = mypath break return found_path mypaths = self.entropyTools.collectLinkerPaths() found_path = do_resolve(mypaths) if not found_path: mypaths = self.entropyTools.read_elf_linker_paths(requiring_executable) found_path = do_resolve(mypaths) return found_path def get_missing_rdepends(self, dbconn, idpackage, self_check = False): rdepends = {} rdepends_plain = set() neededs = dbconn.retrieveNeeded(idpackage, extended = True) ldpaths = set(self.entropyTools.collectLinkerPaths()) deps_content = set() dependencies = self._get_deep_dependency_list(dbconn, idpackage, atoms = True) dependencies_cache = set() def update_depscontent(mycontent, dbconn, ldpaths): return set( \ [ x for x in mycontent if os.path.dirname(x) in ldpaths \ and (dbconn.isNeededAvailable(os.path.basename(x)) > 0) ] \ ) def is_in_content(myneeded, content): for item in content: item = os.path.basename(item) if myneeded == item: return True return False for dependency in dependencies: match = dbconn.atomMatch(dependency) if match[0] != -1: mycontent = dbconn.retrieveContent(match[0]) deps_content |= update_depscontent(mycontent, dbconn, ldpaths) key, slot = dbconn.retrieveKeySlot(match[0]) dependencies_cache.add((key,slot)) key, slot = dbconn.retrieveKeySlot(idpackage) mycontent = dbconn.retrieveContent(idpackage) deps_content |= update_depscontent(mycontent, dbconn, ldpaths) dependencies_cache.add((key,slot)) idpackages_cache = set() for needed, elfclass in neededs: data_solved = dbconn.resolveNeeded(needed,elfclass) data_size = len(data_solved) data_solved = set([x for x in data_solved if x[0] not in idpackages_cache]) if not data_solved or (data_size != len(data_solved)): continue if self_check: if is_in_content(needed,mycontent): continue found = False for data in data_solved: if data[1] in deps_content: found = True break if not found: for data in data_solved: key, slot = dbconn.retrieveKeySlot(data[0]) if (key,slot) not in dependencies_cache: if not dbconn.isSystemPackage(data[0]): if not rdepends.has_key((needed,elfclass)): rdepends[(needed,elfclass)] = set() rdepends[(needed,elfclass)].add(key+":"+slot) rdepends_plain.add(key+":"+slot) idpackages_cache.add(data[0]) return rdepends, rdepends_plain def _get_deep_dependency_list(self, dbconn, idpackage, atoms = False): mybuffer = self.entropyTools.lifobuffer() matchcache = set() depcache = set() mydeps = dbconn.retrieveDependencies(idpackage) for mydep in mydeps: mybuffer.push(mydep) mydep = mybuffer.pop() while mydep: if mydep in depcache: mydep = mybuffer.pop() continue mymatch = dbconn.atomMatch(mydep) if atoms: matchcache.add(mydep) else: matchcache.add(mymatch[0]) if mymatch[0] != -1: owndeps = dbconn.retrieveDependencies(mymatch[0]) for owndep in owndeps: mybuffer.push(owndep) depcache.add(mydep) mydep = mybuffer.pop() if atoms and -1 in matchcache: matchcache.discard(-1) return matchcache ''' Entropy FTP interface ''' class FtpInterface: # this must be run before calling the other functions def __init__(self, ftpuri, EntropyInterface, verbose = True): if not isinstance(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)) and \ not issubclass(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)): mytxt = _("A valid TextInterface based instance is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,)) import socket, ftplib, entropyTools self.socket, self.ftplib, self.entropyTools = socket, ftplib, entropyTools self.Entropy = EntropyInterface self.verbose = verbose self.oldprogress = 0.0 # import FTP modules self.socket.setdefaulttimeout(60) self.ftpuri = ftpuri self.ftphost = self.entropyTools.extractFTPHostFromUri(self.ftpuri) self.ftpuser, self.ftppassword, self.ftpport, self.ftpdir = self.entropyTools.extract_ftp_data(ftpuri) count = 10 while 1: count -= 1 try: self.ftpconn = self.ftplib.FTP(self.ftphost) break except (self.socket.gaierror,), e: raise exceptionTools.ConnectionError('ConnectionError: %s' % (e,)) except: if not count: raise continue if self.verbose: mytxt = _("connecting with user") self.Entropy.updateProgress( "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpuser),), importance = 1, type = "info", header = darkgreen(" * ") ) try: self.ftpconn.login(self.ftpuser,self.ftppassword) except self.ftplib.error_perm, e: raise exceptionTools.FtpError('FtpError: %s' % (e,)) if self.verbose: mytxt = _("switching to") self.Entropy.updateProgress( "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpdir),), importance = 1, type = "info", header = darkgreen(" * ") ) self.setCWD(self.ftpdir, dodir = True) def setBasedir(self): rc = self.setCWD(self.ftpdir) return rc # this can be used in case of exceptions def reconnectHost(self): # import FTP modules self.socket.setdefaulttimeout(60) counter = 10 while 1: counter -= 1 try: self.ftpconn = self.ftplib.FTP(self.ftphost) break except: if not counter: raise continue if self.verbose: mytxt = _("reconnecting with user") self.Entropy.updateProgress( "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpuser),), importance = 1, type = "info", header = darkgreen(" * ") ) self.ftpconn.login(self.ftpuser,self.ftppassword) if self.verbose: mytxt = _("switching to") self.Entropy.updateProgress( "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpdir),), importance = 1, type = "info", header = darkgreen(" * ") ) self.setCWD(self.currentdir) def getHost(self): return self.ftphost def getPort(self): return self.ftpport def getDir(self): return self.ftpdir def getCWD(self): pwd = self.ftpconn.pwd() return pwd def setCWD(self, mydir, dodir = False): try: return self._setCWD(mydir, dodir) except self.ftplib.error_perm, e: raise exceptionTools.FtpError('FtpError: %s' % (e,)) def _setCWD(self, mydir, dodir = False): if self.verbose: mytxt = _("switching to") self.Entropy.updateProgress( "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(mydir),), importance = 1, type = "info", header = darkgreen(" * ") ) try: self.ftpconn.cwd(mydir) except self.ftplib.error_perm, e: if e[0][:3] == '550' and dodir: self.recursiveMkdir(mydir) self.ftpconn.cwd(mydir) else: raise self.currentdir = self.getCWD() def setPASV(self,bool): self.ftpconn.set_pasv(bool) def setChmod(self,chmodvalue,file): return self.ftpconn.voidcmd("SITE CHMOD "+str(chmodvalue)+" "+str(file)) def getFileMtime(self,path): rc = self.ftpconn.sendcmd("mdtm "+path) return rc.split()[-1] def spawnCommand(self,cmd): return self.ftpconn.sendcmd(cmd) # list files and directory of a FTP # @returns a list def listDir(self): # directory is: self.ftpdir try: rc = self.ftpconn.nlst() _rc = [] for i in rc: _rc.append(i.split("/")[-1]) rc = _rc except: return [] return rc # list if the file is available # @returns True or False def isFileAvailable(self,filename): # directory is: self.ftpdir try: rc = self.ftpconn.nlst() _rc = [] for i in rc: _rc.append(i.split("/")[-1]) rc = _rc for i in rc: if i == filename: return True return False except: return False def deleteFile(self,file): try: rc = self.ftpconn.delete(file) if rc.startswith("250"): return True else: return False except: return False def recursiveMkdir(self, mypath): mydirs = [x for x in mypath.split("/") if x] mycurpath = "" for mydir in mydirs: mycurpath = os.path.join(mycurpath,mydir) if not self.isFileAvailable(mycurpath): try: self.mkdir(mycurpath) except self.ftplib.error_perm, e: if e[0].lower().find("permission denied") != -1: raise elif e[0][:3] != '550': raise def mkdir(self,directory): return self.ftpconn.mkd(directory) # this function also supports callback, because storbinary doesn't def advancedStorBinary(self, cmd, fp, callback=None): ''' Store a file in binary mode. Our version supports a callback function''' self.ftpconn.voidcmd('TYPE I') conn = self.ftpconn.transfercmd(cmd) while 1: buf = fp.readline() if not buf: break conn.sendall(buf) if callback: callback(buf) conn.close() # that's another workaround #return "226" try: rc = self.ftpconn.voidresp() except: self.reconnectHost() return "226" return rc def updateProgress(self, buf_len): # get the buffer size self.mykByteCount += float(buf_len)/1024 # create percentage if self.myFileSize < 1: myUploadPercentage = 100.0 else: myUploadPercentage = round((round(self.mykByteCount,1)/self.myFileSize)*100,1) currentprogress = myUploadPercentage myUploadSize = round(self.mykByteCount,1) if (currentprogress > self.oldprogress+0.5) and (myUploadPercentage < 100.1) and (myUploadSize <= self.myFileSize): myUploadPercentage = str(myUploadPercentage)+"%" # create text mytxt = _("Upload status") currentText = brown(" <-> %s: " % (mytxt,)) + \ green(str(myUploadSize)) + "/" + red(str(self.myFileSize)) + " kB " + \ brown("[") + str(myUploadPercentage) + brown("]") print_info(currentText, back = True) # XXX too slow, reimplement self.updateProgress and do whatever you want #self.Entropy.updateProgress(currentText, importance = 0, type = "info", back = True) self.oldprogress = currentprogress def uploadFile(self,file,ascii = False): self.oldprogress = 0.0 def uploadFileAndUpdateProgress(buf): self.updateProgress(len(buf)) for i in range(10): # ten tries filename = file.split("/")[len(file.split("/"))-1] try: f = open(file,"r") # get file size self.myFileSize = round(float(self.entropyTools.get_file_size(file))/1024,1) self.mykByteCount = 0 if self.isFileAvailable(filename+".tmp"): self.deleteFile(filename+".tmp") if (ascii): rc = self.ftpconn.storlines("STOR "+filename+".tmp",f) else: rc = self.advancedStorBinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress ) # now we can rename the file with its original name self.renameFile(filename+".tmp",filename) f.close() if rc.find("226") != -1: # upload complete return True else: return False except Exception, e: # connection reset by peer import traceback traceback.print_exc() self.Entropy.updateProgress(" ", importance = 0, type = "info") mytxt = red("%s: %s, %s... #%s") % ( _("Upload issue"), e, _("retrying"), i+1, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = " " ) self.reconnectHost() # reconnect if self.isFileAvailable(filename): self.deleteFile(filename) if self.isFileAvailable(filename+".tmp"): self.deleteFile(filename+".tmp") pass def downloadFile(self, filename, downloaddir, ascii = False): self.oldprogress = 0.0 def downloadFileStoreAndUpdateProgress(buf): # writing file buffer f.write(buf) # update progress self.mykByteCount += float(len(buf))/1024 # create text cnt = round(self.mykByteCount,1) mytxt = _("Download status") currentText = brown(" <-> %s: " % (mytxt,)) + green(str(cnt)) + "/" + \ red(str(self.myFileSize)) + " kB" self.Entropy.updateProgress( currentText, importance = 0, type = "info", back = True, count = (cnt, self.myFileSize), percent = True ) # look if the file exist if self.isFileAvailable(filename): self.mykByteCount = 0 # get the file size self.myFileSize = self.getFileSizeCompat(filename) if (self.myFileSize): self.myFileSize = round(float(int(self.myFileSize))/1024,1) if (self.myFileSize == 0): self.myFileSize = 1 else: self.myFileSize = 0 if (not ascii): f = open(downloaddir+"/"+filename,"wb") rc = self.ftpconn.retrbinary('RETR '+filename, downloadFileStoreAndUpdateProgress, 1024) else: f = open(downloaddir+"/"+filename,"w") rc = self.ftpconn.retrlines('RETR '+filename, f.write) f.flush() f.close() if rc.find("226") != -1: # upload complete return True else: return False else: return None # also used to move files def renameFile(self,fromfile,tofile): rc = self.ftpconn.rename(fromfile,tofile) return rc def getFileSize(self,file): return self.ftpconn.size(file) def getFileSizeCompat(self,file): data = self.getRoughList() for item in data: if item.find(file) != -1: return item.split()[4] return "" def bufferizer(self,buf): self.FTPbuffer.append(buf) def getRoughList(self): self.FTPbuffer = [] self.ftpconn.dir(self.bufferizer) return self.FTPbuffer def closeConnection(self): try: self.ftpconn.quit() except (EOFError,AttributeError,self.socket.timeout,self.ftplib.error_reply,): # AttributeError is raised when socket gets trashed # EOFError is raised when the connection breaks # timeout, who cares! pass class rssFeed: import entropyTools def __init__(self, filename, title, description, maxentries = 100): self.feed_title = title self.feed_title = self.feed_title.strip() self.feed_description = description self.feed_language = "en-EN" self.feed_editor = etpConst['rss-managing-editor'] self.feed_copyright = "%s - (C) %s" % ( etpConst['systemname'], self.entropyTools.getYear(), ) self.file = filename self.items = {} self.itemscounter = 0 self.maxentries = maxentries from xml.dom import minidom self.minidom = minidom # sanity check broken = False if os.path.isfile(self.file): try: self.xmldoc = self.minidom.parse(self.file) except: broken = True if not os.path.isfile(self.file) or broken: self.title = self.feed_title self.description = self.feed_description self.language = self.feed_language self.cright = self.feed_copyright self.editor = self.feed_editor self.link = etpConst['rss-website-url'] f = open(self.file,"w") f.write('') f.close() else: self.rssdoc = self.xmldoc.getElementsByTagName("rss")[0] self.channel = self.rssdoc.getElementsByTagName("channel")[0] self.title = self.channel.getElementsByTagName("title")[0].firstChild.data.strip() self.link = self.channel.getElementsByTagName("link")[0].firstChild.data.strip() self.description = self.channel.getElementsByTagName("description")[0].firstChild.data.strip() try: self.language = self.channel.getElementsByTagName("language")[0].firstChild.data.strip() except IndexError: self.language = 'en' try: self.cright = self.channel.getElementsByTagName("copyright")[0].firstChild.data.strip() except IndexError: self.cright = '' try: self.editor = self.channel.getElementsByTagName("managingEditor")[0].firstChild.data.strip() except IndexError: self.editor = '' entries = self.channel.getElementsByTagName("item") self.itemscounter = len(entries) if self.itemscounter > self.maxentries: self.itemscounter = self.maxentries mycounter = self.itemscounter for item in entries: if mycounter == 0: # max entries reached break mycounter -= 1 self.items[mycounter] = {} self.items[mycounter]['title'] = item.getElementsByTagName("title")[0].firstChild.data.strip() description = item.getElementsByTagName("description")[0].firstChild if description: self.items[mycounter]['description'] = description.data.strip() else: self.items[mycounter]['description'] = "" link = item.getElementsByTagName("link")[0].firstChild if link: self.items[mycounter]['link'] = link.data.strip() else: self.items[mycounter]['link'] = "" self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data.strip() self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data.strip() dcs = item.getElementsByTagName("dc:creator") if dcs: self.items[mycounter]['dc:creator'] = dcs[0].firstChild.data.strip() def addItem(self, title, link = '', description = '', pubDate = ''): self.itemscounter += 1 self.items[self.itemscounter] = {} self.items[self.itemscounter]['title'] = title if pubDate: self.items[self.itemscounter]['pubDate'] = pubDate else: self.items[self.itemscounter]['pubDate'] = time.strftime("%a, %d %b %Y %X +0000") self.items[self.itemscounter]['description'] = description self.items[self.itemscounter]['link'] = link if link: self.items[self.itemscounter]['guid'] = link else: myguid = etpConst['systemname'].lower() myguid = myguid.replace(" ","") self.items[self.itemscounter]['guid'] = myguid+"~"+description+str(self.itemscounter) return self.itemscounter def removeEntry(self, id): if id in self.items: del self.items[id] self.itemscounter -= 1 return self.itemscounter def getEntries(self): return self.items, self.itemscounter def writeChanges(self, reverse = True): # filter entries to fit in maxentries if self.itemscounter > self.maxentries: tobefiltered = self.itemscounter - self.maxentries for index in range(tobefiltered): try: del self.items[index] except KeyError: pass doc = self.minidom.Document() rss = doc.createElement("rss") rss.setAttribute("version","2.0") rss.setAttribute("xmlns:atom","http://www.w3.org/2005/Atom") channel = doc.createElement("channel") # title title = doc.createElement("title") title_text = doc.createTextNode(unicode(self.title)) title.appendChild(title_text) channel.appendChild(title) # link link = doc.createElement("link") link_text = doc.createTextNode(unicode(self.link)) link.appendChild(link_text) channel.appendChild(link) # description description = doc.createElement("description") desc_text = doc.createTextNode(unicode(self.description)) description.appendChild(desc_text) channel.appendChild(description) # language language = doc.createElement("language") lang_text = doc.createTextNode(unicode(self.language)) language.appendChild(lang_text) channel.appendChild(language) # copyright cright = doc.createElement("copyright") cr_text = doc.createTextNode(unicode(self.cright)) cright.appendChild(cr_text) channel.appendChild(cright) # managingEditor managingEditor = doc.createElement("managingEditor") ed_text = doc.createTextNode(unicode(self.editor)) managingEditor.appendChild(ed_text) channel.appendChild(managingEditor) keys = self.items.keys() if reverse: keys.reverse() for key in keys: # sanity check, you never know if not self.items.has_key(key): self.removeEntry(key) continue k_error = False for item in ['title','link','guid','description','pubDate']: if not self.items[key].has_key(item): k_error = True break if k_error: self.removeEntry(key) continue # item item = doc.createElement("item") # title item_title = doc.createElement("title") item_title_text = doc.createTextNode(unicode(self.items[key]['title'])) item_title.appendChild(item_title_text) item.appendChild(item_title) # link item_link = doc.createElement("link") item_link_text = doc.createTextNode(unicode(self.items[key]['link'])) item_link.appendChild(item_link_text) item.appendChild(item_link) # guid item_guid = doc.createElement("guid") item_guid.setAttribute("isPermaLink","true") item_guid_text = doc.createTextNode(unicode(self.items[key]['guid'])) item_guid.appendChild(item_guid_text) item.appendChild(item_guid) # description item_desc = doc.createElement("description") item_desc_text = doc.createTextNode(unicode(self.items[key]['description'])) item_desc.appendChild(item_desc_text) item.appendChild(item_desc) # pubdate item_date = doc.createElement("pubDate") item_date_text = doc.createTextNode(unicode(self.items[key]['pubDate'])) item_date.appendChild(item_date_text) item.appendChild(item_date) # add item to channel channel.appendChild(item) # add channel to rss rss.appendChild(channel) doc.appendChild(rss) f = open(self.file,"w") f.writelines(doc.toprettyxml(indent=" ").encode('utf-8')) f.flush() f.close() class TriggerInterface: import entropyTools def __init__(self, EquoInstance, phase, pkgdata, package_action = None): if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid Entropy Instance is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.Entropy = EquoInstance self.clientLog = self.Entropy.clientLog self.validPhases = ("preinstall","postinstall","preremove","postremove") self.pkgdata = pkgdata self.prepared = False self.triggers = set() self.gentoo_compat = etpConst['gentoo-compat'] self.package_action = package_action ''' @ description: Gentoo toolchain variables ''' self.MODULEDB_DIR="/var/lib/module-rebuild/" self.INITSERVICES_DIR="/var/lib/init.d/" ''' portage stuff ''' if self.gentoo_compat: try: Spm = self.Entropy.Spm() self.Spm = Spm except Exception, e: self.entropyTools.printTraceback() mytxt = darkred("%s, %s: %s, %s !") % ( _("Portage interface can't be loaded"), _("Error"), e, _("please fix"), ) self.Entropy.updateProgress( mytxt, importance = 0, header = bold(" !!! ") ) self.gentoo_compat = False self.phase = phase # validate phase self.phaseValidation() def phaseValidation(self): if self.phase not in self.validPhases: mytxt = "%s: %s" % (_("Valid phases"),self.validPhases,) raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,)) def prepare(self): self.triggers = eval("self."+self.phase)() remove = set() for trigger in self.triggers: if trigger in etpUi[self.phase+'_triggers_disable']: remove.add(trigger) self.triggers = [x for x in self.triggers if x not in remove] del remove self.prepared = True def run(self): for trigger in self.triggers: eval("self.trigger_"+trigger)() def kill(self): self.prepared = False del self.triggers[:] def postinstall(self): functions = [] # Gentoo hook if self.gentoo_compat: functions.append('ebuild_postinstall') # equo purge cache if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-apps/entropy": functions.append("purgecache") # binutils configuration if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-devel/binutils": functions.append("binutilsswitch") # opengl configuration if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")): if "ebuild_postinstall" in functions: functions.remove("ebuild_postinstall") # disabling gentoo postinstall since we reimplemented it functions.append("openglsetup") # load linker paths ldpaths = self.Entropy.entropyTools.collectLinkerPaths() for x in self.pkgdata['content']: if (x.startswith("/etc/conf.d") or x.startswith("/etc/init.d")) and ("conftouch" not in functions): functions.append('conftouch') if x.startswith('/lib/modules/') and ("kernelmod" not in functions): if "ebuild_postinstall" in functions: # disabling gentoo postinstall since we reimplemented it functions.remove("ebuild_postinstall") functions.append('kernelmod') if x.startswith('/boot/kernel-') and ("addbootablekernel" not in functions): functions.append('addbootablekernel') if x.startswith('/usr/src/') and ("createkernelsym" not in functions): functions.append('createkernelsym') if x.startswith('/etc/env.d/') and ("env_update" not in functions): functions.append('env_update') if (os.path.dirname(x) in ldpaths) and ("run_ldconfig" not in functions): if x.find(".so") > -1: functions.append('run_ldconfig') if self.pkgdata['trigger']: functions.append('call_ext_postinstall') del ldpaths return functions def preinstall(self): functions = [] # Gentoo hook if self.gentoo_compat: functions.append('ebuild_preinstall') for x in self.pkgdata['content']: if x.startswith("/etc/init.d/") and ("initinform" not in functions): functions.append('initinform') if x.startswith("/boot") and ("mountboot" not in functions): functions.append('mountboot') if self.pkgdata['trigger']: functions.append('call_ext_preinstall') return functions def postremove(self): functions = [] # load linker paths ldpaths = self.Entropy.entropyTools.collectLinkerPaths() for x in self.pkgdata['removecontent']: if x.startswith('/boot/kernel-') and ("removebootablekernel" not in functions): functions.append('removebootablekernel') if x.startswith('/etc/init.d/') and ("initdisable" not in functions): functions.append('initdisable') if x.endswith('.py') and ("cleanpy" not in functions): functions.append('cleanpy') if x.startswith('/etc/env.d/') and ("env_update" not in functions): functions.append('env_update') if (os.path.dirname(x) in ldpaths) and ("run_ldconfig" not in functions): if x.find(".so") > -1: functions.append('run_ldconfig') if self.pkgdata['trigger']: functions.append('call_ext_postremove') del ldpaths return functions def preremove(self): functions = [] # Gentoo hook if self.gentoo_compat: functions.append('ebuild_preremove') functions.append('ebuild_postremove') # doing here because we need /var/db/pkg stuff in place and also because doesn't make any difference # opengl configuration if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")): if "ebuild_preremove" in functions: functions.remove("ebuild_preremove") if "ebuild_postremove" in functions: # disabling gentoo postinstall since we reimplemented it functions.remove("ebuild_postremove") if self.package_action not in ["remove_conflict"]: functions.append("openglsetup_xorg") for x in self.pkgdata['removecontent']: if x.startswith("/boot"): functions.append('mountboot') break if self.pkgdata['trigger']: functions.append('call_ext_preremove') return functions ''' Real triggers ''' def trigger_call_ext_preinstall(self): return self.trigger_call_ext_generic() def trigger_call_ext_postinstall(self): return self.trigger_call_ext_generic() def trigger_call_ext_preremove(self): return self.trigger_call_ext_generic() def trigger_call_ext_postremove(self): return self.trigger_call_ext_generic() def trigger_call_ext_generic(self): try: return self.do_trigger_call_ext_generic() except Exception, e: mykey = self.pkgdata['category']+"/"+self.pkgdata['name'] self.entropyTools.printTraceback() self.entropyTools.printTraceback(f = self.Entropy.clientLog) self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] ATTENTION Cannot run External trigger for "+mykey+"!! "+str(Exception)+": "+str(e) ) mytxt = "%s: %s %s. %s." % ( bold(_("QA")), brown(_("Cannot run External trigger for")), bold(mykey), brown(_("Please report it")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return 0 def do_trigger_call_ext_generic(self): # if mute, supress portage output if etpUi['mute']: oldsystderr = sys.stderr oldsysstdout = sys.stdout stdfile = open("/dev/null","w") sys.stdout = stdfile sys.stderr = stdfile triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber()) while os.path.isfile(triggerfile): triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber()) triggerdir = os.path.dirname(triggerfile) if not os.path.isdir(triggerdir): os.makedirs(triggerdir) f = open(triggerfile,"w") for x in self.pkgdata['trigger']: f.write(x) f.close() # if mute, restore old stdout/stderr if etpUi['mute']: sys.stderr = oldsystderr sys.stdout = oldsysstdout stdfile.close() stage = self.phase pkgdata = self.pkgdata # since I am sick of seeing pychecker reporting this # let me do a nasty thing x = type(stage),type(pkgdata) del x my_ext_status = 0 execfile(triggerfile) os.remove(triggerfile) return my_ext_status def trigger_purgecache(self): self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Purging Entropy cache..." ) mytxt = "%s: %s." % (_("Please remember"),_("It is always better to leave Entropy updates isolated"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) mytxt = "%s ..." % (_("Purging Entropy cache"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) self.Entropy.purge_cache(False) def trigger_conftouch(self): self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Updating {conf.d,init.d} mtime..." ) mytxt = "%s ..." % (_("Updating {conf.d,init.d} mtime"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) for item in self.pkgdata['content']: if not (item.startswith("/etc/conf.d") or item.startswith("/etc/conf.d")): continue if not os.path.isfile(item): continue if not os.access(item,os.W_OK): continue try: f = open(item,"abw") f.flush() f.close() except (OSError,IOError,): pass def trigger_binutilsswitch(self): self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Configuring Binutils Profile..." ) mytxt = "%s ..." % (_("Configuring Binutils Profile"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) # get binutils profile pkgsplit = self.Entropy.entropyTools.catpkgsplit( self.pkgdata['category'] + "/" + self.pkgdata['name'] + "-" + self.pkgdata['version'] ) profile = self.pkgdata['chost']+"-"+pkgsplit[2] self.trigger_set_binutils_profile(profile) def trigger_kernelmod(self): if self.pkgdata['category'] != "sys-kernel": self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Updating moduledb..." ) mytxt = "%s ..." % (_("Updating moduledb"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) item = 'a:1:'+self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] self.trigger_update_moduledb(item) mytxt = "%s ..." % (_("Running depmod"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) # get kernel modules dir name name = '' for item in self.pkgdata['content']: item = etpConst['systemroot']+item if item.startswith(etpConst['systemroot']+"/lib/modules/"): name = item[len(etpConst['systemroot']):] name = name.split("/")[3] break if name: self.trigger_run_depmod(name) def trigger_initdisable(self): for item in self.pkgdata['removecontent']: item = etpConst['systemroot']+item if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item): myroot = "/" if etpConst['systemroot']: myroot = etpConst['systemroot']+"/" runlevels_dir = etpConst['systemroot']+"/etc/runlevels" runlevels = [] if os.path.isdir(runlevels_dir) and os.access(runlevels_dir,os.R_OK): runlevels = [x for x in os.listdir(runlevels_dir) \ if os.path.isdir(os.path.join(runlevels_dir,x)) \ and os.path.isfile(os.path.join(runlevels_dir,x,os.path.basename(item))) ] for runlevel in runlevels: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Removing boot service: %s, runlevel: %s" % (os.path.basename(item),runlevel,) ) mytxt = "%s: %s : %s" % (brown(_("Removing boot service")),os.path.basename(item),runlevel,) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) cmd = 'ROOT="%s" rc-update del %s %s' % (myroot, os.path.basename(item), runlevel) subprocess.call(cmd, shell = True) def trigger_initinform(self): for item in self.pkgdata['content']: item = etpConst['systemroot']+item if item.startswith(etpConst['systemroot']+"/etc/init.d/") and not os.path.isfile(etpConst['systemroot']+item): self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] A new service will be installed: %s" % (item,) ) mytxt = "%s: %s" % (brown(_("A new service will be installed")),item,) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) def trigger_openglsetup(self): opengl = "xorg-x11" if self.pkgdata['name'] == "nvidia-drivers": opengl = "nvidia" elif self.pkgdata['name'] == "ati-drivers": opengl = "ati" # is there eselect ? eselect = subprocess.call("eselect opengl &> /dev/null", shell = True) if eselect == 0: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Reconfiguring OpenGL to %s ..." % (opengl,) ) mytxt = "%s ..." % (brown(_("Reconfiguring OpenGL")),) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) quietstring = '' if etpUi['quiet']: quietstring = " &>/dev/null" if etpConst['systemroot']: subprocess.call('echo "eselect opengl set --use-old %s" | chroot %s %s' % (opengl,etpConst['systemroot'],quietstring,), shell = True) else: subprocess.call('eselect opengl set --use-old %s %s' % (opengl,quietstring,), shell = True) else: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Eselect NOT found, cannot run OpenGL trigger" ) mytxt = "%s !" % (brown(_("Eselect NOT found, cannot run OpenGL trigger")),) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ##") ) def trigger_openglsetup_xorg(self): eselect = subprocess.call("eselect opengl &> /dev/null", shell = True) if eselect == 0: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Reconfiguring OpenGL to fallback xorg-x11 ..." ) mytxt = "%s ..." % (brown(_("Reconfiguring OpenGL")),) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) quietstring = '' if etpUi['quiet']: quietstring = " &>/dev/null" if etpConst['systemroot']: subprocess.call('echo "eselect opengl set xorg-x11" | chroot %s %s' % (etpConst['systemroot'],quietstring,), shell = True) else: subprocess.call('eselect opengl set xorg-x11 %s' % (quietstring,), shell = True) else: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Eselect NOT found, cannot run OpenGL trigger" ) mytxt = "%s !" % (brown(_("Eselect NOT found, cannot run OpenGL trigger")),) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ##") ) # FIXME: this only supports grub (no lilo support) def trigger_addbootablekernel(self): boot_mount = False if os.path.ismount("/boot"): boot_mount = True kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")] if boot_mount: kernels = [x[len("/boot"):] for x in kernels] for kernel in kernels: mykernel = kernel.split('/kernel-')[1] initramfs = "/boot/initramfs-"+mykernel if initramfs not in self.pkgdata['content']: initramfs = '' elif boot_mount: initramfs = initramfs[len("/boot"):] # configure GRUB self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Configuring GRUB bootloader. Adding the new kernel..." ) mytxt = "%s. %s ..." % ( _("Configuring GRUB bootloader"), _("Adding the new kernel"), ) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) self.trigger_configure_boot_grub(kernel,initramfs) # FIXME: this only supports grub (no lilo support) def trigger_removebootablekernel(self): kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")] for kernel in kernels: initramfs = "/boot/initramfs-"+kernel[13:] if initramfs not in self.pkgdata['content']: initramfs = '' # configure GRUB self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Configuring GRUB bootloader. Removing the selected kernel..." ) mytxt = "%s. %s ..." % ( _("Configuring GRUB bootloader"), _("Removing the selected kernel"), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) self.trigger_remove_boot_grub(kernel,initramfs) def trigger_mountboot(self): # is in fstab? if etpConst['systemroot']: return if os.path.isfile("/etc/fstab"): f = open("/etc/fstab","r") fstab = f.readlines() fstab = self.Entropy.entropyTools.listToUtf8(fstab) f.close() for line in fstab: fsline = line.split() if len(fsline) > 1: if fsline[1] == "/boot": if not os.path.ismount("/boot"): # trigger mount /boot rc = subprocess.call("mount /boot &> /dev/null", shell = True) if rc == 0: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] Mounted /boot successfully" ) self.Entropy.updateProgress( brown(_("Mounted /boot successfully")), importance = 0, header = red(" ## ") ) elif rc != 8192: # already mounted self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] Cannot mount /boot automatically !!" ) self.Entropy.updateProgress( brown(_("Cannot mount /boot automatically !!")), importance = 0, header = red(" ## ") ) break def trigger_cleanpy(self): pyfiles = [x for x in self.pkgdata['content'] if x.endswith(".py")] for item in pyfiles: item = etpConst['systemroot']+item if os.path.isfile(item+"o"): try: os.remove(item+"o") except OSError: pass if os.path.isfile(item+"c"): try: os.remove(item+"c") except OSError: pass def trigger_createkernelsym(self): for item in self.pkgdata['content']: item = etpConst['systemroot']+item if item.startswith(etpConst['systemroot']+"/usr/src/"): # extract directory try: todir = item[len(etpConst['systemroot']):] todir = todir.split("/")[3] except: continue if os.path.isdir(etpConst['systemroot']+"/usr/src/"+todir): # link to /usr/src/linux self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Creating kernel symlink "+etpConst['systemroot']+"/usr/src/linux for /usr/src/"+todir ) mytxt = "%s %s %s %s" % ( _("Creating kernel symlink"), etpConst['systemroot']+"/usr/src/linux", _("for"), "/usr/src/"+todir, ) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) if os.path.isfile(etpConst['systemroot']+"/usr/src/linux") or \ os.path.islink(etpConst['systemroot']+"/usr/src/linux"): os.remove(etpConst['systemroot']+"/usr/src/linux") if os.path.isdir(etpConst['systemroot']+"/usr/src/linux"): mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber()) while os.path.isdir(mydir): mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber()) shutil.move(etpConst['systemroot']+"/usr/src/linux",mydir) try: os.symlink(todir,etpConst['systemroot']+"/usr/src/linux") except OSError: # not important in the end pass break def trigger_run_ldconfig(self): if not etpConst['systemroot']: myroot = "/" else: myroot = etpConst['systemroot']+"/" self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Running ldconfig" ) mytxt = "%s %s" % (_("Regenerating"),"/etc/ld.so.cache",) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) def trigger_env_update(self): self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] Running env-update" ) if os.access(etpConst['systemroot']+"/usr/sbin/env-update",os.X_OK): mytxt = "%s ..." % (_("Updating environment"),) self.Entropy.updateProgress( brown(mytxt), importance = 0, header = red(" ## ") ) if etpConst['systemroot']: subprocess.call("echo 'env-update --no-ldconfig' | chroot %s &> /dev/null" % (etpConst['systemroot'],), shell = True) else: subprocess.call('env-update --no-ldconfig &> /dev/null', shell = True) def trigger_ebuild_postinstall(self): stdfile = open("/dev/null","w") oldstderr = sys.stderr oldstdout = sys.stdout sys.stderr = stdfile myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(etpConst['spm']['source_build_ext'])] if myebuild: myebuild = myebuild[0] portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] self.Entropy.updateProgress( brown("Ebuild: pkg_postinst()"), importance = 0, header = red(" ## ") ) try: if not os.path.isfile(self.pkgdata['unpackdir']+"/portage/"+portage_atom+"/temp/environment"): # if environment is not yet created, we need to run pkg_setup() sys.stdout = stdfile rc = self.Spm.spm_doebuild( myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license'] ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup())" " trigger for "+str(portage_atom)+". Something bad happened." ) sys.stdout = oldstdout rc = self.Spm.spm_doebuild( myebuild, mydo = "postinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license'] ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_postinst()) trigger for " + \ str(portage_atom) + ". Something bad happened." ) except Exception, e: sys.stdout = oldstdout self.entropyTools.printTraceback() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[POST] ATTENTION Cannot run Portage trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e) ) mytxt = "%s: %s %s. %s. %s: %s" % ( bold(_("QA")), brown(_("Cannot run Portage trigger for")), bold(str(portage_atom)), brown(_("Please report it")), bold(_("Attach this")), darkred(etpConst['spmlogfile']), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) sys.stderr = oldstderr sys.stdout = oldstdout stdfile.close() return 0 def trigger_ebuild_preinstall(self): stdfile = open("/dev/null","w") oldstderr = sys.stderr oldstdout = sys.stdout sys.stderr = stdfile myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(etpConst['spm']['source_build_ext'])] if myebuild: myebuild = myebuild[0] portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] self.Entropy.updateProgress( brown(" Ebuild: pkg_preinst()"), importance = 0, header = red(" ##") ) try: sys.stdout = stdfile rc = self.Spm.spm_doebuild( myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license'] ) # create mysettings["T"]+"/environment" if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot properly run Portage preinstall (pkg_setup()) trigger for " + \ str(portage_atom) + ". Something bad happened." ) sys.stdout = oldstdout rc = self.Spm.spm_doebuild( myebuild, mydo = "preinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license'] ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_preinst()) trigger for " + \ str(portage_atom)+". Something bad happened." ) except Exception, e: sys.stdout = oldstdout self.entropyTools.printTraceback() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot run Gentoo preinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e) ) mytxt = "%s: %s %s. %s. %s: %s" % ( bold(_("QA")), brown(_("Cannot run Portage trigger for")), bold(str(portage_atom)), brown(_("Please report it")), bold(_("Attach this")), darkred(etpConst['spmlogfile']), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) sys.stderr = oldstderr sys.stdout = oldstdout stdfile.close() return 0 def trigger_ebuild_preremove(self): stdfile = open("/dev/null","w") oldstderr = sys.stderr sys.stderr = stdfile portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] try: myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+etpConst['spm']['source_build_ext'] except: myebuild = '' self.myebuild_moved = None if os.path.isfile(myebuild): try: myebuild = self._setup_remove_ebuild_environment(myebuild, portage_atom) except EOFError, e: sys.stderr = oldstderr stdfile.close() # stuff on system is broken, ignore it self.Entropy.updateProgress( darkred("!!! Ebuild: pkg_prerm() failed, EOFError: ")+str(e)+darkred(" - ignoring"), importance = 1, type = "warning", header = red(" ## ") ) return 0 except ImportError, e: sys.stderr = oldstderr stdfile.close() # stuff on system is broken, ignore it self.Entropy.updateProgress( darkred("!!! Ebuild: pkg_prerm() failed, ImportError: ")+str(e)+darkred(" - ignoring"), importance = 1, type = "warning", header = red(" ## ") ) return 0 if os.path.isfile(myebuild): self.Entropy.updateProgress( brown(" Ebuild: pkg_prerm()"), importance = 0, header = red(" ##") ) try: rc = self.Spm.spm_doebuild( myebuild, mydo = "prerm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir'] + "/" + portage_atom ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot properly run Portage trigger for " + \ str(portage_atom)+". Something bad happened." ) except Exception, e: sys.stderr = oldstderr stdfile.close() self.entropyTools.printTraceback() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot run Portage preremove trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e) ) mytxt = "%s: %s %s. %s. %s: %s" % ( bold(_("QA")), brown(_("Cannot run Portage trigger for")), bold(str(portage_atom)), brown(_("Please report it")), bold(_("Attach this")), darkred(etpConst['spmlogfile']), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return 0 sys.stderr = oldstderr stdfile.close() self._remove_overlayed_ebuild() return 0 def trigger_ebuild_postremove(self): stdfile = open("/dev/null","w") oldstderr = sys.stderr sys.stderr = stdfile portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] try: myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+etpConst['spm']['source_build_ext'] except: myebuild = '' self.myebuild_moved = None if os.path.isfile(myebuild): try: myebuild = self._setup_remove_ebuild_environment(myebuild, portage_atom) except EOFError, e: sys.stderr = oldstderr stdfile.close() # stuff on system is broken, ignore it self.Entropy.updateProgress( darkred("!!! Ebuild: pkg_postrm() failed, EOFError: ")+str(e)+darkred(" - ignoring"), importance = 1, type = "warning", header = red(" ## ") ) return 0 except ImportError, e: sys.stderr = oldstderr stdfile.close() # stuff on system is broken, ignore it self.Entropy.updateProgress( darkred("!!! Ebuild: pkg_postrm() failed, ImportError: ")+str(e)+darkred(" - ignoring"), importance = 1, type = "warning", header = red(" ## ") ) return 0 if os.path.isfile(myebuild): self.Entropy.updateProgress( brown(" Ebuild: pkg_postrm()"), importance = 0, header = red(" ##") ) try: rc = self.Spm.spm_doebuild( myebuild, mydo = "postrm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom ) if rc == 1: self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot properly run Gentoo postremove trigger for " + \ str(portage_atom)+". Something bad happened." ) except Exception, e: sys.stderr = oldstderr stdfile.close() self.entropyTools.printTraceback() self.Entropy.clientLog.log( ETP_LOGPRI_INFO, ETP_LOGLEVEL_NORMAL, "[PRE] ATTENTION Cannot run Gentoo postremove trigger for " + \ portage_atom+"!! "+str(Exception)+": "+str(e) ) mytxt = "%s: %s %s. %s. %s: %s" % ( bold(_("QA")), brown(_("Cannot run Portage trigger for")), bold(str(portage_atom)), brown(_("Please report it")), bold(_("Attach this")), darkred(etpConst['spmlogfile']), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return 0 sys.stderr = oldstderr stdfile.close() self._remove_overlayed_ebuild() return 0 def _setup_remove_ebuild_environment(self, myebuild, portage_atom): ebuild_dir = os.path.dirname(myebuild) ebuild_file = os.path.basename(myebuild) # copy the whole directory in a safe place dest_dir = os.path.join(etpConst['entropyunpackdir'],"vardb/"+portage_atom) if os.path.exists(dest_dir): if os.path.isdir(dest_dir): shutil.rmtree(dest_dir,True) elif os.path.isfile(dest_dir) or os.path.islink(dest_dir): os.remove(dest_dir) os.makedirs(dest_dir) items = os.listdir(ebuild_dir) for item in items: myfrom = os.path.join(ebuild_dir,item) myto = os.path.join(dest_dir,item) shutil.copy2(myfrom,myto) newmyebuild = os.path.join(dest_dir,ebuild_file) if os.path.isfile(newmyebuild): myebuild = newmyebuild self.myebuild_moved = myebuild self._ebuild_env_setup_hook(myebuild) return myebuild def _ebuild_env_setup_hook(self, myebuild): ebuild_path = os.path.dirname(myebuild) if not etpConst['systemroot']: myroot = "/" else: myroot = etpConst['systemroot']+"/" # we need to fix ROOT= if it's set inside environment bz2envfile = os.path.join(ebuild_path,"environment.bz2") if os.path.isfile(bz2envfile) and os.path.isdir(myroot): import bz2 envfile = self.Entropy.entropyTools.unpackBzip2(bz2envfile) bzf = bz2.BZ2File(bz2envfile,"w") f = open(envfile,"r") line = f.readline() while line: if line.startswith("ROOT="): line = "ROOT=%s\n" % (myroot,) bzf.write(line) line = f.readline() f.close() bzf.close() os.remove(envfile) def _remove_overlayed_ebuild(self): if not self.myebuild_moved: return if os.path.isfile(self.myebuild_moved): mydir = os.path.dirname(self.myebuild_moved) shutil.rmtree(mydir,True) mydir = os.path.dirname(mydir) content = os.listdir(mydir) while not content: os.rmdir(mydir) mydir = os.path.dirname(mydir) content = os.listdir(mydir) ''' Internal ones ''' ''' @description: set chosen gcc profile @output: returns int() as exit status ''' def trigger_set_gcc_profile(self, profile): if os.access(etpConst['systemroot']+'/usr/bin/gcc-config',os.X_OK): redirect = "" if etpUi['quiet']: redirect = " &> /dev/null" if etpConst['systemroot']: subprocess.call("echo '/usr/bin/gcc-config %s' | chroot %s %s" % (profile,etpConst['systemroot'],redirect,), shell = True) else: subprocess.call('/usr/bin/gcc-config %s %s' % (profile,redirect,), shell = True) return 0 ''' @description: set chosen binutils profile @output: returns int() as exit status ''' def trigger_set_binutils_profile(self, profile): if os.access(etpConst['systemroot']+'/usr/bin/binutils-config',os.X_OK): redirect = "" if etpUi['quiet']: redirect = " &> /dev/null" if etpConst['systemroot']: subprocess.call("echo '/usr/bin/binutils-config %s' | chroot %s %s" % (profile,etpConst['systemroot'],redirect,), shell = True) else: subprocess.call('/usr/bin/binutils-config %s %s' % (profile,redirect,), shell = True) return 0 ''' @description: updates moduledb @output: returns int() as exit status ''' def trigger_update_moduledb(self, item): if os.access(etpConst['systemroot']+'/usr/sbin/module-rebuild',os.X_OK): if os.path.isfile(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb'): f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"r") moduledb = f.readlines() moduledb = self.Entropy.entropyTools.listToUtf8(moduledb) f.close() avail = [x for x in moduledb if x.strip() == item] if (not avail): f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"aw") f.write(item+"\n") f.flush() f.close() return 0 ''' @description: insert kernel object into kernel modules db @output: returns int() as exit status ''' def trigger_run_depmod(self, name): if os.access('/sbin/depmod',os.X_OK): if not etpConst['systemroot']: myroot = "/" else: myroot = etpConst['systemroot']+"/" subprocess.call('/sbin/depmod -a -b %s -r %s &> /dev/null' % (myroot,name,), shell = True) return 0 def __get_entropy_kernel_grub_line(self, kernel): return "title="+etpConst['systemname']+" ("+os.path.basename(kernel)+")\n" ''' @description: append kernel entry to grub.conf @output: returns int() as exit status ''' def trigger_configure_boot_grub(self, kernel,initramfs): if not os.path.isdir(etpConst['systemroot']+"/boot/grub"): os.makedirs(etpConst['systemroot']+"/boot/grub") if os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"): # open in append grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","aw") shutil.copy2(etpConst['systemroot']+"/boot/grub/grub.conf",etpConst['systemroot']+"/boot/grub/grub.conf.old.add") # get boot dev boot_dev = self.trigger_get_grub_boot_dev() # test if entry has been already added grubtest = open(etpConst['systemroot']+"/boot/grub/grub.conf","r") content = grubtest.readlines() content = [unicode(x,'raw_unicode_escape') for x in content] for line in content: if line.find(self.__get_entropy_kernel_grub_line(kernel)) != -1: grubtest.close() return # also check if we have the same kernel listed if (line.find("kernel") != 1) and (line.find(os.path.basename(kernel)) != -1) and not line.strip().startswith("#"): grubtest.close() return else: # create boot_dev = "(hd0,0)" grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","w") # write header - guess (hd0,0)... since it is weird having a running system without a bootloader, at least, grub. grub_header = ''' default=0 timeout=10 ''' grub.write(grub_header) cmdline = ' ' if os.path.isfile("/proc/cmdline"): f = open("/proc/cmdline","r") cmdline = " "+f.readline().strip() params = cmdline.split() if "dolvm" not in params: # support new kernels >= 2.6.23 cmdline += " dolvm " f.close() grub.write(self.__get_entropy_kernel_grub_line(kernel)) grub.write("\troot "+boot_dev+"\n") grub.write("\tkernel "+kernel+cmdline+"\n") if initramfs: grub.write("\tinitrd "+initramfs+"\n") grub.write("\n") grub.flush() grub.close() def trigger_remove_boot_grub(self, kernel,initramfs): if os.path.isdir(etpConst['systemroot']+"/boot/grub") and os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"): shutil.copy2(etpConst['systemroot']+"/boot/grub/grub.conf",etpConst['systemroot']+"/boot/grub/grub.conf.old.remove") f = open(etpConst['systemroot']+"/boot/grub/grub.conf","r") grub_conf = f.readlines() f.close() content = [unicode(x,'raw_unicode_escape') for x in grub_conf] try: kernel, initramfs = (unicode(kernel,'raw_unicode_escape'),unicode(initramfs,'raw_unicode_escape')) except TypeError: pass #kernelname = os.path.basename(kernel) new_conf = [] skip = False for line in content: if (line.find(self.__get_entropy_kernel_grub_line(kernel)) != -1): skip = True continue if line.strip().startswith("title"): skip = False if not skip or line.strip().startswith("#"): new_conf.append(line) f = open(etpConst['systemroot']+"/boot/grub/grub.conf","w") for line in new_conf: try: f.write(line) except UnicodeEncodeError: f.write(line.encode('utf-8')) f.flush() f.close() def trigger_get_grub_boot_dev(self): if etpConst['systemroot']: return "(hd0,0)" import re df_avail = subprocess.call("which df &> /dev/null", shell = True) if df_avail != 0: mytxt = "%s: %s! %s. %s (hd0,0)." % ( bold(_("QA")), brown(_("Cannot find df")), brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" grub_avail = subprocess.call("which grub &> /dev/null", shell = True) if grub_avail != 0: mytxt = "%s: %s! %s. %s (hd0,0)." % ( bold(_("QA")), brown(_("Cannot find grub")), brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" gboot = commands.getoutput("df /boot").split("\n")[-1].split()[0] if gboot.startswith("/dev/"): # it's ok - handle /dev/md if gboot.startswith("/dev/md"): md = os.path.basename(gboot) if not md.startswith("md"): md = "md"+md f = open("/proc/mdstat","r") mdstat = f.readlines() mdstat = [x for x in mdstat if x.startswith(md)] f.close() if mdstat: mdstat = mdstat[0].strip().split() mddevs = [] for x in mdstat: if x.startswith("sd"): mddevs.append(x[:-3]) mddevs = sorted(mddevs) if mddevs: gboot = "/dev/"+mddevs[0] else: gboot = "/dev/sda1" else: gboot = "/dev/sda1" # get disk match = re.subn("[0-9]","",gboot) gdisk = match[0] if gdisk == '': mytxt = "%s: %s %s %s. %s! %s (hd0,0)." % ( bold(_("QA")), brown(_("cannot match device")), brown(str(gboot)), brown(_("with a grub one")), # 'cannot match device /dev/foo with a grub one' brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" match = re.subn("[a-z/]","",gboot) try: gpartnum = str(int(match[0])-1) except ValueError: mytxt = "%s: %s: %s. %s. %s (hd0,0)." % ( bold(_("QA")), brown(_("grub translation not supported for")), brown(str(gboot)), brown(_("Cannot properly configure grub.conf")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" # now match with grub device_map = etpConst['packagestmpdir']+"/grub.map" if os.path.isfile(device_map): os.remove(device_map) # generate device.map subprocess.call('echo "quit" | grub --device-map="%s" --no-floppy --batch &> /dev/null' % (device_map,), shell = True) if os.path.isfile(device_map): f = open(device_map,"r") device_map_file = f.readlines() f.close() grub_dev = [x for x in device_map_file if (x.find(gdisk) != -1)] if grub_dev: grub_disk = grub_dev[0].strip().split()[0] grub_dev = grub_disk[:-1]+","+gpartnum+")" return grub_dev else: mytxt = "%s: %s. %s! %s (hd0,0)." % ( bold(_("QA")), brown(_("cannot match grub device with a Linux one")), brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" else: mytxt = "%s: %s. %s! %s (hd0,0)." % ( bold(_("QA")), brown(_("cannot find generated device.map")), brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" else: mytxt = "%s: %s. %s! %s (hd0,0)." % ( bold(_("QA")), brown(_("cannot run df /boot")), brown(_("Cannot properly configure the kernel")), brown(_("Defaulting to")), ) self.Entropy.updateProgress( mytxt, importance = 0, header = red(" ## ") ) return "(hd0,0)" class SystemSettings: import entropyTools def __init__(self, EquoInstance): if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid Equo instance or subclass is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.Entropy = EquoInstance self.etpSettingFiles = { 'keywords': etpConst['confpackagesdir']+"/package.keywords", # keywording configuration files 'unmask': etpConst['confpackagesdir']+"/package.unmask", # unmasking configuration files 'mask': etpConst['confpackagesdir']+"/package.mask", # masking configuration files 'license_mask': etpConst['confpackagesdir']+"/license.mask", # masking configuration files 'repos_system_mask': {}, 'system_mask': etpConst['confpackagesdir']+"/system.mask", 'repos_mask': {}, 'repos_license_whitelist': {}, 'system_package_sets': {}, 'conflicting_tagged_packages': {}, 'system_dirs': etpConst['confdir']+"/fsdirs.conf", 'system_dirs_mask': etpConst['confdir']+"/fsdirsmask.conf", } ## XXX trunk support, for a while - exp. date 10/10/2009 trunk_fsdirs_conf = "../conf/fsdirs.conf" trunk_fsdirsmask_conf = "../conf/fsdirsmask.conf" if os.path.isfile(trunk_fsdirs_conf): self.etpSettingFiles['system_dirs'] = trunk_fsdirs_conf if os.path.isfile(trunk_fsdirsmask_conf): self.etpSettingFiles['system_dirs_mask'] = trunk_fsdirsmask_conf self.etpMtimeFiles = { 'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", 'unmask_mtime': etpConst['dumpstoragedir']+"/unmask.mtime", 'mask_mtime': etpConst['dumpstoragedir']+"/mask.mtime", 'license_mask_mtime': etpConst['dumpstoragedir']+"/license_mask.mtime", 'system_mask_mtime': etpConst['dumpstoragedir']+"/system_mask.mtime", 'repos_system_mask': {}, 'repos_mask': {}, 'repos_license_whitelist': {}, } import threading self.L = threading.Lock() self.__settings = None self.__persistent_settings = { 'pkg_masking_reasons': { 0: _('reason not available'), 1: _('user package.mask'), 2: _('system keywords'), 3: _('user package.unmask'), 4: _('user repo package.keywords (all packages)'), 5: _('user repo package.keywords'), 6: _('user package.keywords'), 7: _('completely masked'), 8: _('repository general packages.db.mask'), 9: _('repository in branch packages.db.mask'), # FIXME: this has been removed 10: _('user license.mask'), 11: _('user live unmask'), 12: _('user live mask'), }, 'pkg_masking_reference': { 'reason_not_avail': 0, 'user_package_mask': 1, 'system_keyword': 2, 'user_package_unmask': 3, 'user_repo_package_keywords_all': 4, 'user_repo_package_keywords': 5, 'user_package_keywords': 6, 'completely_masked': 7, 'repository_packages_db_mask': 8, 'repository_in_branch_pacakges_db_mask': 9, 'user_license_mask': 10, 'user_live_unmask': 11, 'user_live_mask': 12, }, } def scan(self): self.__settings = self.parse() # merge universal keywords for x in self.__settings['keywords']['universal']: etpConst['keywords'].add(x) # live package masking / unmasking self.__settings.update( { 'live_packagemasking': { 'unmask_matches': set(), 'mask_matches': set(), } } ) # match installed packages of system_mask self.__settings['repos_system_mask_installed'] = [] self.__settings['repos_system_mask_installed_keys'] = {} if isinstance(self.Entropy.clientDbconn,EntropyDatabaseInterface): while 1: try: self.Entropy.clientDbconn.validateDatabase() except exceptionTools.SystemDatabaseError: break mc_cache = set() m_list = self.__settings['repos_system_mask']+self.__settings['system_mask'] for atom in m_list: m_ids,m_r = self.Entropy.clientDbconn.atomMatch(atom, multiMatch = True) if m_r != 0: continue mykey = self.entropyTools.dep_getkey(atom) if mykey not in self.__settings['repos_system_mask_installed_keys']: self.__settings['repos_system_mask_installed_keys'][mykey] = set() for xm in m_ids: if xm in mc_cache: continue mc_cache.add(xm) self.__settings['repos_system_mask_installed'].append(xm) self.__settings['repos_system_mask_installed_keys'][mykey].add(xm) break # Live package masking self.__settings.update(self.__persistent_settings) def __setitem__(self, mykey, myvalue): with self.L: if self.__settings == None: self.scan() if mykey in self.__persistent_settings: # backup here too self.__persistent_settings[mykey] = myvalue self.__settings.update(self.__persistent_settings) self.__settings[mykey] = myvalue def __getitem__(self, mykey): with self.L: if self.__settings == None: self.scan() return self.__settings[mykey] def __contains__(self, mykey): with self.L: if self.__settings == None: self.scan() return mykey in self.__settings def __cmp__(self, other): with self.L: if self.__settings == None: self.scan() return cmp(self.__settings,other) def get(self, mykey, default = None): with self.L: if self.__settings == None: self.scan() return self.__settings.get(mykey, default) def has_key(self, mykey): with self.L: if self.__settings == None: self.scan() return mykey in self.__settings def clear(self): with self.L: self.__settings = None def keys(self): with self.L: if self.__settings == None: self.scan() return self.__settings.keys() def items(self): with self.L: if self.__settings == None: self.scan() return self.__settings.items() def iteritems(self): with self.L: if self.__settings == None: self.scan() return self.__settings.iteritems() def iterkeys(self): with self.L: if self.__settings == None: self.scan() return self.__settings.iterkeys() def popitem(self): with self.L: if self.__settings == None: self.scan() return self.__settings.popitem() def setdefault(self, key, default = None): with self.L: if self.__settings == None: self.scan() return self.__settings.setdefault(key, default) def values(self): with self.L: if self.__settings == None: self.scan() return self.__settings.values() def copy(self): with self.L: if self.__settings == None: self.scan() return self.__settings.copy() def fromkeys(self, seq, value = None): with self.L: if self.__settings == None: self.scan() return self.__settings.fromkeys(seq, value) def parse(self): # append repositories mask files # append repositories mtime files for repoid in etpRepositoriesOrder: maskpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasemaskfile']) wlpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabaselicwhitelistfile']) sm_path = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasesytemmaskfile']) ct_path = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabaseconflictingtaggedfile']) if os.path.isfile(maskpath) and os.access(maskpath,os.R_OK): self.etpSettingFiles['repos_mask'][repoid] = maskpath self.etpMtimeFiles['repos_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasemaskfile']+".mtime" if os.path.isfile(wlpath) and os.access(wlpath,os.R_OK): self.etpSettingFiles['repos_license_whitelist'][repoid] = wlpath self.etpMtimeFiles['repos_license_whitelist'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabaselicwhitelistfile']+".mtime" if os.path.isfile(sm_path) and os.access(sm_path,os.R_OK): self.etpSettingFiles['repos_system_mask'][repoid] = sm_path self.etpMtimeFiles['repos_system_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasesytemmaskfile']+".mtime" if os.path.isfile(ct_path) and os.access(ct_path,os.R_OK): self.etpSettingFiles['conflicting_tagged_packages'][repoid] = ct_path # user defined package sets sets_dir = etpConst['confsetsdir'] if (os.path.isdir(sets_dir) and os.access(sets_dir,os.R_OK)): set_files = [x for x in os.listdir(sets_dir) if (os.path.isfile(os.path.join(sets_dir,x)) and os.access(os.path.join(sets_dir,x),os.R_OK))] for set_file in set_files: try: set_file = str(set_file) except (UnicodeDecodeError,UnicodeEncodeError,): continue self.etpSettingFiles['system_package_sets'][set_file] = os.path.join(sets_dir,set_file) data = {} for item in self.etpSettingFiles: myattr = '%s_parser' % (item,) if not hasattr(self,myattr): continue f = getattr(self,myattr) data[item] = f() return data ''' parser of package.keywords file ''' def keywords_parser(self): data = { 'universal': set(), 'packages': {}, 'repositories': {}, } self.__validateEntropyCache(self.etpSettingFiles['keywords'],self.etpMtimeFiles['keywords_mtime']) content = [x.split() for x in self.__generic_parser(self.etpSettingFiles['keywords']) if len(x.split()) < 4] for keywordinfo in content: # skip wrong lines if len(keywordinfo) > 3: continue if len(keywordinfo) == 1: # inversal keywording, check if it's not repo= # repo=? if keywordinfo[0].startswith("repo="): continue #kinfo = keywordinfo[0] if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format data['universal'].add(keywordinfo[0]) continue # needed? if len(keywordinfo) in (2,3): # inversal keywording, check if it's not repo= # repo=? if keywordinfo[0].startswith("repo="): continue # add to repo? items = keywordinfo[1:] if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format reponame = [x for x in items if x.startswith("repo=") and (len(x.split("=")) == 2)] if reponame: reponame = reponame[0].split("=")[1] if reponame not in data['repositories']: data['repositories'][reponame] = {} # repository unmask or package in repository unmask? if keywordinfo[0] not in data['repositories'][reponame]: data['repositories'][reponame][keywordinfo[0]] = set() if len(items) == 1: # repository unmask data['repositories'][reponame][keywordinfo[0]].add('*') else: if "*" not in data['repositories'][reponame][keywordinfo[0]]: item = [x for x in items if not x.startswith("repo=")] data['repositories'][reponame][keywordinfo[0]].add(item[0]) else: # it's going to be a faulty line!!?? if len(items) == 2: # can't have two items and no repo= continue # add keyword to packages if keywordinfo[0] not in data['packages']: data['packages'][keywordinfo[0]] = set() data['packages'][keywordinfo[0]].add(items[0]) return data def unmask_parser(self): self.__validateEntropyCache(self.etpSettingFiles['unmask'],self.etpMtimeFiles['unmask_mtime']) return self.__generic_parser(self.etpSettingFiles['unmask']) def mask_parser(self): self.__validateEntropyCache(self.etpSettingFiles['mask'],self.etpMtimeFiles['mask_mtime']) return self.__generic_parser(self.etpSettingFiles['mask']) def system_mask_parser(self): self.__validateEntropyCache(self.etpSettingFiles['system_mask'],self.etpMtimeFiles['system_mask_mtime']) return self.__generic_parser(self.etpSettingFiles['system_mask']) def license_mask_parser(self): self.__validateEntropyCache(self.etpSettingFiles['license_mask'],self.etpMtimeFiles['license_mask_mtime']) return self.__generic_parser(self.etpSettingFiles['license_mask']) def repos_license_whitelist_parser(self): data = {} for repoid in self.etpSettingFiles['repos_license_whitelist']: self.__validateEntropyCache(self.etpSettingFiles['repos_license_whitelist'][repoid],self.etpMtimeFiles['repos_license_whitelist'][repoid], repoid = repoid) data[repoid] = self.__generic_parser(self.etpSettingFiles['repos_license_whitelist'][repoid]) return data def repos_mask_parser(self): data = {} for repoid in self.etpSettingFiles['repos_mask']: self.__validateEntropyCache(self.etpSettingFiles['repos_mask'][repoid],self.etpMtimeFiles['repos_mask'][repoid], repoid = repoid) data[repoid] = self.__generic_parser(self.etpSettingFiles['repos_mask'][repoid]) # why ? line = line.split()[0] in the previous one? return data def repos_system_mask_parser(self): data = [] for repoid in self.etpSettingFiles['repos_system_mask']: self.__validateEntropyCache(self.etpSettingFiles['repos_system_mask'][repoid],self.etpMtimeFiles['repos_system_mask'][repoid], repoid = repoid) data += [x for x in self.__generic_parser(self.etpSettingFiles['repos_system_mask'][repoid]) if x not in data] # why ? line = line.split()[0] in the previous one? return data def system_package_sets_parser(self): data = {} for set_name in self.etpSettingFiles['system_package_sets']: set_filepath = self.etpSettingFiles['system_package_sets'][set_name] set_elements = self.entropyTools.extract_packages_from_set_file(set_filepath) if set_elements: data[set_name] = set_elements.copy() return data def system_dirs_parser(self): return self.__generic_parser(self.etpSettingFiles['system_dirs']) def system_dirs_mask_parser(self): return self.__generic_parser(self.etpSettingFiles['system_dirs_mask']) def conflicting_tagged_packages_parser(self): data = {} # keep priority order repoids = [x for x in etpRepositoriesOrder if x in self.etpSettingFiles['conflicting_tagged_packages']] for repoid in repoids: filepath = self.etpSettingFiles['conflicting_tagged_packages'].get(repoid) if os.path.isfile(filepath) and os.access(filepath,os.R_OK): f = open(filepath,"r") content = f.readlines() f.close() content = [x.strip().rsplit("#",1)[0].strip().split() for x in content if not x.startswith("#") and x.strip()] for mydata in content: if len(mydata) < 2: continue data[mydata[0]] = mydata[1:] return data ''' internal functions ''' def __generic_parser(self, filepath): data = [] if os.path.isfile(filepath) and os.access(filepath,os.R_OK): f = open(filepath,"r") content = f.readlines() f.close() # filter comments and white lines content = [x.strip().rsplit("#",1)[0].strip() for x in content if not x.startswith("#") and x.strip()] for line in content: if line in data: continue data.append(line) return data def __removeRepoCache(self, repoid = None): if os.path.isdir(etpConst['dumpstoragedir']): if repoid: self.Entropy.repository_move_clear_cache(repoid) return for repoid in etpRepositoriesOrder: self.Entropy.repository_move_clear_cache(repoid) else: os.makedirs(etpConst['dumpstoragedir']) def __saveFileMtime(self,toread,tosaveinto): if not os.path.isfile(toread): currmtime = 0.0 else: currmtime = os.path.getmtime(toread) if not os.path.isdir(etpConst['dumpstoragedir']): os.makedirs(etpConst['dumpstoragedir'],0775) const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid']) f = open(tosaveinto,"w") f.write(str(currmtime)) f.flush() f.close() os.chmod(tosaveinto,0664) if etpConst['entropygid'] != None: os.chown(tosaveinto,0,etpConst['entropygid']) def __validateEntropyCache(self, maskfile, mtimefile, repoid = None): if os.getuid() != 0: # can't validate if running as user, moreover users can't make changes, so... return # handle on-disk cache validation # in this case, repositories cache # if package.keywords is changed, we must destroy cache if not os.path.isfile(mtimefile): # we can't know if package.keywords has been updated # remove repositories caches self.__removeRepoCache(repoid = repoid) self.__saveFileMtime(maskfile,mtimefile) else: # check mtime try: f = open(mtimefile,"r") mtime = float(f.readline().strip()) f.close() # compare with current mtime try: currmtime = os.path.getmtime(maskfile) except OSError: currmtime = 0.0 if mtime != currmtime: self.__removeRepoCache(repoid = repoid) self.__saveFileMtime(maskfile,mtimefile) except: self.__removeRepoCache(repoid = repoid) self.__saveFileMtime(maskfile,mtimefile) class Callable: def __init__(self, anycallable): self.__call__ = anycallable class MultipartPostHandler(urllib2.BaseHandler): handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first def http_request(self, request): import urllib doseq = 1 data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in data.items(): if type(value) == file: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError, "not a valid non-string sequence or mapping object", traceback if len(v_files) == 0: data = urllib.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary ''' if (request.has_header('Content-Type') and request.get_header('Content-Type').find('multipart/form-data') != 0): print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data') ''' request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request def multipart_encode(vars, files, boundary = None, buf = None): from cStringIO import StringIO import mimetools, mimetypes if boundary is None: boundary = mimetools.choose_boundary() if buf is None: buf = StringIO() for(key, value) in vars: buf.write('--%s\r\n' % boundary) buf.write('Content-Disposition: form-data; name="%s"' % key) buf.write('\r\n\r\n' + value + '\r\n') for(key, fd) in files: file_size = os.fstat(fd.fileno())[stat.ST_SIZE] filename = fd.name.split('/')[-1] contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' buf.write('--%s\r\n' % boundary) buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)) buf.write('Content-Type: %s\r\n' % contenttype) # buffer += 'Content-Length: %s\r\n' % file_size fd.seek(0) buf.write('\r\n' + fd.read() + '\r\n') buf.write('--' + boundary + '--\r\n\r\n') buf = buf.getvalue() return boundary, buf multipart_encode = Callable(multipart_encode) https_request = http_request class ErrorReportInterface: import entropyTools def __init__(self, post_url = etpConst['handlers']['errorsend']): self.url = post_url self.opener = urllib2.build_opener(MultipartPostHandler) self.generated = False self.params = {} mydict = {} if etpConst['proxy']['ftp']: mydict['ftp'] = etpConst['proxy']['ftp'] if etpConst['proxy']['http']: mydict['http'] = etpConst['proxy']['http'] if mydict: mydict['username'] = etpConst['proxy']['username'] mydict['password'] = etpConst['proxy']['password'] self.entropyTools.add_proxy_opener(urllib2,mydict) else: # unset urllib2._opener = None def prepare(self, tb_text, name, email, report_data = "", description = ""): self.params['arch'] = etpConst['currentarch'] self.params['stacktrace'] = tb_text self.params['name'] = name self.params['email'] = email self.params['version'] = etpConst['entropyversion'] self.params['errordata'] = report_data self.params['description'] = description self.params['arguments'] = ' '.join(sys.argv) self.params['uid'] = etpConst['uid'] self.params['system_version'] = "N/A" if os.access(etpConst['systemreleasefile'],os.R_OK): f = open(etpConst['systemreleasefile'],"r") self.params['system_version'] = f.readlines() f.close() self.params['processes'] = commands.getoutput('ps auxf') self.params['lspci'] = commands.getoutput('/usr/sbin/lspci') self.params['dmesg'] = commands.getoutput('dmesg') self.params['locale'] = commands.getoutput('locale -v') self.params['stacktrace'] += "\n\n"+self.params['locale'] # just for a while, won't hurt self.generated = True # params is a dict, key(HTTP post item name): value def submit(self): if self.generated: result = self.opener.open(self.url, self.params).read() if result.strip() == "1": return True return False else: mytxt = _("Not prepared yet") raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,)) ''' ~~ GIVES YOU WINGS ~~ ''' class SecurityInterface: # thanks to Gentoo "gentoolkit" package, License below: # This program is licensed under the GPL, version 2 # WARNING: this code is only tested by a few people and should NOT be used # on production systems at this stage. There are possible security holes and probably # bugs in this code. If you test it please report ANY success or failure to # me (genone@gentoo.org). # The following planned features are currently on hold: # - getting GLSAs from http/ftp servers (not really useful without the fixed ebuilds) # - GPG signing/verification (until key policy is clear) def __init__(self, EquoInstance): if not isinstance(EquoInstance,EquoInterface): mytxt = _("A valid EquoInterface instance or subclass is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) self.Entropy = EquoInstance self.lastfetch = None self.previous_checksum = "0" self.advisories_changed = None self.adv_metadata = None self.affected_atoms = None from xml.dom import minidom self.minidom = minidom self.op_mappings = { "le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", "rge": ">=", # >=~ "rle": "<=", # <=~ "rgt": ">", # >~ "rlt": "<" # <~ } self.unpackdir = os.path.join(etpConst['entropyunpackdir'],"security-"+str(self.Entropy.entropyTools.getRandomNumber())) self.security_url = etpConst['securityurl'] self.unpacked_package = os.path.join(self.unpackdir,"glsa_package") self.security_url_checksum = etpConst['securityurl']+etpConst['packageshashfileext'] self.download_package = os.path.join(self.unpackdir,os.path.basename(etpConst['securityurl'])) self.download_package_checksum = self.download_package+etpConst['packageshashfileext'] self.old_download_package_checksum = os.path.join(etpConst['dumpstoragedir'],os.path.basename(etpConst['securityurl']))+etpConst['packageshashfileext'] self.security_package = os.path.join(etpConst['securitydir'],os.path.basename(etpConst['securityurl'])) self.security_package_checksum = self.security_package+etpConst['packageshashfileext'] try: if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']): os.remove(etpConst['securitydir']) if not os.path.isdir(etpConst['securitydir']): os.makedirs(etpConst['securitydir'],0775) except OSError: pass const_setup_perms(etpConst['securitydir'],etpConst['entropygid']) if os.path.isfile(self.old_download_package_checksum): f = open(self.old_download_package_checksum) try: self.previous_checksum = f.readline().strip().split()[0] except: pass f.close() def __prepare_unpack(self): if os.path.isfile(self.unpackdir) or os.path.islink(self.unpackdir): os.remove(self.unpackdir) if os.path.isdir(self.unpackdir): shutil.rmtree(self.unpackdir,True) try: os.rmdir(self.unpackdir) except OSError: pass os.makedirs(self.unpackdir,0775) const_setup_perms(self.unpackdir,etpConst['entropygid']) def __download_glsa_package(self): return self.__generic_download(self.security_url, self.download_package) def __download_glsa_package_checksum(self): return self.__generic_download(self.security_url_checksum, self.download_package_checksum, show_speed = False) def __generic_download(self, url, save_to, show_speed = True): fetchConn = self.Entropy.urlFetcher(url, save_to, resume = False, show_speed = show_speed) fetchConn.progress = self.Entropy.progress rc = fetchConn.download() del fetchConn if rc in ("-1","-2","-3"): return False # setup permissions self.Entropy.setup_default_file_perms(save_to) return True def __verify_checksum(self): # read checksum if not os.path.isfile(self.download_package_checksum) or not os.access(self.download_package_checksum,os.R_OK): return 1 f = open(self.download_package_checksum) try: checksum = f.readline().strip().split()[0] f.close() except: return 2 if checksum == self.previous_checksum: self.advisories_changed = False else: self.advisories_changed = True md5res = self.Entropy.entropyTools.compareMd5(self.download_package,checksum) if not md5res: return 3 return 0 def __unpack_advisories(self): rc = self.Entropy.entropyTools.uncompressTarBz2( self.download_package, self.unpacked_package, catchEmpty = True ) const_setup_perms(self.unpacked_package,etpConst['entropygid']) return rc def __clear_previous_advisories(self): if os.listdir(etpConst['securitydir']): shutil.rmtree(etpConst['securitydir'],True) if not os.path.isdir(etpConst['securitydir']): os.makedirs(etpConst['securitydir'],0775) const_setup_perms(self.unpackdir,etpConst['entropygid']) def __put_advisories_in_place(self): for advfile in os.listdir(self.unpacked_package): from_file = os.path.join(self.unpacked_package,advfile) to_file = os.path.join(etpConst['securitydir'],advfile) shutil.move(from_file,to_file) def __cleanup_garbage(self): shutil.rmtree(self.unpackdir,True) def clear(self, xcache = False): self.adv_metadata = None if xcache: self.Entropy.clear_dump_cache(etpCache['advisories']) def get_advisories_cache(self): if self.adv_metadata != None: return self.adv_metadata if self.Entropy.xcache: dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir']) c_hash = "%s%s" % (etpCache['advisories'],hash("%s|%s|%s" % (hash(etpConst['branch']),hash(dir_checksum),hash(etpConst['systemroot']),)),) adv_metadata = self.Entropy.Cacher.pop(c_hash) if adv_metadata != None: self.adv_metadata = adv_metadata.copy() return self.adv_metadata def set_advisories_cache(self, adv_metadata): if self.Entropy.xcache: dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir']) c_hash = "%s%s" % (etpCache['advisories'],hash("%s|%s|%s" % (hash(etpConst['branch']),hash(dir_checksum),hash(etpConst['systemroot']),)),) self.Entropy.Cacher.push(c_hash,adv_metadata.copy()) def get_advisories_list(self): if not self.check_advisories_availability(): return [] xmls = os.listdir(etpConst['securitydir']) xmls = sorted([x for x in xmls if x.endswith(".xml") and x.startswith("glsa-")]) return xmls def get_advisories_metadata(self): cached = self.get_advisories_cache() if cached != None: return cached adv_metadata = {} xmls = self.get_advisories_list() maxlen = len(xmls) count = 0 for xml in xmls: count += 1 if not etpUi['quiet']: self.Entropy.updateProgress(":: "+str(round((float(count)/maxlen)*100,1))+"% ::", importance = 0, type = "info", back = True) xml_metadata = None exc_string = "" exc_err = "" try: xml_metadata = self.get_xml_metadata(xml) except KeyboardInterrupt: return {} except Exception, e: exc_string = str(Exception) exc_err = str(e) if xml_metadata == None: more_info = "" if exc_string: mytxt = _("Error") more_info = " %s: %s: %s" % (mytxt,exc_string,exc_err,) mytxt = "%s: %s: %s! %s" % ( blue(_("Warning")), bold(xml), blue(_("advisory broken")), more_info, ) self.Entropy.updateProgress( mytxt, importance = 1, type = "warning", header = red(" !!! ") ) continue elif not xml_metadata: continue adv_metadata.update(xml_metadata) adv_metadata = self.filter_advisories(adv_metadata) self.set_advisories_cache(adv_metadata) self.adv_metadata = adv_metadata.copy() return adv_metadata # this function filters advisories for packages that aren't # in the repositories. Note: only keys will be matched def filter_advisories(self, adv_metadata): keys = adv_metadata.keys() for key in keys: valid = True if adv_metadata[key]['affected']: affected = adv_metadata[key]['affected'] affected_keys = affected.keys() valid = False skipping_keys = set() for a_key in affected_keys: match = self.Entropy.atomMatch(a_key) if match[0] != -1: # it's in the repos, it's valid valid = True else: skipping_keys.add(a_key) if not valid: del adv_metadata[key] for a_key in skipping_keys: try: del adv_metadata[key]['affected'][a_key] except KeyError: pass try: if not adv_metadata[key]['affected']: del adv_metadata[key] except KeyError: pass return adv_metadata def is_affected(self, adv_key, adv_data = {}): if not adv_data: adv_data = self.get_advisories_metadata() if adv_key not in adv_data: return False mydata = adv_data[adv_key].copy() del adv_data if not mydata['affected']: return False for key in mydata['affected']: vul_atoms = mydata['affected'][key][0]['vul_atoms'] unaff_atoms = mydata['affected'][key][0]['unaff_atoms'] unaffected_atoms = set() if not vul_atoms: return False # XXX: does multimatch work correctly? for atom in unaff_atoms: matches = self.Entropy.clientDbconn.atomMatch(atom, multiMatch = True) if matches[1] == 0: for idpackage in matches[0]: unaffected_atoms.add((idpackage,0)) for atom in vul_atoms: match = self.Entropy.clientDbconn.atomMatch(atom) if (match[0] != -1) and (match not in unaffected_atoms): if self.affected_atoms == None: self.affected_atoms = set() self.affected_atoms.add(atom) return True return False def get_vulnerabilities(self): return self.get_affection() def get_fixed_vulnerabilities(self): return self.get_affection(affected = False) # if not affected: not affected packages will be returned # if affected: affected packages will be returned def get_affection(self, affected = True): adv_data = self.get_advisories_metadata() adv_data_keys = adv_data.keys() valid_keys = set() for adv in adv_data_keys: is_affected = self.is_affected(adv,adv_data) if affected == is_affected: valid_keys.add(adv) # we need to filter our adv_data and return for key in adv_data_keys: if key not in valid_keys: try: del adv_data[key] except KeyError: pass # now we need to filter packages in adv_dat for adv in adv_data: for key in adv_data[adv]['affected'].keys(): atoms = adv_data[adv]['affected'][key][0]['vul_atoms'] applicable = True for atom in atoms: if atom in self.affected_atoms: applicable = False break if applicable == affected: del adv_data[adv]['affected'][key] return adv_data def get_affected_atoms(self): adv_data = self.get_advisories_metadata() adv_data_keys = adv_data.keys() del adv_data self.affected_atoms = set() for key in adv_data_keys: self.is_affected(key) return self.affected_atoms def get_xml_metadata(self, xmlfilename): xml_data = {} xmlfile = os.path.join(etpConst['securitydir'],xmlfilename) try: xmldoc = self.minidom.parse(xmlfile) except: return None # get base data glsa_tree = xmldoc.getElementsByTagName("glsa")[0] glsa_product = glsa_tree.getElementsByTagName("product")[0] if glsa_product.getAttribute("type") != "ebuild": return {} glsa_id = glsa_tree.getAttribute("id") glsa_title = glsa_tree.getElementsByTagName("title")[0].firstChild.data glsa_synopsis = glsa_tree.getElementsByTagName("synopsis")[0].firstChild.data glsa_announced = glsa_tree.getElementsByTagName("announced")[0].firstChild.data glsa_revised = glsa_tree.getElementsByTagName("revised")[0].firstChild.data xml_data['filename'] = xmlfilename xml_data['url'] = "http://www.gentoo.org/security/en/glsa/%s" % (xmlfilename,) xml_data['title'] = glsa_title.strip() xml_data['synopsis'] = glsa_synopsis.strip() xml_data['announced'] = glsa_announced.strip() xml_data['revised'] = glsa_revised.strip() xml_data['bugs'] = ["https://bugs.gentoo.org/show_bug.cgi?id="+x.firstChild.data.strip() for x in glsa_tree.getElementsByTagName("bug")] xml_data['access'] = "" try: xml_data['access'] = glsa_tree.getElementsByTagName("access")[0].firstChild.data.strip() except IndexError: pass # references references = glsa_tree.getElementsByTagName("references")[0] xml_data['references'] = [x.getAttribute("link").strip() for x in references.getElementsByTagName("uri")] try: xml_data['description'] = "" xml_data['description_items'] = [] desc = glsa_tree.getElementsByTagName("description")[0].getElementsByTagName("p")[0].firstChild.data.strip() xml_data['description'] = desc items = glsa_tree.getElementsByTagName("description")[0].getElementsByTagName("ul") for item in items: li_items = item.getElementsByTagName("li") for li_item in li_items: xml_data['description_items'].append(' '.join([x.strip() for x in li_item.firstChild.data.strip().split("\n")])) except IndexError: xml_data['description'] = "" xml_data['description_items'] = [] try: workaround = glsa_tree.getElementsByTagName("workaround")[0] xml_data['workaround'] = workaround.getElementsByTagName("p")[0].firstChild.data.strip() except IndexError: xml_data['workaround'] = "" try: xml_data['resolution'] = [] resolution = glsa_tree.getElementsByTagName("resolution")[0] p_elements = resolution.getElementsByTagName("p") for p_elem in p_elements: xml_data['resolution'].append(p_elem.firstChild.data.strip()) except IndexError: xml_data['resolution'] = [] try: impact = glsa_tree.getElementsByTagName("impact")[0] xml_data['impact'] = impact.getElementsByTagName("p")[0].firstChild.data.strip() except IndexError: xml_data['impact'] = "" xml_data['impacttype'] = glsa_tree.getElementsByTagName("impact")[0].getAttribute("type").strip() try: background = glsa_tree.getElementsByTagName("background")[0] xml_data['background'] = background.getElementsByTagName("p")[0].firstChild.data.strip() except IndexError: xml_data['background'] = "" # affection information affected = glsa_tree.getElementsByTagName("affected")[0] affected_packages = {} # we will then filter affected_packages using repositories information # if not affected_packages: advisory will be dropped for p in affected.getElementsByTagName("package"): name = p.getAttribute("name") if not affected_packages.has_key(name): affected_packages[name] = [] pdata = {} pdata["arch"] = p.getAttribute("arch").strip() pdata["auto"] = (p.getAttribute("auto") == "yes") pdata["vul_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("vulnerable")] pdata["unaff_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("unaffected")] pdata["vul_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("vulnerable")] pdata["unaff_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("unaffected")] affected_packages[name].append(pdata) xml_data['affected'] = affected_packages.copy() return {glsa_id: xml_data} def __make_version(self, vnode): """ creates from the information in the I{versionNode} a version string (format ). @type vnode: xml.dom.Node @param vnode: a or Node that contains the version information for this atom @rtype: String @return: the version string """ return self.op_mappings[vnode.getAttribute("range")] + vnode.firstChild.data.strip() def __make_atom(self, pkgname, vnode): """ creates from the given package name and information in the I{versionNode} a (syntactical) valid portage atom. @type pkgname: String @param pkgname: the name of the package for this atom @type vnode: xml.dom.Node @param vnode: a or Node that contains the version information for this atom @rtype: String @return: the portage atom """ return str(self.op_mappings[vnode.getAttribute("range")] + pkgname + "-" + vnode.firstChild.data.strip()) def check_advisories_availability(self): if not os.path.lexists(etpConst['securitydir']): return False if not os.path.isdir(etpConst['securitydir']): return False else: return True return False def fetch_advisories(self, do_cache = True): mytxt = "%s: %s" % (bold(_("Security Advisories")),blue(_("testing service connection")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "info", header = red(" @@ "), footer = red(" ...") ) mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("getting latest GLSAs")),red("..."),) self.Entropy.updateProgress( mytxt, importance = 2, type = "info", header = red(" @@ ") ) gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock) if gave_up: return 7 locked = self.Entropy.application_lock_check() if locked: self.Entropy._resources_run_remove_lock() return 4 # lock self.Entropy._resources_run_create_lock() try: rc = self.run_fetch() except: self.Entropy._resources_run_remove_lock() raise if rc != 0: return rc self.Entropy._resources_run_remove_lock() if self.advisories_changed: advtext = "%s: %s" % (bold(_("Security Advisories")),darkgreen(_("updated successfully")),) else: advtext = "%s: %s" % (bold(_("Security Advisories")),darkgreen(_("already up to date")),) if do_cache and self.Entropy.xcache: self.get_advisories_metadata() self.Entropy.updateProgress( advtext, importance = 2, type = "info", header = red(" @@ ") ) return 0 def run_fetch(self): # prepare directories self.__prepare_unpack() # download package status = self.__download_glsa_package() self.lastfetch = status if not status: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("unable to download the package, sorry")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 1 mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("Verifying checksum")),red("..."),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" # "), back = True ) # download digest status = self.__download_glsa_package_checksum() if not status: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot download the checksum, sorry")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 2 # verify digest status = self.__verify_checksum() if status == 1: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot open packages, sorry")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 3 elif status == 2: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot read the checksum, sorry")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 4 elif status == 3: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("digest verification failed, sorry")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 5 elif status == 0: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkgreen(_("verification Successful")),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" # ") ) else: mytxt = _("Return status not valid") raise exceptionTools.InvalidData("InvalidData: %s." % (mytxt,)) # save downloaded md5 if os.path.isfile(self.download_package_checksum) and os.path.isdir(etpConst['dumpstoragedir']): if os.path.isfile(self.old_download_package_checksum): os.remove(self.old_download_package_checksum) shutil.copy2(self.download_package_checksum,self.old_download_package_checksum) self.Entropy.setup_default_file_perms(self.old_download_package_checksum) # now unpack in place status = self.__unpack_advisories() if status != 0: mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("digest verification failed, try again later")),) self.Entropy.updateProgress( mytxt, importance = 2, type = "error", header = red(" ## ") ) self.Entropy._resources_run_remove_lock() return 6 mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("installing")),red("..."),) self.Entropy.updateProgress( mytxt, importance = 1, type = "info", header = red(" # ") ) # clear previous self.__clear_previous_advisories() # copy over self.__put_advisories_in_place() # remove temp stuff self.__cleanup_garbage() return 0 class SpmInterface: def __init__(self, OutputInterface): if not isinstance(OutputInterface, (EquoInterface, TextInterface, ServerInterface)): if OutputInterface == None: OutputInterface = TextInterface() else: mytxt = _("A valid TextInterface based instance is needed") raise exceptionTools.IncorrectParameter( "IncorrectParameter: %s" % (mytxt,) ) self.spm_backend = etpConst['spm']['backend'] self.valid_backends = etpConst['spm']['available_backends'] if self.spm_backend not in self.valid_backends: mytxt = "%s: %s" % (_("Invalid backend"),self.spm_backend,) raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) if self.spm_backend == "portage": self.intf = PortageInterface(OutputInterface) @staticmethod def get_spm_interface(): backend = etpConst['spm']['backend'] if backend == "portage": return PortageInterface class PortageInterface: import entropyTools class paren_normalize(list): """Take a dependency structure as returned by paren_reduce or use_reduce and generate an equivalent structure that has no redundant lists.""" def __init__(self, src): list.__init__(self) self._zap_parens(src, self) def _zap_parens(self, src, dest, disjunction=False): if not src: return dest i = iter(src) for x in i: if isinstance(x, basestring): if x == '||': x = self._zap_parens(i.next(), [], disjunction=True) if len(x) == 1: dest.append(x[0]) else: dest.append("||") dest.append(x) elif x.endswith("?"): dest.append(x) dest.append(self._zap_parens(i.next(), [])) else: dest.append(x) else: if disjunction: x = self._zap_parens(x, []) if len(x) == 1: dest.append(x[0]) else: dest.append(x) else: self._zap_parens(x, dest) return dest def __init__(self, OutputInterface): if not isinstance(OutputInterface, (EquoInterface, TextInterface, ServerInterface)): mytxt = _("A valid TextInterface based instance is needed") raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) # interface only needed OutputInterface functions self.updateProgress = OutputInterface.updateProgress self.askQuestion = OutputInterface.askQuestion sys.path.append("/usr/lib/gentoolkit/pym") # importing portage stuff import portage self.portage = portage self.EAPI = 1 try: import portage.const as portage_const except ImportError: import portage_const if hasattr(portage_const,"EAPI"): self.EAPI = portage_const.EAPI self.portage_const = portage_const try: from portage.versions import best self.portage_best = best except ImportError: from portage_versions import best self.portage_best = best try: import portage.util as portage_util except ImportError: import portage_util self.portage_util = portage_util try: import portage.sets as portage_sets self.portage_sets = portage_sets except ImportError: self.portage_sets = None try: import glsa self.glsa = glsa except ImportError: self.glsa = None if hasattr(self.portage,'exception'): self.portage_exception = self.portage.exception else: # portage <2.2 workaround self.portage_exception = Exception self.builtin_pkg_sets = [ "system","world","installed","module-rebuild", "security","preserved-rebuild","live-rebuild", "downgrade","unavailable" ] def write_to_log(self, message): spmLog = LogFile( level = etpConst['spmloglevel'], filename = etpConst['spmlogfile'], header = "[spm]" ) spmLog.write(message) spmLog.flush() spmLog.close() def write_traceback_to_log(self): spmLog = LogFile( level = etpConst['spmloglevel'], filename = etpConst['spmlogfile'], header = "[spm]" ) self.entropyTools.printTraceback(f = spmLog) spmLog.flush() spmLog.close() def list_glsa_packages(self, command = "affected"): if not self.glsa: return if command not in ['new','all','affected']: return glsaconfig = self.glsa.checkconfig(self.portage.config(clone=self.portage.settings)) completelist = self.glsa.get_glsa_list(glsaconfig["GLSA_DIR"], glsaconfig) glsalist = [] if command == "new": checklist = [] if os.access(glsaconfig["CHECKFILE"], os.R_OK): checklist = [line.strip() for line in open(glsaconfig["CHECKFILE"], "r").readlines()] glsalist = [e for e in completelist if e not in checklist] elif command == "all": glsalist = completelist elif command == "affected": # maybe this should be todolist instead for x in completelist: try: myglsa = self.glsa.Glsa(x, glsaconfig) except (self.glsa.GlsaTypeException, self.glsa.GlsaFormatException), e: continue if not myglsa.isVulnerable(): continue glsalist.append(x) return glsalist def get_glsa_id_information(self, glsa_id): if not self.glsa: return {} glsaconfig = self.glsa.checkconfig(self.portage.config(clone=self.portage.settings)) try: myglsa = self.glsa.Glsa(glsa_id, glsaconfig) except (self.glsa.GlsaTypeException, self.glsa.GlsaFormatException): return {} mydict = { 'glsa_id': glsa_id, 'number': myglsa.nr, 'access': myglsa.access, 'title': myglsa.title, 'synopsis': myglsa.synopsis, 'announced': myglsa.announced, 'revised': myglsa.revised, 'bugs': myglsa.bugs, 'description': myglsa.description, 'resolution': myglsa.resolution, 'impact': myglsa.impact_text, 'impacttype': myglsa.impact_type, 'affected': myglsa.affected, 'background': myglsa.background, 'glsatype': myglsa.glsatype, 'packages': myglsa.packages, 'services': myglsa.services, 'product': myglsa.product, 'references': myglsa.references, 'workaround': myglsa.workaround, } if myglsa.isApplied(): status = "[A]" elif myglsa.isVulnerable(): status = "[N]" else: status = "[U]" mydict['status'] = status return mydict.copy() def run_fixpackages(self, myroot = None): if myroot == None: myroot = etpConst['systemroot']+"/" mydb = {} mydb[myroot] = {} mydb[myroot]['vartree'] = self._get_portage_vartree(myroot) mydb[myroot]['porttree'] = self._get_portage_portagetree(myroot) mydb[myroot]['bintree'] = self._get_portage_binarytree(myroot) mydb[myroot]['virtuals'] = self.portage.settings.getvirtuals(myroot) if etpUi['mute']: pid = os.fork() if pid > 0: os.waitpid(pid, 0) else: f = open("/dev/null","w") old_stdout = sys.stdout old_stderr = sys.stderr sys.stdout = f sys.stderr = f self.portage._global_updates(mydb, {}) sys.stdout = old_stdout sys.stderr = old_stderr f.close() os._exit(0) else: self.portage._global_updates(mydb, {}) # always force def get_world_file(self): return os.path.join(etpConst['systemroot'],"/",self.portage_const.WORLD_FILE) def get_third_party_mirrors(self, mirrorname): x = [] if self.portage.thirdpartymirrors.has_key(mirrorname): x = self.portage.thirdpartymirrors[mirrorname] return x def get_spm_setting(self, var): return self.portage.settings[var] def get_config_protect_and_mask(self): config_protect = self.portage.settings['CONFIG_PROTECT'] config_protect = config_protect.split() config_protect_mask = self.portage.settings['CONFIG_PROTECT_MASK'] config_protect_mask = config_protect_mask.split() # explode protect = [] for x in config_protect: x = os.path.expandvars(x) protect.append(x) mask = [] for x in config_protect_mask: x = os.path.expandvars(x) mask.append(x) return ' '.join(protect),' '.join(mask) def _get_portage_vartree(self, root): if not etpConst['spm']['cache'].has_key('portage'): etpConst['spm']['cache']['portage'] = {} if not etpConst['spm']['cache']['portage'].has_key('vartree'): etpConst['spm']['cache']['portage']['vartree'] = {} cached = etpConst['spm']['cache']['portage']['vartree'].get(root) if cached != None: return cached try: mytree = self.portage.vartree(root=root) except Exception, e: raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,)) etpConst['spm']['cache']['portage']['vartree'][root] = mytree return mytree def _get_portage_portagetree(self, root): if not etpConst['spm']['cache'].has_key('portage'): etpConst['spm']['cache']['portage'] = {} if not etpConst['spm']['cache']['portage'].has_key('portagetree'): etpConst['spm']['cache']['portage']['portagetree'] = {} cached = etpConst['spm']['cache']['portage']['portagetree'].get(root) if cached != None: return cached try: mytree = self.portage.portagetree(root=root) except Exception, e: raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,)) etpConst['spm']['cache']['portage']['portagetree'][root] = mytree return mytree def _get_portage_binarytree(self, root): if not etpConst['spm']['cache'].has_key('portage'): etpConst['spm']['cache']['portage'] = {} if not etpConst['spm']['cache']['portage'].has_key('binarytree'): etpConst['spm']['cache']['portage']['binarytree'] = {} cached = etpConst['spm']['cache']['portage']['binarytree'].get(root) if cached != None: return cached pkgdir = root+self.portage.settings['PKGDIR'] try: mytree = self.portage.binarytree(root,pkgdir) except Exception, e: raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,)) etpConst['spm']['cache']['portage']['binarytree'][root] = mytree return mytree def _get_portage_config(self, config_root, root, use_cache = True): if use_cache: if not etpConst['spm']['cache'].has_key('portage'): etpConst['spm']['cache']['portage'] = {} if not etpConst['spm']['cache']['portage'].has_key('config'): etpConst['spm']['cache']['portage']['config'] = {} cached = etpConst['spm']['cache']['portage']['config'].get((config_root,root)) if cached != None: return cached try: mysettings = self.portage.config(config_root = config_root, target_root = root, config_incrementals = self.portage_const.INCREMENTALS) except Exception, e: raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,)) if use_cache: etpConst['spm']['cache']['portage']['config'][(config_root,root)] = mysettings return mysettings # resolve atoms automagically (best, not current!) # sys-libs/application --> sys-libs/application-1.2.3-r1 def get_best_atom(self, atom, match = "bestmatch-visible"): try: return self.portage.portdb.xmatch(match,str(atom)) except ValueError: return None # same as above but includes masked ebuilds def get_best_masked_atom(self, atom): atoms = self.portage.portdb.xmatch("match-all",str(atom)) return self.portage_best(atoms) def get_category_description_data(self, category): from xml.dom import minidom data = {} portdir = self.portage.settings['PORTDIR'] myfile = os.path.join(portdir,category,"metadata.xml") if os.access(myfile,os.R_OK) and os.path.isfile(myfile): doc = minidom.parse(myfile) longdescs = doc.getElementsByTagName("longdescription") for longdesc in longdescs: data[longdesc.getAttribute("lang").strip()] = ' '.join([x.strip() for x in longdesc.firstChild.data.strip().split("\n")]) return data def get_atom_category(self, atom): try: return self.portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0] except: return None # Packages in system (in the Portage language -> emerge system, remember?) def get_atoms_in_system(self): system = self.portage.settings.packages sysoutput = [] for x in system: y = self.get_installed_atoms(x) if (y != None): for z in y: sysoutput.append(z) sysoutput.extend(etpConst['spm']['system_packages']) # add our packages return sysoutput def get_installed_atom(self, atom): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) rc = mytree.dep_match(str(atom)) if rc: return rc[-1] def get_package_description(self, atom): if atom.startswith("="): atom = atom[1:] return self.portage.portdb.aux_get(atom,['DESCRIPTION'])[0] def get_package_ebuild_path(self, atom): if atom.startswith("="): atom = atom[1:] return self.portage.portdb.findname(atom) def get_package_changelog(self, atom): if atom.startswith("="): atom = atom[1:] ebuild_path = self.get_package_ebuild_path(atom) if isinstance(ebuild_path,basestring): cp = os.path.join(os.path.dirname(ebuild_path),"ChangeLog") if os.path.isfile(cp) and os.access(cp,os.R_OK): f = open(cp,"r") txt = f.read() f.close() return txt def get_installed_package_description(self, atom): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) if atom.startswith("="): atom = atom[1:] rc = mytree.dbapi.aux_get(atom, ["DESCRIPTION"])[0] if rc: return rc def get_package_slot(self, atom): if atom.startswith("="): atom = atom[1:] return self.portage.portdb.aux_get(atom,['SLOT'])[0] def get_installed_package_slot(self, atom): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) if atom.startswith("="): atom = atom[1:] rc = mytree.getslot(atom) if rc: return rc def get_installed_atoms(self, atom): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) rc = mytree.dep_match(str(atom)) if rc: return rc def search_keys(self, key): key_split = key.split("/") cat = key_split[0] name = key_split[1] cat_dir = os.path.join(self.get_vdb_path(),cat) if not os.path.isdir(cat_dir): return None dir_content = [os.path.join(cat,x) for x in os.listdir(cat_dir) if x.startswith(name)] if not dir_content: return None return dir_content # create a .tbz2 file in the specified path def quickpkg(self, atom, dirpath): # getting package info pkgname = atom.split("/")[1] pkgcat = atom.split("/")[0] #pkgfile = pkgname+".tbz2" if not os.path.isdir(dirpath): os.makedirs(dirpath) dirpath += "/"+pkgname+etpConst['packagesext'] dbdir = self.get_vdb_path()+"/"+pkgcat+"/"+pkgname+"/" import tarfile import stat trees = self.portage.db["/"] vartree = trees["vartree"] dblnk = self.portage.dblink(pkgcat, pkgname, "/", vartree.settings, treetype="vartree", vartree=vartree) dblnk.lockdb() tar = tarfile.open(dirpath,"w:bz2") contents = dblnk.getcontents() id_strings = {} paths = sorted(contents.keys()) for path in paths: try: exist = os.lstat(path) except OSError: continue # skip file ftype = contents[path][0] lpath = path arcname = path[1:] if 'dir' == ftype and \ not stat.S_ISDIR(exist.st_mode) and \ os.path.isdir(lpath): lpath = os.path.realpath(lpath) tarinfo = tar.gettarinfo(lpath, arcname) tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) if stat.S_ISREG(exist.st_mode): tarinfo.type = tarfile.REGTYPE f = open(path) try: tar.addfile(tarinfo, f) finally: f.close() else: tar.addfile(tarinfo) tar.close() # appending xpak informations import etpXpak tbz2 = etpXpak.tbz2(dirpath) tbz2.recompose(dbdir) dblnk.unlockdb() if os.path.isfile(dirpath): return dirpath else: raise exceptionTools.FileNotFound("FileNotFound: Spm:quickpkg %s: %s %s" % ( _("error"), dirpath, _("not found"), ) ) def get_package_use_file(self): return os.path.join(self.portage_const.USER_CONFIG_PATH,'package.use') def enable_package_useflags(self, atom, useflags): result = self.unset_package_useflags(atom, useflags) if not result: return False return self._handle_new_useflags(atom, useflags, "") def disable_package_useflags(self, atom, useflags): result = self.unset_package_useflags(atom, useflags) if not result: return False return self._handle_new_useflags(atom, useflags, "-") def _handle_new_useflags(self, atom, useflags, mark): matched_atom = self.get_best_atom(atom) if not matched_atom: return False use_file = self.get_package_use_file() if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)): return False f = open(use_file,"r") content = [x.strip() for x in f.readlines()] f.close() def handle_line(line, useflags): data = line.split() if len(data) < 2: return False, line myatom = data[0] if matched_atom != self.get_best_atom(myatom): return False, line flags = data[1:] base_flags = [] added_flags = [] for flag in flags: myflag = flag if myflag.startswith("+"): myflag = myflag[1:] elif myflag.startswith("-"): myflag = myflag[1:] if not myflag: continue base_flags.append(myflag) for useflag in useflags: if mark+useflag in base_flags: continue added_flags.append(mark+useflag) new_line = "%s %s" % (myatom, ' '.join(flags+added_flags)) return True, new_line atom_found = False new_content = [] for line in content: changed, elaborated_line = handle_line(line, useflags) if changed: atom_found = True new_content.append(elaborated_line) if not atom_found: myline = "%s %s" % (atom, ' '.join([mark+x for x in useflags])) new_content.append(myline) f = open(use_file+".tmp","w") for line in new_content: f.write(line+"\n") f.flush() f.close() shutil.move(use_file+".tmp",use_file) return True def unset_package_useflags(self, atom, useflags): matched_atom = self.get_best_atom(atom) if not matched_atom: return False use_file = self.get_package_use_file() if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)): return False f = open(use_file,"r") content = [x.strip() for x in f.readlines()] f.close() new_content = [] for line in content: data = line.split() if len(data) < 2: new_content.append(line) continue myatom = data[0] if matched_atom != self.get_best_atom(myatom): new_content.append(line) continue flags = data[1:] new_flags = [] for flag in flags: myflag = flag if myflag.startswith("+"): myflag = myflag[1:] elif myflag.startswith("-"): myflag = myflag[1:] if myflag in useflags: continue elif not flag: continue new_flags.append(flag) if new_flags: new_line = "%s %s" % (myatom, ' '.join(new_flags)) new_content.append(new_line) f = open(use_file+".tmp","w") for line in new_content: f.write(line+"\n") f.flush() f.close() shutil.move(use_file+".tmp",use_file) return True def get_package_useflags(self, atom): matched_atom = self.get_best_atom(atom) if not matched_atom: return {} global_useflags = self.get_useflags() use_force = self.get_useflags_force() use_mask = self.get_useflags_mask() package_use_useflags = self.get_package_use_useflags(atom) data = {} data['use_force'] = use_force.copy() data['use_mask'] = use_mask.copy() data['global_use'] = global_useflags.split() iuse = self.get_package_setting(atom, "IUSE") if not isinstance(iuse,basestring): iuse = '' data['iuse'] = iuse.split()[:] iuse = set() for myiuse in data['iuse']: if myiuse.startswith("+"): myiuse = myiuse[1:] iuse.add(myiuse) use = [f for f in data['global_use']+list(package_use_useflags['enabled']) if (f in iuse) and (f not in use_mask) and (f not in package_use_useflags['disabled'])] use_disabled = [f for f in iuse if (f not in data['global_use']) and (f not in use_mask) and (f not in package_use_useflags['enabled'])] data['use'] = use[:] data['use_disabled'] = use_disabled[:] matched_slot = self.get_package_slot(matched_atom) try: installed_atom = self.get_installed_atom("%s:%s" % (self.entropyTools.dep_getkey(atom),matched_slot,)) except self.portage_exception: installed_atom = None if installed_atom: # get its useflags previous_iuse = self.get_installed_package_setting(installed_atom, "IUSE").split() previous_use = self.get_installed_package_setting(installed_atom, "USE").split() new_previous_iuse = set() for myuse in previous_iuse: if myuse.startswith("+"): myuse = myuse[1:] new_previous_iuse.add(myuse) previous_iuse = list(new_previous_iuse) inst_use = [f for f in previous_iuse if (f in previous_use) and (f not in use_mask)] #inst_use_disabled = [f for f in previous_use if (f not in previous_iuse) and (f not in use_mask)] # check removed use use_removed = [] for myuse in inst_use: if myuse not in use: use_removed.append(myuse) # use not available use_not_avail = [] for myuse in previous_iuse: if (myuse not in iuse) and (myuse not in use_removed): use_not_avail.append(myuse) # check new use t_use = [] for myuse in use: if myuse not in inst_use: myuse = "+%s*" % (myuse,) t_use.append(myuse) use = t_use # check disabled use t_use_disabled = [] for myuse in use_disabled: if myuse in inst_use: if myuse in use_removed+use_not_avail: continue myuse = "-%s*" % (myuse,) else: myuse = "-%s" % (myuse,) t_use_disabled.append(myuse) use_disabled = t_use_disabled for myuse in use_removed: use_disabled.append("(-%s*)" % (myuse,)) for myuse in use_not_avail: use_disabled.append("(-%s)" % (myuse,)) else: use_disabled = ["-"+x for x in use_disabled] data['use_string'] = ' '.join(sorted(use)+sorted([x for x in use_disabled])) data['use_string_colored'] = ' '.join( sorted([darkred(x) for x in use if not x.startswith("+")] + \ [darkgreen(x) for x in use if x.startswith("+")]) + \ sorted([darkblue(x) for x in use_disabled if x.startswith("-")] + \ [brown(x) for x in use_disabled if x.startswith("(") and (x.find("*") == -1)] + \ [purple(x) for x in use_disabled if x.startswith("(") and (x.find("*") != -1)] ) ) return data def get_installed_package_useflags(self, atom): matched_atom = self.get_installed_atom(atom) if not matched_atom: return {} global_use = self.get_installed_package_setting(matched_atom, "USE") use_mask = self.get_useflags_mask() data = {} data['use_mask'] = use_mask.copy() data['global_use'] = global_use.split() iuse = self.get_installed_package_setting(matched_atom, "IUSE") if not isinstance(iuse,basestring): iuse = '' data['iuse'] = iuse.split()[:] iuse = set() for myiuse in data['iuse']: if myiuse.startswith("+"): myiuse = myiuse[1:] iuse.add(myiuse) use = [f for f in data['global_use'] if (f in iuse) and (f not in use_mask)] use_disabled = [f for f in iuse if (f not in data['global_use']) and (f not in use_mask)] data['use'] = use[:] data['use_disabled'] = use_disabled[:] data['use_string'] = ' '.join(sorted(use)+sorted([x for x in use_disabled])) data['use_string_colored'] = ' '.join( sorted([darkred(x) for x in use if not x.startswith("+")] + \ [darkgreen(x) for x in use if x.startswith("+")]) + \ sorted([darkblue(x) for x in use_disabled if x.startswith("-")] + \ [brown(x) for x in use_disabled if x.startswith("(") and (x.find("*") == -1)] + \ [purple(x) for x in use_disabled if x.startswith("(") and (x.find("*") != -1)] ) ) return data # package.use def get_package_use_useflags(self, atom): data = { 'enabled': set(), 'disabled': set(), } matched_atom = self.get_best_atom(atom) if not matched_atom: return data use_file = self.get_package_use_file() if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)): return data use_data = self.portage_util.grabdict(use_file) for myatom in use_data: mymatch = self.get_best_atom(myatom) if mymatch != matched_atom: continue for flag in use_data[myatom]: if flag.startswith("-"): myflag = flag[1:] data['enabled'].discard(myflag) data['disabled'].add(myflag) else: myflag = flag if myflag.startswith("+"): myflag = myflag[1:] data['disabled'].discard(myflag) data['enabled'].add(myflag) return data def get_useflags(self): return self.portage.settings['USE'] def get_useflags_force(self): return self.portage.settings.useforce def get_useflags_mask(self): return self.portage.settings.usemask def get_installed_package_setting(self, atom, setting): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) if atom.startswith("="): atom = atom[1:] return mytree.dbapi.aux_get(atom, [setting])[0] def get_package_setting(self, atom, setting): if atom.startswith("="): atom = atom[1:] return self.portage.portdb.aux_get(atom,[setting])[0] def query_files(self, atom): mypath = etpConst['systemroot']+"/" mysplit = atom.split("/") content = self.portage.dblink(mysplit[0], mysplit[1], mypath, self.portage.settings).getcontents() return content.keys() def query_belongs(self, filename, like = False): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) packages = mytree.dbapi.cpv_all() matches = set() for package in packages: mysplit = package.split("/") content = self.portage.dblink(mysplit[0], mysplit[1], mypath, self.portage.settings).getcontents() if not like: if filename in content: matches.add(package) else: for myfile in content: if myfile.find(filename) != -1: matches.add(package) return matches def query_belongs_multiple(self, filenames, like = False): mypath = etpConst['systemroot']+"/" mytree = self._get_portage_vartree(mypath) packages = mytree.dbapi.cpv_all() matches = {} filenames = filenames.copy() for package in packages: cat, pkgv = package.split("/") content = self.portage.dblink(cat, pkgv, mypath, self.portage.settings).getcontents() if not like: for filename in filenames: if filename in content: myslot = self.get_installed_package_slot(package) if not matches.has_key((package,myslot)): matches[(package,myslot)] = set() matches[(package,myslot)].add(filename) else: for filename in filenames: for myfile in content: if myfile.find(filename) != -1: myslot = self.get_installed_package_slot(package) if not matches.has_key((package,myslot)): matches[(package,myslot)] = set() matches[(package,myslot)].add(filename) return matches def calculate_dependencies(self, my_iuse, my_use, my_license, my_depend, my_rdepend, my_pdepend, my_provide, my_src_uri): metadata = {} metadata['USE'] = my_use metadata['IUSE'] = my_iuse metadata['LICENSE'] = my_license metadata['DEPEND'] = my_depend metadata['PDEPEND'] = my_pdepend metadata['RDEPEND'] = my_rdepend metadata['PROVIDE'] = my_provide metadata['SRC_URI'] = my_src_uri use = metadata['USE'].split() raw_use = use metadata['USE_MASK'] = self.get_useflags_mask() iuse = set() for myiuse in metadata['IUSE'].split(): if myiuse.startswith("+"): myiuse = myiuse[1:] if (myiuse not in use) and ("-"+myiuse not in use): use.append(myiuse) elif myiuse.startswith("-"): myiuse = myiuse[1:] iuse.add(myiuse) use = sorted([f for f in use if f in iuse]) metadata['USE'] = " ".join(use) for k in "LICENSE", "RDEPEND", "DEPEND", "PDEPEND", "PROVIDE", "SRC_URI": try: deps = self.paren_reduce(metadata[k]) deps = self.use_reduce(deps, uselist=raw_use) deps = self.paren_normalize(deps) if k == "LICENSE": deps = self.paren_license_choose(deps) else: deps = self.paren_choose(deps) if k.endswith("DEPEND"): deps = self.usedeps_reduce(deps) deps = ' '.join(deps) except Exception, e: self.entropyTools.printTraceback() self.updateProgress( darkred("%s: %s: %s :: %s") % ( _("Error calculating dependencies"), str(Exception), k, e, ), importance = 1, type = "error", header = red(" !!! ") ) deps = '' continue metadata[k] = deps return metadata def usedeps_reduce(self, dependencies): newlist = [] for dependency in dependencies: use_deps = self.entropyTools.dep_getusedeps(dependency) if use_deps: use_deps = [x for x in use_deps if (x[0] not in ("!",)) and (x[-1] not in ("=","?",))] if use_deps: dependency = "%s[%s]" % (self.entropyTools.remove_usedeps(dependency),','.join(use_deps),) else: dependency = self.entropyTools.remove_usedeps(dependency) newlist.append(dependency) return newlist def paren_reduce(self, mystr,tokenize=1): """ # deps.py -- Portage dependency resolution functions # Copyright 2003-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $ Take a string and convert all paren enclosed entities into sublists, optionally futher splitting the list elements by spaces. Example usage: >>> paren_reduce('foobar foo ( bar baz )',1) ['foobar', 'foo', ['bar', 'baz']] >>> paren_reduce('foobar foo ( bar baz )',0) ['foobar foo ', [' bar baz ']] @param mystr: The string to reduce @type mystr: String @param tokenize: Split on spaces to produces further list breakdown @type tokenize: Integer @rtype: Array @return: The reduced string in an array """ mylist = [] while mystr: left_paren = mystr.find("(") has_left_paren = left_paren != -1 right_paren = mystr.find(")") has_right_paren = right_paren != -1 if not has_left_paren and not has_right_paren: freesec = mystr subsec = None tail = "" elif mystr[0] == ")": return [mylist,mystr[1:]] elif has_left_paren and not has_right_paren: raise exceptionTools.InvalidDependString( "InvalidDependString: %s: '%s'" % (_("missing right parenthesis"),mystr,)) elif has_left_paren and left_paren < right_paren: freesec,subsec = mystr.split("(",1) subsec,tail = self.paren_reduce(subsec,tokenize) else: subsec,tail = mystr.split(")",1) if tokenize: subsec = self.strip_empty(subsec.split(" ")) return [mylist+subsec,tail] return mylist+[subsec],tail mystr = tail if freesec: if tokenize: mylist = mylist + self.strip_empty(freesec.split(" ")) else: mylist = mylist + [freesec] if subsec is not None: mylist = mylist + [subsec] return mylist def strip_empty(self, myarr): """ # deps.py -- Portage dependency resolution functions # Copyright 2003-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $ Strip all empty elements from an array @param myarr: The list of elements @type myarr: List @rtype: Array @return: The array with empty elements removed """ for x in range(len(myarr)-1, -1, -1): if not myarr[x]: del myarr[x] return myarr def use_reduce(self, deparray, uselist=[], masklist=[], matchall=0, excludeall=[]): """ # deps.py -- Portage dependency resolution functions # Copyright 2003-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $ Takes a paren_reduce'd array and reduces the use? conditionals out leaving an array with subarrays @param deparray: paren_reduce'd list of deps @type deparray: List @param uselist: List of use flags @type uselist: List @param masklist: List of masked flags @type masklist: List @param matchall: Resolve all conditional deps unconditionally. Used by repoman @type matchall: Integer @rtype: List @return: The use reduced depend array """ # Quick validity checks for x in range(len(deparray)): if deparray[x] in ["||","&&"]: if len(deparray) - 1 == x or not isinstance(deparray[x+1], list): mytxt = _("missing atom list in") raise exceptionTools.InvalidDependString(deparray[x]+" "+mytxt+" \""+str(deparray)+"\"") if deparray and deparray[-1] and deparray[-1][-1] == "?": mytxt = _("Conditional without target in") raise exceptionTools.InvalidDependString("InvalidDependString: "+mytxt+" \""+str(deparray)+"\"") # This is just for use by emerge so that it can enable a backward compatibility # mode in order to gracefully deal with installed packages that have invalid # atoms or dep syntax. For backward compatibility with api consumers, strict # behavior will be explicitly enabled as necessary. _dep_check_strict = False mydeparray = deparray[:] rlist = [] while mydeparray: head = mydeparray.pop(0) if isinstance(head,list): additions = self.use_reduce(head, uselist, masklist, matchall, excludeall) if additions: rlist.append(additions) elif rlist and rlist[-1] == "||": #XXX: Currently some DEPEND strings have || lists without default atoms. # raise portage_exception.InvalidDependString("No default atom(s) in \""+paren_enclose(deparray)+"\"") rlist.append([]) else: if head[-1] == "?": # Use reduce next group on fail. # Pull any other use conditions and the following atom or list into a separate array newdeparray = [head] while isinstance(newdeparray[-1], str) and newdeparray[-1][-1] == "?": if mydeparray: newdeparray.append(mydeparray.pop(0)) else: raise ValueError, _("Conditional with no target") # Deprecation checks warned = 0 if len(newdeparray[-1]) == 0: mytxt = "%s. (%s)" % (_("Empty target in string"),_("Deprecated"),) self.updateProgress( darkred("PortageInterface.use_reduce(): %s" % (mytxt,)), importance = 0, type = "error", header = bold(" !!! ") ) warned = 1 if len(newdeparray) != 2: mytxt = "%s. (%s)" % (_("Nested use flags without parenthesis"),_("Deprecated"),) self.updateProgress( darkred("PortageInterface.use_reduce(): %s" % (mytxt,)), importance = 0, type = "error", header = bold(" !!! ") ) warned = 1 if warned: self.updateProgress( darkred("PortageInterface.use_reduce(): "+" ".join(map(str,[head]+newdeparray))), importance = 0, type = "error", header = bold(" !!! ") ) # Check that each flag matches ismatch = True missing_flag = False for head in newdeparray[:-1]: head = head[:-1] if not head: missing_flag = True break if head.startswith("!"): head_key = head[1:] if not head_key: missing_flag = True break if not matchall and head_key in uselist or \ head_key in excludeall: ismatch = False break elif head not in masklist: if not matchall and head not in uselist: ismatch = False break else: ismatch = False if missing_flag: mytxt = _("Conditional without flag") raise exceptionTools.InvalidDependString( "InvalidDependString: "+mytxt+": \"" + \ str([head+"?", newdeparray[-1]])+"\"") # If they all match, process the target if ismatch: target = newdeparray[-1] if isinstance(target, list): additions = self.use_reduce(target, uselist, masklist, matchall, excludeall) if additions: rlist.append(additions) elif not _dep_check_strict: # The old deprecated behavior. rlist.append(target) else: mytxt = _("Conditional without parenthesis") raise exceptionTools.InvalidDependString( "InvalidDependString: "+mytxt+": '%s?'" % head) else: rlist += [head] return rlist def paren_choose(self, dep_list): newlist = [] do_skip = False for idx in range(len(dep_list)): if do_skip: do_skip = False continue item = dep_list[idx] if item == "||": # or next_item = dep_list[idx+1] if not next_item: # || ( asd? ( atom ) dsa? ( atom ) ) => [] if use asd and dsa are disabled do_skip = True continue item = self.dep_or_select(next_item) # must be a list if not item: # no matches, transform to string and append, so reagent will fail newlist.append(str(next_item)) else: newlist += item do_skip = True elif isinstance(item, list): # and item = self.dep_and_select(item) newlist += item else: newlist.append(item) return newlist def dep_and_select(self, and_list): do_skip = False newlist = [] for idx in range(len(and_list)): if do_skip: do_skip = False continue x = and_list[idx] if x == "||": x = self.dep_or_select(and_list[idx+1]) do_skip = True if not x: x = str(and_list[idx+1]) else: newlist += x elif isinstance(x, list): x = self.dep_and_select(x) newlist += x else: newlist.append(x) # now verify if all are satisfied for x in newlist: match = self.get_installed_atom(x) if match == None: return [] return newlist def dep_or_select(self, or_list): do_skip = False for idx in range(len(or_list)): if do_skip: do_skip = False continue x = or_list[idx] if x == "||": # or x = self.dep_or_select(or_list[idx+1]) do_skip = True elif isinstance(x, list): # and x = self.dep_and_select(x) if not x: continue # found return x else: x = [x] for y in x: match = self.get_installed_atom(y) if match != None: return [y] return [] def paren_license_choose(self, dep_list): newlist = set() for item in dep_list: if isinstance(item, list): # match the first data = set(self.paren_license_choose(item)) newlist.update(data) else: if item not in ["||"]: newlist.add(item) return list(newlist) def get_vdb_path(self): rc = etpConst['systemroot']+"/"+self.portage_const.VDB_PATH if (not rc.endswith("/")): return rc+"/" return rc def get_available_packages(self, categories = [], filter_reinstalls = True): mypath = etpConst['systemroot']+"/" mysettings = self._get_portage_config("/",mypath) portdb = self.portage.portdbapi(mysettings["PORTDIR"], mysettings = mysettings) cps = portdb.cp_all() visibles = set() for cp in cps: if categories and cp.split("/")[0] not in categories: continue # get slots slots = set() atoms = self.get_best_atom(cp, "match-visible") if atoms: for atom in atoms: slots.add(portdb.aux_get(atom, ["SLOT"])[0]) for slot in slots: visibles.add(cp+":"+slot) del cps # now match visibles available = set() for visible in visibles: match = self.get_best_atom(visible) if match == None: continue if filter_reinstalls: installed = self.get_installed_atom(visible) # if not installed, installed == None if installed != match: available.add(match) else: available.add(match) del visibles return available # Collect installed packages def get_installed_packages(self, dbdir = None, categories = []): if not dbdir: appDbDir = self.get_vdb_path() else: appDbDir = dbdir dbDirs = os.listdir(appDbDir) installedAtoms = set() for pkgsdir in dbDirs: if os.path.isdir(appDbDir+pkgsdir): pkgdir = os.listdir(appDbDir+pkgsdir) for pdir in pkgdir: pkgcat = pkgsdir.split("/")[-1] if categories and (pkgcat not in categories): continue pkgatom = pkgcat+"/"+pdir if pkgatom.find("-MERGING-") == -1: installedAtoms.add(pkgatom) return sorted(list(installedAtoms)), len(installedAtoms) def get_installed_packages_counter(self, dbdir = None): if not dbdir: appDbDir = self.get_vdb_path() else: appDbDir = dbdir installedAtoms = set() for current_dirpath, subdirs, files in os.walk(appDbDir): pvs = os.listdir(current_dirpath) for mypv in pvs: if mypv.startswith("-MERGING-"): continue mypvpath = current_dirpath+"/"+mypv if not os.path.isdir(mypvpath): continue mycounter_file = mypvpath+"/"+etpConst['spm']['xpak_entries']['counter'] if not os.access(mycounter_file,os.R_OK): continue f = open(mycounter_file) try: counter = int(f.readline().strip()) except (IOError, ValueError): f.close() continue installedAtoms.add((os.path.basename(current_dirpath)+"/"+mypv,counter)) return installedAtoms def _load_sets_config(self, settings, trees): # from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH setconfigpaths = [os.path.join(self.portage_const.GLOBAL_CONFIG_PATH, etpConst['setsconffilename'])] setconfigpaths.append(os.path.join(settings["PORTDIR"], etpConst['setsconffilename'])) setconfigpaths += [os.path.join(x, etpConst['setsconffilename']) for x in settings["PORTDIR_OVERLAY"].split()] setconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], self.portage_const.USER_CONFIG_PATH.lstrip(os.path.sep), etpConst['setsconffilename'])) return self.portage_sets.SetConfig(setconfigpaths, settings, trees) def get_set_config(self): # old portage if self.portage_sets == None: return myroot = etpConst['systemroot']+"/" return self._load_sets_config( self.portage.settings, self.portage.db[myroot] ) def get_sets(self, builtin_sets): config = self.get_set_config() if config == None: return {} mysets = config.getSets() if not builtin_sets: builtin_pkg_sets = [x for x in self.builtin_pkg_sets if x in mysets] for pkg_set in builtin_pkg_sets: mysets.pop(pkg_set) return mysets def get_sets_expanded(self, builtin_sets = True): config = self.get_set_config() if config == None: return {} mysets = {} sets = config.getSets() if not builtin_sets: builtin_pkg_sets = [x for x in self.builtin_pkg_sets if x in sets] for pkg_set in builtin_pkg_sets: sets.pop(pkg_set) for myset in sorted(sets): try: atoms = config.getSetAtoms(myset).copy() except: continue mysets[myset] = atoms return mysets def refill_counter(self, dbdir = None): if not dbdir: appDbDir = self.get_vdb_path() else: appDbDir = dbdir counters = set() for catdir in os.listdir(appDbDir): catdir = appDbDir+catdir if not os.path.isdir(catdir): continue for pkgdir in os.listdir(catdir): pkgdir = catdir+"/"+pkgdir if not os.path.isdir(pkgdir): continue counterfile = pkgdir+"/"+etpConst['spm']['xpak_entries']['counter'] if not os.path.isfile(pkgdir+"/"+etpConst['spm']['xpak_entries']['counter']): continue try: f = open(counterfile,"r") counter = int(f.readline().strip()) counters.add(counter) f.close() except: continue if counters: newcounter = max(counters) else: newcounter = 0 if not os.path.isdir(os.path.dirname(etpConst['edbcounter'])): os.makedirs(os.path.dirname(etpConst['edbcounter'])) try: f = open(etpConst['edbcounter'],"w") except IOError, e: if e[0] == 21: shutil.rmtree(etpConst['edbcounter'],True) try: os.rmdir(etpConst['edbcounter']) except: pass f = open(etpConst['edbcounter'],"w") f.write(str(newcounter)) f.flush() f.close() del counters return newcounter def spm_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = [], fork = False): if fork: # memory leak: some versions of portage were memleaking here return self.entropyTools.spawnFunction( self._portage_doebuild, myebuild, mydo, tree, cpv, portage_tmpdir, licenses ) return self._portage_doebuild(myebuild, mydo, tree, cpv, portage_tmpdir, licenses) def _portage_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []): # myebuild = path/to/ebuild.ebuild with a valid unpacked xpak metadata # tree = "bintree" # cpv = atom # mydbapi = portage.fakedbapi(settings=portage.settings) # vartree = portage.vartree(root=myroot) oldsystderr = sys.stderr f = open("/dev/null","w") if not etpUi['debug']: sys.stderr = f ### SETUP ENVIRONMENT # if mute, supress portage output domute = False if etpUi['mute']: domute = True oldsysstdout = sys.stdout sys.stdout = f mypath = etpConst['systemroot']+"/" os.environ["SKIP_EQUO_SYNC"] = "1" os.environ["CD_ROOT"] = "/tmp" # workaround for scripts asking for user intervention os.environ["ROOT"] = mypath if licenses: os.environ["ACCEPT_LICENSE"] = str(' '.join(licenses)) # we already do this early # load metadata myebuilddir = os.path.dirname(myebuild) keys = self.portage.auxdbkeys metadata = {} for key in keys: mykeypath = os.path.join(myebuilddir,key) if os.path.isfile(mykeypath) and os.access(mykeypath,os.R_OK): f = open(mykeypath,"r") metadata[key] = f.readline().strip() f.close() ### END SETUP ENVIRONMENT # find config mysettings = self._get_portage_config("/",mypath) mysettings['EBUILD_PHASE'] = mydo mysettings['EAPI'] = "0" if metadata.has_key('EAPI'): mysettings['EAPI'] = metadata['EAPI'] mysettings.backup_changes("EAPI") try: # this is a >portage-2.1.4_rc11 feature mysettings._environ_whitelist = set(mysettings._environ_whitelist) # put our vars into whitelist mysettings._environ_whitelist.add("SKIP_EQUO_SYNC") mysettings._environ_whitelist.add("ACCEPT_LICENSE") mysettings._environ_whitelist.add("CD_ROOT") mysettings._environ_whitelist.add("ROOT") mysettings._environ_whitelist = frozenset(mysettings._environ_whitelist) except: self.write_traceback_to_log() cpv = str(cpv) mysettings.setcpv(cpv) portage_tmpdir_created = False # for pkg_postrm, pkg_prerm if portage_tmpdir: if not os.path.isdir(portage_tmpdir): os.makedirs(portage_tmpdir) portage_tmpdir_created = True mysettings['PORTAGE_TMPDIR'] = str(portage_tmpdir) mysettings.backup_changes("PORTAGE_TMPDIR") mydbapi = self.portage.fakedbapi(settings=mysettings) mydbapi.cpv_inject(cpv, metadata = metadata) # cached vartree class vartree = self._get_portage_vartree(mypath) try: rc = self.portage.doebuild( myebuild = str(myebuild), mydo = str(mydo), myroot = mypath, tree = tree, mysettings = mysettings, mydbapi = mydbapi, vartree = vartree, use_cache = 0 ) except: self.write_traceback_to_log() raise # if mute, restore old stdout/stderr if domute: sys.stdout = oldsysstdout sys.stderr = oldsystderr f.close() if portage_tmpdir_created: shutil.rmtree(portage_tmpdir,True) del mydbapi del metadata del keys return rc class LogFile: def __init__ (self, level = 0, filename = None, header = "[LOG]"): self.handler = self.default_handler self.level = level self.header = header self.logFile = None self.open(filename) def __del__(self): self.close() def close(self): try: self.logFile.close() except (IOError,OSError,): pass def flush(self): self.logFile.flush() def fileno(self): return self.getFile() def isatty(self): return False def read(self, a): return '' def readline(self): return '' def readlines(self): return [] def seek(self, a): return self.logFile.seek(a) def tell(self): return self.logFile.tell() def truncate(self): return self.logFile.truncate() def open (self, file = None): if isinstance(file,basestring): if os.access(file,os.W_OK) and os.path.isfile(file): self.logFile = open(file, "aw") else: self.logFile = open("/dev/null", "aw") elif hasattr(file,'write'): self.logFile = file else: self.logFile = sys.stderr def getFile (self): return self.logFile.fileno() def __call__(self, format, *args): self.handler (format % args) def default_handler (self, mystr): try: self.logFile.write ("* %s\n" % (mystr)) except UnicodeEncodeError: self.logFile.write ("* %s\n" % (mystr.encode('utf-8'),)) self.logFile.flush() def set_loglevel(self, level): self.level = level def log(self, messagetype, level, message): if self.level >= level and not etpUi['nolog']: self.handler("%s %s %s %s" % (self.getTimeDateHeader(),messagetype,self.header,message,)) def write(self, s): self.handler(s) def writelines(self, lst): for s in lst: self.write(s) def getTimeDateHeader(self): return time.strftime('[%H:%M:%S %d/%m/%Y %Z]') class SocketCommandsSkel: def __str__(self): return self.inst_name inst_name = 'command-skel' def __init__(self, HostInterface, inst_name = 'command-skel'): self.HostInterface = HostInterface self.no_acked_commands = [] self.termination_commands = [] self.initialization_commands = [] self.login_pass_commands = [] self.no_session_commands = [] self.raw_commands = [] self.config_commands = [] self.valid_commands = set() self.inst_name = inst_name def register( self, valid_commands, no_acked_commands, termination_commands, initialization_commands, login_pass_commads, no_session_commands, raw_commands, config_commands ): valid_commands.update(self.valid_commands) no_acked_commands.extend(self.no_acked_commands) termination_commands.extend(self.termination_commands) initialization_commands.extend(self.initialization_commands) login_pass_commads.extend(self.login_pass_commands) no_session_commands.extend(self.no_session_commands) raw_commands.extend(self.raw_commands) config_commands.extend(self.config_commands) class SocketAuthenticatorSkel: def __init__(self, HostInterface): self.HostInterface = HostInterface self.session = None def set_session(self, session): self.session = session class SocketHostInterface: import socket import SocketServer import entropyTools from threading import Thread class BasicPamAuthenticator(SocketAuthenticatorSkel): import entropyTools def __init__(self, HostInterface, *args, **kwargs): self.valid_auth_types = [ "plain", "shadow", "md5" ] SocketAuthenticatorSkel.__init__(self, HostInterface) def docmd_login(self, arguments): # filter n00bs if not arguments or (len(arguments) != 3): return False,None,None,'wrong arguments' user = arguments[0] auth_type = arguments[1] auth_string = arguments[2] # check auth type validity if auth_type not in self.valid_auth_types: return False,user,None,'invalid auth type' udata = self.__get_user_data(user) if udata == None: return False,user,None,'invalid user' uid = udata[2] # check if user is in the Entropy group if not self.entropyTools.is_user_in_entropy_group(uid): return False,user,uid,'user not in %s group' % (etpConst['sysgroup'],) # now validate password valid = self.__validate_auth(user,auth_type,auth_string) if not valid: return False,user,uid,'auth failed' if not uid: self.HostInterface.sessions[self.session]['admin'] = True else: self.HostInterface.sessions[self.session]['user'] = True return True,user,uid,"ok" # it we get here is because user is logged in def docmd_userdata(self): auth_uid = self.HostInterface.sessions[self.session]['auth_uid'] mydata = {} udata = self.__get_uid_data(auth_uid) if udata: mydata['username'] = udata[0] mydata['uid'] = udata[2] mydata['gid'] = udata[3] mydata['references'] = udata[4] mydata['home'] = udata[5] mydata['shell'] = udata[6] return True,mydata,'ok' def __get_uid_data(self, user_id): import pwd # check user validty try: udata = pwd.getpwuid(user_id) except KeyError: return None return udata def __get_user_data(self, user): import pwd # check user validty try: udata = pwd.getpwnam(user) except KeyError: return None return udata def __validate_auth(self, user, auth_type, auth_string): valid = False if auth_type == "plain": valid = self.__do_auth(user, auth_string) elif auth_type == "shadow": valid = self.__do_auth(user, auth_string, auth_type = "shadow") elif auth_type == "md5": valid = self.__do_auth(user, auth_string, auth_type = "md5") return valid def __do_auth(self, user, password, auth_type = None): import spwd try: enc_pass = spwd.getspnam(user)[1] except KeyError: return False if auth_type == None: # plain import crypt generated_pass = crypt.crypt(str(password), enc_pass) elif auth_type == "shadow": generated_pass = password elif auth_type == "md5": # md5 import hashlib m = hashlib.md5() m.update(enc_pass) enc_pass = m.hexdigest() generated_pass = str(password) else: # haha, fuck! generated_pass = None if generated_pass == enc_pass: return True return False def docmd_logout(self, myargs): # filter n00bs if (len(myargs) < 1) or (len(myargs) > 1): return False,None,'wrong arguments' user = myargs[0] # filter n00bs if not user or not isinstance(user,basestring): return False,None,"wrong user" return True,user,"ok" def set_exc_permissions(self, uid, gid): if gid != None: os.setgid(gid) if uid != None: os.setuid(uid) def hide_login_data(self, args): myargs = args[:] myargs[-1] = 'hidden' return myargs def terminate_instance(self): pass class HostServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): class ConnWrapper: ''' Base class for implementing the rest of the wrappers in this module. Operates by taking a connection argument which is used when 'self' doesn't provide the functionality being requested. ''' def __init__(self, connection) : self.connection = connection def __getattr__(self, function) : return getattr(self.connection, function) import socket as socket_mod import select import SocketServer # This means the main server will not do the equivalent of a # pthread_join() on the new threads. With this set, Ctrl-C will # kill the server reliably. daemon_threads = True # By setting this we allow the server to re-bind to the address by # setting SO_REUSEADDR, meaning you don't have to wait for # timeouts when you kill the server and the sockets don't get # closed down correctly. allow_reuse_address = True def __init__(self, server_address, RequestHandlerClass, processor, HostInterface, authorized_clients_only = False): self.alive = True self.socket = self.socket_mod self.processor = processor self.server_address = server_address self.HostInterface = HostInterface self.SSL = self.HostInterface.SSL self.real_sock = None self.ssl_authorized_clients_only = authorized_clients_only if self.SSL: self.SocketServer.BaseServer.__init__(self, server_address, RequestHandlerClass) self.load_ssl_context() self.make_ssl_connection_alive() else: try: self.SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass) except self.socket_mod.error, e: if e[0] == 13: raise exceptionTools.ConnectionError('ConnectionError: %s' % (_("Cannot bind the service"),)) raise def load_ssl_context(self): # setup an SSL context. self.context = self.SSL['m'].Context(self.SSL['m'].SSLv23_METHOD) self.context.set_verify(self.SSL['m'].VERIFY_PEER, self.verify_ssl_cb) # ask for a certificate self.context.set_options(self.SSL['m'].OP_NO_SSLv2) # load up certificate stuff. self.context.use_privatekey_file(self.SSL['key']) self.context.use_certificate_file(self.SSL['cert']) self.context.load_verify_locations(self.SSL['ca_cert']) self.context.load_client_ca(self.SSL['ca_cert']) self.HostInterface.updateProgress('SSL context loaded, key: %s - cert: %s, CA cert: %s, CA pkey: %s' % ( self.SSL['key'], self.SSL['cert'], self.SSL['ca_cert'], self.SSL['ca_pkey'] ) ) def make_ssl_connection_alive(self): self.real_sock = self.socket_mod.socket(self.address_family, self.socket_type) self.socket = self.ConnWrapper(self.SSL['m'].Connection(self.context, self.real_sock)) self.server_bind() self.server_activate() # this function should do the authentication checking to see that # the client is who they say they are. def verify_ssl_cb(self, conn, cert, errnum, depth, ok) : return ok def verify_request(self, request, client_address): self.do_ssl = self.HostInterface.SSL if self.do_ssl: self.do_ssl = True else: self.do_ssl = False allowed = self.ip_blacklist_check(client_address[0]) if allowed: allowed = self.ip_max_connections_check(client_address[0]) if not allowed: self.HostInterface.updateProgress( '[from: %s | SSL: %s] connection refused, ip blacklisted or maximum connections per IP reached' % ( client_address, self.do_ssl, ) ) return False allowed = self.max_connections_check(request) if not allowed: self.HostInterface.updateProgress( '[from: %s | SSL: %s] connection refused (max connections reached: %s)' % ( client_address, self.do_ssl, self.HostInterface.max_connections, ) ) return False ### let's go! self.HostInterface.connections += 1 self.HostInterface.updateProgress( '[from: %s | SSL: %s] connection established (%s of %s max connections)' % ( client_address, self.do_ssl, self.HostInterface.connections, self.HostInterface.max_connections, ) ) return True def ip_blacklist_check(self, client_addr): if client_addr in self.HostInterface.ip_blacklist: return False return True def ip_max_connections_check(self, ip_address): max_conn_per_ip = self.HostInterface.max_connections_per_host max_conn_per_ip_barrier = self.HostInterface.max_connections_per_host_barrier per_host_connections = self.HostInterface.per_host_connections conn_data = per_host_connections.get(ip_address) if conn_data == None: per_host_connections[ip_address] = 1 else: conn_data += 1 per_host_connections[ip_address] += 1 if conn_data > max_conn_per_ip: self.HostInterface.updateProgress( '[from: %s] ------- :EEK: !! connection closed too many simultaneous connections from host (current: %s | limit: %s) -------' % ( ip_address, conn_data, max_conn_per_ip, ) ) return False elif conn_data > max_conn_per_ip_barrier: times = [5,6,7,8] self.HostInterface.updateProgress( '[from: %s] ------- :EEEK: !! connection warning simultaneous connection barrier reached from host (current: %s | soft limit: %s) -------' % ( ip_address, conn_data, max_conn_per_ip_barrier, ) ) time.sleep(times[abs(hash(os.urandom(1)))%len(times)]) return True def max_connections_check(self, request): current = self.HostInterface.connections maximum = self.HostInterface.max_connections if current >= maximum: try: self.HostInterface.transmit( request, self.HostInterface.answers['mcr'] ) except: pass return False else: return True def serve_forever(self): while self.alive: #r,w,e = self.select.select([self.socket], [], [], 1) #if r: self.handle_request() # taken from SocketServer.py def finish_request(self, request, client_address): """Finish one request by instantiating RequestHandlerClass.""" self.RequestHandlerClass(request, client_address, self) self.HostInterface.updateProgress( '[from: %s] connection closed (%s of %s max connections)' % ( client_address, self.HostInterface.connections - 1, self.HostInterface.max_connections, ) ) per_host_connections = self.HostInterface.per_host_connections conn_data = per_host_connections.get(client_address[0]) if conn_data != None: if conn_data < 1: del per_host_connections[client_address[0]] else: per_host_connections[client_address[0]] -= 1 def close_request(self, request): if self.HostInterface.connections > 0: self.HostInterface.connections -= 1 class RequestHandler(SocketServer.BaseRequestHandler): import SocketServer import select import socket import entropyTools import gc timed_out = False def __init__(self, request, client_address, server): self.SocketServer.BaseRequestHandler.__init__(self, request, client_address, server) def data_receiver(self): if self.timed_out: return True self.timed_out = True try: ready_to_read, ready_to_write, in_error = self.select.select([self.request], [], [], self.default_timeout) except KeyboardInterrupt: self.timed_out = True return True if len(ready_to_read) == 1 and ready_to_read[0] == self.request: self.timed_out = False try: data = self.request.recv(1024) if self.ssl: while self.request.pending(): data += self.request.recv(1024) if self.data_counter == None: if data == '': # client wants to close return True elif data == self.server.processor.HostInterface.answers['noop']: return False elif len(data) < len(self.myeos): self.server.processor.HostInterface.updateProgress( 'interrupted: %s, reason: %s - from client: %s - data: "%s" - counter: %s' % ( self.server.server_address, "malformed EOS", self.client_address, repr(data), self.data_counter, ) ) self.buffered_data = '' return True mystrlen = data.split(self.myeos)[0] self.data_counter = int(mystrlen) data = data[len(mystrlen)+1:] self.data_counter -= len(data) self.buffered_data += data # command length exceeds our command length limit if self.data_counter > self.max_command_length: raise exceptionTools.InterruptError('InterruptError: command too long: %s, limit: %s' % (self.data_counter,self.max_command_length,)) while self.data_counter > 0: if self.ssl: x = '' while self.request.pending(): x += self.request.recv(1024) else: x = self.request.recv(1024) xlen = len(x) self.data_counter -= xlen self.buffered_data += x if not xlen: break self.data_counter = None except ValueError: #self.entropyTools.printTraceback() self.server.processor.HostInterface.updateProgress( 'interrupted: %s, reason: %s - from client: %s' % ( self.server.server_address, "malformed transmission", self.client_address, ) ) return True except self.socket.timeout, e: self.server.processor.HostInterface.updateProgress( 'interrupted: %s, reason: %s - from client: %s' % ( self.server.server_address, e, self.client_address, ) ) return True except self.socket.sslerror, e: self.server.processor.HostInterface.updateProgress( 'interrupted: %s, SSL socket error reason: %s - from client: %s' % ( self.server.server_address, e, self.client_address, ) ) return True except (self.ssl_exceptions['WantReadError'], self.ssl_exceptions['WantX509LookupError'],): return False except self.ssl_exceptions['ZeroReturnError']: return True except self.ssl_exceptions['Error'], e: self.server.processor.HostInterface.updateProgress( 'interrupted: SSL Error, reason: %s - from client: %s' % ( e, self.client_address, ) ) return True except exceptionTools.InterruptError, e: self.server.processor.HostInterface.updateProgress( 'interrupted: Command Error, reason: %s - from client: %s' % ( e, self.client_address, ) ) return True if not self.buffered_data: return True cmd = self.server.processor.process(self.buffered_data, self.request, self.client_address) if cmd == 'close': # send KAPUTT signal JA! self.server.processor.transmit(self.server.processor.HostInterface.answers['cl']) return True self.buffered_data = '' return False def fork_lock_acquire(self): if hasattr(self.server.processor.HostInterface,'ForkLock'): x = getattr(self.server.processor.HostInterface,'ForkLock') if hasattr(x,'acquire') and hasattr(x,'release') and hasattr(x,'locked'): x.acquire() def fork_lock_release(self): if hasattr(self.server.processor.HostInterface,'ForkLock'): x = getattr(self.server.processor.HostInterface,'ForkLock') if hasattr(x,'acquire') and hasattr(x,'release') and hasattr(x,'locked'): if x.locked(): x.release() def handle(self): # not using spawnFunction because it causes some mess # forking this way avoids having memory leaks if self.server.processor.HostInterface.fork_requests: self.fork_lock_acquire() try: my_timeout = self.server.processor.HostInterface.fork_request_timeout_seconds pid = os.fork() seconds = 0 if pid > 0: # parent here # pid killer after timeout passed_away = False while 1: time.sleep(1) seconds += 1 try: dead = os.waitpid(pid, os.WNOHANG)[0] except OSError, e: if e.errno != 10: raise dead = True if passed_away: break if dead: break if seconds > my_timeout: self.server.processor.HostInterface.updateProgress( 'interrupted: forked request timeout: %s,%s from client: %s' % ( seconds, dead, self.client_address, ) ) if not dead: import signal os.kill(pid,signal.SIGKILL) passed_away = True # in this way, the process table should be clean continue break else: self.do_handle() os._exit(0) finally: self.fork_lock_release() else: self.do_handle() #self.entropyTools.spawnFunction(self.do_handle) def do_handle(self): self.default_timeout = self.server.processor.HostInterface.timeout self.ssl = self.server.processor.HostInterface.SSL self.ssl_exceptions = self.server.processor.HostInterface.SSL_exceptions self.myeos = self.server.processor.HostInterface.answers['eos'] self.max_command_length = self.server.processor.HostInterface.max_command_length while 1: try: dobreak = self.data_receiver() if dobreak: break except Exception, e: self.server.processor.HostInterface.updateProgress( 'interrupted: Unhandled exception: %s, error: %s - from client: %s' % ( Exception, e, self.client_address, ) ) # print exception tb = self.entropyTools.getTraceback() print tb self.server.processor.HostInterface.socketLog.write(tb) break self.request.close() def setup(self): self.data_counter = None self.buffered_data = '' class CommandProcessor: import entropyTools import socket import gc def __init__(self, HostInterface): self.HostInterface = HostInterface self.channel = None self.lastoutput = None def handle_termination_commands(self, data): if data.strip() in self.HostInterface.termination_commands: self.HostInterface.updateProgress('close: %s' % (self.client_address,)) self.transmit(self.HostInterface.answers['cl']) return "close" if not data.strip(): return "ignore" def handle_command_string(self, string): # validate command args = string.strip().split() session = args[0] if (session in self.HostInterface.initialization_commands) or \ (session in self.HostInterface.no_session_commands) or \ len(args) < 2: cmd = args[0] session = None else: cmd = args[1] args = args[1:] # remove session stream_enabled = False if (session != None) and self.HostInterface.sessions.has_key(session): stream_enabled = self.HostInterface.sessions[session].get('stream_mode') if stream_enabled and (cmd not in self.HostInterface.config_commands): session_len = 0 if session: session_len = len(session)+1 return cmd,[string[session_len+len(cmd)+1:]],session else: myargs = [] if len(args) > 1: myargs = args[1:] return cmd,myargs,session def handle_end_answer(self, cmd, whoops, valid_cmd): if not valid_cmd: self.transmit(self.HostInterface.answers['no']) elif whoops: self.transmit(self.HostInterface.answers['er']) elif cmd not in self.HostInterface.no_acked_commands: self.transmit(self.HostInterface.answers['ok']) def validate_command(self, cmd, args, session): # answer to invalid commands if (cmd not in self.HostInterface.valid_commands): return False,"not a valid command" if session == None: if cmd not in self.HostInterface.no_session_commands: return False,"need a valid session" elif session not in self.HostInterface.sessions: return False,"session is not alive" # check if command needs authentication if session != None: auth = self.HostInterface.valid_commands[cmd]['auth'] if auth: # are we? authed = self.HostInterface.sessions[session]['auth_uid'] if authed == None: # nope return False,"not authenticated" # keep session alive if session != None: self.HostInterface.set_session_running(session) self.HostInterface.update_session_time(session) return True,"all good" def load_authenticator(self): f, args, kwargs = self.HostInterface.AuthenticatorInst myinst = f(*args,**kwargs) return myinst def load_service_interface(self, session): uid = None if session != None: uid = self.HostInterface.sessions[session]['auth_uid'] intf = self.HostInterface.EntropyInstantiation[0] args = self.HostInterface.EntropyInstantiation[1] kwds = self.HostInterface.EntropyInstantiation[2] Entropy = intf(*args, **kwds) Entropy.urlFetcher = SocketUrlFetcher Entropy.updateProgress = self.remoteUpdateProgress try: Entropy.clientDbconn.updateProgress = self.remoteUpdateProgress except AttributeError: pass Entropy.progress = self.remoteUpdateProgress return Entropy def process(self, data, channel, client_address): self.channel = channel self.client_address = client_address term = self.handle_termination_commands(data) if term: del authenticator return term cmd, args, session = self.handle_command_string(data) valid_cmd, reason = self.validate_command(cmd, args, session) # decide if we need to load authenticator or Entropy authenticator = None cmd_data = self.HostInterface.valid_commands.get(cmd) if not isinstance(cmd_data,dict): self.HostInterface.updateProgress( '[from: %s] command error: invalid command: %s' % ( self.client_address, cmd, ) ) return "close" elif (("authenticator" in cmd_data['args']) or (cmd in self.HostInterface.login_pass_commands)): try: authenticator = self.load_authenticator() except exceptionTools.ConnectionError, e: self.HostInterface.updateProgress( '[from: %s] authenticator error: cannot load: %s' % ( self.client_address, e, ) ) tb = self.entropyTools.getTraceback() print tb self.server.processor.HostInterface.socketLog.write(tb) return "close" except Exception, e: self.HostInterface.updateProgress( '[from: %s] authenticator error: cannot load: %s - unknown error' % ( self.client_address, e, ) ) tb = self.entropyTools.getTraceback() print tb self.server.processor.HostInterface.socketLog.write(tb) return "close" p_args = args if (cmd in self.HostInterface.login_pass_commands) and authenticator != None: p_args = authenticator.hide_login_data(p_args) elif cmd in self.HostInterface.raw_commands: p_args = ['raw data'] self.HostInterface.updateProgress( '[from: %s] command validation :: called %s: length: %s, args: %s, session: %s, valid: %s, reason: %s' % ( self.client_address, cmd, len(data), p_args, session, valid_cmd, reason, ) ) whoops = False if valid_cmd: if authenticator != None: # now set session authenticator.set_session(session) Entropy = None if "Entropy" in cmd_data['args']: Entropy = self.load_service_interface(session) try: self.run_task(cmd, args, session, Entropy, authenticator) except self.socket.timeout: self.HostInterface.updateProgress( '[from: %s] command error: timeout, closing connection' % ( self.client_address, ) ) # close connection del authenticator del Entropy return "close" except self.socket.error, e: self.HostInterface.updateProgress( '[from: %s] command error: socket error: %s' % ( self.client_address, e, ) ) # close connection del authenticator del Entropy return "close" except self.HostInterface.SSL_exceptions['SysCallError'], e: self.HostInterface.updateProgress( '[from: %s] command error: SSL SysCallError: %s' % ( self.client_address, e, ) ) # close connection del authenticator del Entropy return "close" except Exception, e: # write to self.HostInterface.socketLog tb = self.entropyTools.getTraceback() print tb self.HostInterface.socketLog.write(tb) # store error self.HostInterface.updateProgress( '[from: %s] command error: %s, type: %s' % ( self.client_address, e, type(e), ) ) if session != None: self.HostInterface.store_rc(str(e),session) whoops = True del Entropy if session != None: self.HostInterface.update_session_time(session) self.HostInterface.unset_session_running(session) rcmd = None try: self.handle_end_answer(cmd, whoops, valid_cmd) except (self.socket.error, self.socket.timeout,self.HostInterface.SSL_exceptions['SysCallError'],): rcmd = "close" if authenticator != None: authenticator.terminate_instance() del authenticator if not self.HostInterface.fork_requests: self.gc.collect() return rcmd def transmit(self, data): self.HostInterface.transmit(self.channel, data) def remoteUpdateProgress( self, text, header = "", footer = "", back = False, importance = 0, type = "info", count = [], percent = False ): if text != self.lastoutput: text = chr(27)+"[2K\r"+text if not back: text += "\n" self.transmit(text) self.lastoutput = text def run_task(self, cmd, args, session, Entropy, authenticator): p_args = args if cmd in self.HostInterface.login_pass_commands: p_args = authenticator.hide_login_data(p_args) elif cmd in self.HostInterface.raw_commands: p_args = ['raw data'] self.HostInterface.updateProgress( '[from: %s] run_task :: called %s: args: %s, session: %s' % ( self.client_address, cmd, p_args, session, ) ) myargs = args mykwargs = {} if cmd not in self.HostInterface.raw_commands: myargs, mykwargs = self._get_args_kwargs(args) rc = self.spawn_function(cmd, myargs, mykwargs, session, Entropy, authenticator) if session != None and self.HostInterface.sessions.has_key(session): self.HostInterface.store_rc(rc, session) return rc def _get_args_kwargs(self, args): myargs = [] mykwargs = {} def is_int(x): try: int(x) except ValueError: return False return True for arg in args: if (arg.find("=") != -1) and not arg.startswith("="): x = arg.split("=") a = x[0] b = ''.join(x[1:]) if (b in ("True","False",)) or is_int(b): mykwargs[a] = eval(b) else: myargs.append(arg) else: if (arg in ("True","False",)) or is_int(arg): myargs.append(eval(arg)) else: myargs.append(arg) return myargs, mykwargs def spawn_function(self, cmd, myargs, mykwargs, session, Entropy, authenticator): p_args = myargs if cmd in self.HostInterface.login_pass_commands: p_args = authenticator.hide_login_data(p_args) elif cmd in self.HostInterface.raw_commands: p_args = ['raw data'] self.HostInterface.updateProgress( '[from: %s] called %s: args: %s, kwargs: %s' % ( self.client_address, cmd, p_args, mykwargs, ) ) return self.do_spawn(cmd, myargs, mykwargs, session, Entropy, authenticator) def do_spawn(self, cmd, myargs, mykwargs, session, Entropy, authenticator): cmd_data = self.HostInterface.valid_commands.get(cmd) do_fork = cmd_data['as_user'] f = cmd_data['cb'] func_args = [] for arg in cmd_data['args']: try: func_args.append(eval(arg)) except (NameError, SyntaxError): func_args.append(str(arg)) if do_fork: myfargs = func_args[:] myfargs.extend(myargs) return self.fork_task(f, session, authenticator, *myfargs, **mykwargs) else: return f(*func_args) def fork_task(self, f, session, authenticator, *args, **kwargs): gid = None uid = None if session != None: logged_in = self.HostInterface.sessions[session]['auth_uid'] if logged_in != None: uid = logged_in gid = etpConst['entropygid'] return self.entropyTools.spawnFunction(self._do_fork, f, authenticator, uid, gid, *args, **kwargs) def _do_fork(self, f, authenticator, uid, gid, *args, **kwargs): authenticator.set_exc_permissions(uid,gid) rc = f(*args,**kwargs) return rc class BuiltInCommands(SocketCommandsSkel): import dumpTools import zlib def __init__(self, HostInterface): SocketCommandsSkel.__init__(self, HostInterface, inst_name = "builtin") self.valid_commands = { 'begin': { 'auth': False, # does it need authentication ? 'built_in': True, # is it built-in ? 'cb': self.docmd_begin, # function to call 'args': ["self.transmit", "self.client_address"], # arguments to be passed before *args and **kwards, in SocketHostInterface.do_spawn() 'as_user': False, # do I have to fork the process and run it as logged user? # needs auth = True 'desc': "instantiate a session", # description 'syntax': "begin", # syntax 'from': unicode(self), # from what class }, 'end': { 'auth': False, 'built_in': True, 'cb': self.docmd_end, 'args': ["self.transmit", "session"], 'as_user': False, 'desc': "end a session", 'syntax': " end", 'from': unicode(self), }, 'session_config': { 'auth': False, 'built_in': True, 'cb': self.docmd_session_config, 'args': ["session","myargs"], 'as_user': False, 'desc': "set session configuration options", 'syntax': " session_config