From efe01088e926f4a173e02395a1ff928ecbdbdd40 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Sun, 23 Mar 2008 11:33:32 +0000 Subject: [PATCH] Entropy: - goodbye portageTools, welcome SpmInterface git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1496 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 - client/text_repositories.py | 4 + client/text_rescue.py | 38 +- client/text_smart.py | 4 +- libraries/databaseTools.py | 23 +- libraries/entropy.py | 740 ++++++++++++++++++++++++++-- libraries/entropyConstants.py | 9 +- libraries/entropyTools.py | 113 +++-- libraries/portageTools.py | 898 ---------------------------------- libraries/reagentTools.py | 20 +- 10 files changed, 799 insertions(+), 1051 deletions(-) delete mode 100644 libraries/portageTools.py diff --git a/TODO b/TODO index cefb4033c..d6d95c6db 100644 --- a/TODO +++ b/TODO @@ -4,7 +4,6 @@ TODO list - split RDEPEND and PDEPEND ( + && and || ) - log messages from portage doebuild() calls - remove conflicting packages right before the conflict (*) - - move portageTools to a class (*) - implement configuration files snapshot tool - migrate server code to ServerInterface [] write a tool that helps keeping packages updated (also supporting injected ones) diff --git a/client/text_repositories.py b/client/text_repositories.py index 586b9ab23..bfff98c76 100644 --- a/client/text_repositories.py +++ b/client/text_repositories.py @@ -45,6 +45,10 @@ def repositories(options): repo_names.append(opt) if (options[0] == "update"): + # check if I am root + if not Equo.entropyTools.isRoot(): + print_error(red("You must be ")+bold("root")+red(".")) + return 1 rc = do_sync(reponames = repo_names, forceUpdate = equoRequestForceUpdate) elif (options[0] == "status"): for repo in etpRepositories: diff --git a/client/text_rescue.py b/client/text_rescue.py index 88c1d8811..dccb4c6db 100644 --- a/client/text_rescue.py +++ b/client/text_rescue.py @@ -30,6 +30,15 @@ from outputTools import * from entropy import EquoInterface Equo = EquoInterface(noclientdb = True) +def test_spm(): + # test if portage is available + try: + Spm = Equo.Spm() + return Spm + except: + print_error(darkred(" * ")+bold("Source Package Manager backend")+red(" is not available.")) + return None + def database(options): if len(options) < 1: @@ -42,14 +51,10 @@ def database(options): if (options[0] == "generate"): - # test if portage is available - try: - import portageTools - except: - print_error(darkred(" * ")+bold("Portage")+red(" is not available.")) + Spm = test_spm() + if Spm == None: return 1 - print_warning(bold("ATTENTION: ")+red("The installed package database will be generated again using Gentoo one.")) print_warning(red("If you dont know what you're doing just, don't do this. Really. I'm not joking.")) rc = Equo.askQuestion(" Understood?") @@ -93,7 +98,7 @@ def database(options): # now collect packages in the system print_info(red(" Transductingactioningintactering databases...")) - portagePackages = portageTools.getInstalledPackages() + portagePackages = Spm.get_installed_packages() portagePackages = portagePackages[0] # do for each database @@ -295,23 +300,19 @@ def database(options): elif (options[0] == "counters"): - try: - import portageTools - except: - print_error(darkred(" * ")+bold("Portage")+red(" is not available.")) + Spm = test_spm() + if Spm == None: return 1 print_info(red(" Regenerating counters table. Please wait...")) - Equo.clientDbconn.regenerateCountersTable(output = True) + Equo.clientDbconn.regenerateCountersTable(Spm.get_vdb_path(), output = True) print_info(red(" Counters table regenerated. Check above for errors.")) return 0 elif (options[0] == "gentoosync"): - try: - import portageTools - except: - print_error(darkred(" * ")+bold("Portage")+red(" is not available.")) + Spm = test_spm() + if Spm == None: return 1 print_info(red(" Scanning Portage and Entropy databases for differences...")) @@ -328,9 +329,8 @@ def database(options): return 1 import shutil - from portageTools import getInstalledPackagesCounters, getPackageSlot print_info(red(" Collecting Portage counters..."), back = True) - installedPackages = getInstalledPackagesCounters() + installedPackages = Spm.get_installed_packages_counter() print_info(red(" Collecting Entropy packages..."), back = True) installedCounters = set() toBeAdded = set() @@ -356,7 +356,7 @@ def database(options): atomslot = Equo.clientDbconn.retrieveSlot(x[1]) add = True for pkgdata in toBeAdded: - addslot = getPackageSlot(pkgdata[0]) + addslot = Spm.get_package_slot(pkgdata[0]) addkey = Equo.entropyTools.dep_getkey(pkgdata[0]) # workaround for ebuilds not having slot if addslot == None: diff --git a/client/text_smart.py b/client/text_smart.py index a8664bb91..eec2fe8c3 100644 --- a/client/text_smart.py +++ b/client/text_smart.py @@ -124,9 +124,9 @@ def CommonFlate(mytbz2s, action, savedir = None): # test if portage is available try: - import portageTools + Spm = Equo.Spm() except: - print_error(darkred(" * ")+bold("Portage")+red(" is not available.")) + print_error(darkred(" * ")+bold("Source Package Manager backend")+red(" is not available.")) return 1 if savedir: diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 3c6a435f3..68688a1e6 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -347,7 +347,6 @@ class etpDatabase: def serverUpdatePackagesData(self): etpConst['treeupdatescalled'] = True - repository = etpConst['officialrepositoryid'] doRescan = False @@ -367,15 +366,16 @@ class etpDatabase: if repositoryUpdatesDigestCache_disk.has_key(repository): portage_dirs_digest = repositoryUpdatesDigestCache_disk.get(repository) else: - import portageTools + from entropy import SpmInterface + SpmIntf = SpmInterface(self.OutputInterface) + Spm = SpmIntf.intf # grab portdir - updates_dir = etpConst['systemroot']+portageTools.getPortageEnv("PORTDIR")+"/profiles/updates" + updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates" if os.path.isdir(updates_dir): # get checksum portage_dirs_digest = entropyTools.md5sum_directory(updates_dir) repositoryUpdatesDigestCache_disk[repository] = portage_dirs_digest del updates_dir - del portageTools if doRescan or (str(stored_digest) != str(portage_dirs_digest)): @@ -386,8 +386,10 @@ class etpDatabase: # reset database tables self.clearTreeupdatesEntries(repository) - import portageTools - updates_dir = etpConst['systemroot']+portageTools.getPortageEnv("PORTDIR")+"/profiles/updates" + from entropy import SpmInterface + SpmIntf = SpmInterface(self.OutputInterface) + Spm = SpmIntf.intf + updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates" update_files = entropyTools.sortUpdateFiles(os.listdir(updates_dir)) update_files = [os.path.join(updates_dir,x) for x in update_files] # now load actions from files @@ -3468,22 +3470,17 @@ class etpDatabase: self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( idpackage, description, homepage, download, digest, datecreation, size )') self.commitChanges() - def regenerateCountersTable(self, output = False): + def regenerateCountersTable(self, vdb_path, output = False): self.checkReadOnly() self.createCountersTable() # assign a counter to an idpackage - try: - from portageTools import getPortageAppDbPath # only if Portage is found - except: - return - appdbpath = getPortageAppDbPath() myids = self.listAllIdpackages() for myid in myids: # get atom myatom = self.retrieveAtom(myid) mybranch = self.retrieveBranch(myid) myatom = entropyTools.remove_tag(myatom) - myatomcounterpath = appdbpath+myatom+"/"+dbCOUNTER + myatomcounterpath = vdb_path+myatom+"/"+dbCOUNTER if os.path.isfile(myatomcounterpath): try: f = open(myatomcounterpath,"r") diff --git a/libraries/entropy.py b/libraries/entropy.py index ad7947878..50adf2487 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -87,6 +87,7 @@ class EquoInterface(TextInterface): self.FileUpdates = self.FileUpdatesInterfaceLoader() self.repoDbCache = {} self.securityCache = {} + self.spmCache = {} # masking parser self.MaskingParser = self.PackageMaskingParserInterfaceLoader() @@ -2218,6 +2219,21 @@ class EquoInterface(TextInterface): Package interface :: end ''' + ''' + Source Package Manager Interface :: begin + ''' + def Spm(self): + myroot = etpConst['systemroot'] + cached = self.spmCache.get(myroot) + if cached != None: + return cached + conn = SpmInterface(self) + self.spmCache[myroot] = conn.intf + return conn.intf + ''' + Source Package Manager Interface :: end + ''' + ''' Triggers interface :: begin ''' @@ -2601,37 +2617,34 @@ class PackageInterface: def __remove_package_from_gentoo_database(self, atom): # handle gentoo-compat - _portage_avail = False try: - from portageTools import getPortageAppDbPath as _portage_getPortageAppDbPath, getInstalledAtoms as _portage_getInstalledAtoms - _portage_avail = True + Spm = self.Entropy.Spm() except: - return -1 # no Portage support + return -1 # no Spm support ?? - if (_portage_avail): - portDbDir = _portage_getPortageAppDbPath() - removePath = portDbDir+atom - try: - shutil.rmtree(removePath,True) - except: - pass - key = self.Entropy.entropyTools.dep_getkey(atom) - othersInstalled = _portage_getInstalledAtoms(key) #FIXME: really slow - if othersInstalled == None: - world_file = os.path.join(etpConst['systemroot'],'var/lib/portage/world') - world_file_tmp = world_file+".entropy.tmp" - if os.access(world_file,os.W_OK): - new = open(world_file_tmp,"w") - old = open(world_file,"r") + portDbDir = Spm.get_vdb_path() + removePath = portDbDir+atom + try: + shutil.rmtree(removePath,True) + except: + pass + key = self.Entropy.entropyTools.dep_getkey(atom) + othersInstalled = Spm.get_installed_atoms(key) #FIXME: really slow + if othersInstalled == None: + world_file = os.path.join(etpConst['systemroot'],'var/lib/portage/world') + world_file_tmp = world_file+".entropy.tmp" + if os.access(world_file,os.W_OK): + new = open(world_file_tmp,"w") + old = open(world_file,"r") + line = old.readline() + while line: + if line.find(key) == -1: + new.write(line) line = old.readline() - while line: - if line.find(key) == -1: - new.write(line) - line = old.readline() - new.flush() - new.close() - old.close() - shutil.move(world_file_tmp,world_file) + new.flush() + new.close() + old.close() + shutil.move(world_file_tmp,world_file) return 0 ''' @@ -2758,15 +2771,12 @@ class PackageInterface: def _install_package_into_gentoo_database(self, newidpackage): # handle gentoo-compat - _portage_avail = False - portDbDir = '' try: - import portageTools - portDbDir = portageTools.getPortageAppDbPath() - _portage_avail = True + Spm = self.Entropy.Spm() except: return -1 # no Portage support - if _portage_avail and os.path.isdir(portDbDir): + portDbDir = Spm.get_vdb_path() + if os.path.isdir(portDbDir): # extract xpak from unpackDir+etpConst['packagecontentdir']+"/"+package key = self.infoDict['category']+"/"+self.infoDict['name'] atomsfound = set() @@ -2781,7 +2791,7 @@ class PackageInterface: if atomsfound: pkgToRemove = '' for atom in atomsfound: - atomslot = portageTools.getPackageSlot(atom) + atomslot = Spm.get_package_slot(atom) # get slot from gentoo db if atomslot == self.infoDict['slot']: pkgToRemove = atom @@ -2823,9 +2833,9 @@ class PackageInterface: f.close() except: # need file recreation, parse gentoo tree - counter = portageTools.refillCounter() + counter = Spm.refill_counter() else: - counter = portageTools.refillCounter() + counter = Spm.refill_counter() # write new counter to file if os.path.isdir(destination): @@ -4471,7 +4481,7 @@ class FtpInterface: def __init__(self, ftpuri, EntropyInterface): if not isinstance(EntropyInterface, (EquoInterface, TextInterface)): - raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based Instance is needed") + raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based instance is needed") self.Entropy = EntropyInterface import entropyTools @@ -5222,11 +5232,10 @@ class TriggerInterface: self.INITSERVICES_DIR="/var/lib/init.d/" ''' portage stuff ''' - self.portageTools = None if self.gentoo_compat: try: - import portageTools - self.portageTools = portageTools + Spm = self.Entropy.Spm() + self.Spm = Spm except Exception, e: self.Entropy.updateProgress( red("Portage interface can't be loaded due to %s: (%s), please fix it !" % (str(Exception),str(e)) ), @@ -6108,14 +6117,14 @@ class TriggerInterface: if not os.path.isfile(self.pkgdata['unpackdir']+"/portage/"+portage_atom+"/temp/environment"): # if environment is not yet created, we need to run pkg_setup() sys.stdout = stdfile - rc = self.portageTools.portage_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) + rc = self.Spm.spm_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup()) trigger for "+str(portage_atom)+". Something bad happened.") sys.stdout = oldstdout - rc = self.portageTools.portage_doebuild(myebuild, mydo = "postinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) + rc = self.Spm.spm_doebuild(myebuild, mydo = "postinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_postinst()) trigger for "+str(portage_atom)+". Something bad happened.") - except Exception, e: # let it crash even if self.portageTools == None + except Exception, e: sys.stdout = oldstdout self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot run Gentoo postinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e)) self.Entropy.updateProgress( @@ -6145,11 +6154,11 @@ class TriggerInterface: ) try: sys.stdout = stdfile - rc = self.portageTools.portage_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) # create mysettings["T"]+"/environment" + rc = self.Spm.spm_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) # create mysettings["T"]+"/environment" if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_setup()) trigger for "+str(portage_atom)+". Something bad happened.") sys.stdout = oldstdout - rc = self.portageTools.portage_doebuild(myebuild, mydo = "preinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) + rc = self.Spm.spm_doebuild(myebuild, mydo = "preinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'], licenses = self.pkgdata['accept_license']) if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_preinst()) trigger for "+str(portage_atom)+". Something bad happened.") except Exception, e: @@ -6172,7 +6181,7 @@ class TriggerInterface: portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] try: - myebuild = self.portageTools.getPortageAppDbPath()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild" + myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild" except: myebuild = '' @@ -6188,7 +6197,7 @@ class TriggerInterface: header = red(" ##") ) try: - rc = self.portageTools.portage_doebuild(myebuild, mydo = "prerm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom) + rc = self.Spm.spm_doebuild(myebuild, mydo = "prerm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom) if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preremove trigger for "+str(portage_atom)+". Something bad happened.") except Exception, e: @@ -6213,7 +6222,7 @@ class TriggerInterface: portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'] try: - myebuild = self.portageTools.getPortageAppDbPath()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild" + myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild" except: myebuild = '' @@ -6228,7 +6237,7 @@ class TriggerInterface: header = red(" ##") ) try: - rc = self.portageTools.portage_doebuild(myebuild, mydo = "postrm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom) + rc = self.Spm.spm_doebuild(myebuild, mydo = "postrm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom) if rc == 1: self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo postremove trigger for "+str(portage_atom)+". Something bad happened.") except Exception, e: @@ -7704,3 +7713,636 @@ class SecurityInterface: ) return 0 + +class SpmInterface: + + def __init__(self, OutputInterface): + if not isinstance(OutputInterface, (EquoInterface, TextInterface)): + if OutputInterface == None: + OutputInterface = TextInterface() + else: + raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based instance is needed") + + self.spm_backend = etpConst['spm']['backend'] + self.valid_backends = etpConst['spm']['available_backends'] + if self.spm_backend not in self.valid_backends: + raise exceptionTools.IncorrectParameter("IncorrectParameter: invalid backend %s" % (self.spm_backend,)) + + if self.spm_backend == "portage": + self.intf = PortageInterface(OutputInterface) + + +class PortageInterface: + + def __init__(self, OutputInterface): + if not isinstance(OutputInterface, (EquoInterface, TextInterface)): + raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based instance is needed") + + # interface only needed OutputInterface functions + self.updateProgress = OutputInterface.updateProgress + self.askQuestion = OutputInterface.askQuestion + + # importing portage stuff + import portage + self.portage = portage + try: + import portage_const + except ImportError: + import portage.const as portage_const + self.portage_const = portage_const + + import entropyTools + self.entropyTools = entropyTools + + def get_third_party_mirrors(self, mirrorname): + try: + return self.portage.thirdpartymirrors[mirrorname] + except KeyError: + return [] + + def get_spm_setting(self, var): + return self.portage.settings[var] + + # Packages in system (in the Portage language -> emerge system, remember?) + def get_atoms_in_system(self): + system = self.portage.settings.packages + sysoutput = [] + for x in system: + y = self.get_installed_atoms(x) + if (y != None): + for z in y: + sysoutput.append(z) + sysoutput.append("sys-kernel/linux-sabayon") # our kernel + sysoutput.append("dev-db/sqlite") # our interface + sysoutput.append("dev-python/pysqlite") # our python interface to our interface (lol) + sysoutput.append("virtual/cron") # our cron service + sysoutput.append("app-admin/equo") # our package manager (client) + sysoutput.append("sys-apps/entropy") # our package manager (server+client) + return sysoutput + + def get_config_protect_and_mask(self): + config_protect = self.portage.settings['CONFIG_PROTECT'] + config_protect = config_protect.split() + config_protect_mask = self.portage.settings['CONFIG_PROTECT_MASK'] + config_protect_mask = config_protect_mask.split() + # explode + protect = [] + for x in config_protect: + if x.startswith("$"): #FIXME: small hack + x = commands.getoutput("echo "+x).split("\n")[0] + protect.append(x) + mask = [] + for x in config_protect_mask: + if x.startswith("$"): #FIXME: small hack + x = commands.getoutput("echo "+x).split("\n")[0] + mask.append(x) + return ' '.join(protect),' '.join(mask) + + # resolve atoms automagically (best, not current!) + # sys-libs/application --> sys-libs/application-1.2.3-r1 + def get_best_atom(self, atom, match = "bestmatch-visible"): + try: + rc = self.portage.portdb.xmatch(match,str(atom)) + return rc + except ValueError: + return "!!conflicts" + + # same as above but includes masked ebuilds + def get_best_masked_atom(self, atom): + atoms = self.portage.portdb.xmatch("match-all",str(atom)) + # find the best + try: + from portage_versions import best + except ImportError: + from portage.versions import best + rc = best(atoms) + return rc + + def get_atom_category(self, atom): + try: + rc = self.portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0] + return rc + except: + return None + + def _get_portage_vartree(self, root): + + if not etpConst['spm']['cache'].has_key('portage'): + etpConst['spm']['cache']['portage'] = {} + if not etpConst['spm']['cache']['portage'].has_key('vartree'): + etpConst['spm']['cache']['portage']['vartree'] = {} + + cached = etpConst['spm']['cache']['portage']['vartree'].get(root) + if cached != None: + return cached + + mytree = self.portage.vartree(root=root) + etpConst['spm']['cache']['portage']['vartree'][root] = mytree + return mytree + + def _get_portage_config(self, config_root, root): + + if not etpConst['spm']['cache'].has_key('portage'): + etpConst['spm']['cache']['portage'] = {} + if not etpConst['spm']['cache']['portage'].has_key('config'): + etpConst['spm']['cache']['portage']['config'] = {} + + cached = etpConst['spm']['cache']['portage']['config'].get((config_root,root)) + if cached != None: + return cached + + mysettings = self.portage.config(config_root = config_root, target_root = root, config_incrementals = self.portage_const.INCREMENTALS) + etpConst['spm']['cache']['portage']['config'][(config_root,root)] = mysettings + return mysettings + + # please always force =pkgcat/pkgname-ver if possible + def get_installed_atom(self, atom): + mypath = etpConst['systemroot']+"/" + mytree = self._get_portage_vartree(mypath) + rc = mytree.dep_match(str(atom)) + if rc: + return rc[-1] + return None + + def get_package_slot(self, atom): + mypath = etpConst['systemroot']+"/" + mytree = self._get_portage_vartree(mypath) + if atom.startswith("="): + atom = atom[1:] + rc = mytree.getslot(atom) + if rc: + return rc + return None + + def get_installed_atoms(self, atom): + mypath = etpConst['systemroot']+"/" + mytree = self._get_portage_vartree(mypath) + rc = mytree.dep_match(str(atom)) + if rc: + return rc + return None + + # create a .tbz2 file in the specified path + def quickpkg(self, atom, dirpath): + + # getting package info + pkgname = atom.split("/")[1] + pkgcat = atom.split("/")[0] + #pkgfile = pkgname+".tbz2" + if not os.path.isdir(dirpath): + os.makedirs(dirpath) + dirpath += "/"+pkgname+".tbz2" + dbdir = self.get_vdb_path()+"/"+pkgcat+"/"+pkgname+"/" + + import tarfile + import stat + trees = self.portage.db["/"] + vartree = trees["vartree"] + dblnk = self.portage.dblink(pkgcat, pkgname, "/", vartree.settings, treetype="vartree", vartree=vartree) + dblnk.lockdb() + tar = tarfile.open(dirpath,"w:bz2") + + contents = dblnk.getcontents() + id_strings = {} + paths = contents.keys() + paths.sort() + + for path in paths: + try: + exist = os.lstat(path) + except OSError: + continue # skip file + ftype = contents[path][0] + lpath = path + arcname = path[1:] + if 'dir' == ftype and \ + not stat.S_ISDIR(exist.st_mode) and \ + os.path.isdir(lpath): + lpath = os.path.realpath(lpath) + tarinfo = tar.gettarinfo(lpath, arcname) + tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) + tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) + + if stat.S_ISREG(exist.st_mode): + tarinfo.type = tarfile.REGTYPE + f = open(path) + try: + tar.addfile(tarinfo, f) + finally: + f.close() + else: + tar.addfile(tarinfo) + + tar.close() + + # appending xpak informations + import etpXpak + tbz2 = etpXpak.tbz2(dirpath) + tbz2.recompose(dbdir) + + dblnk.unlockdb() + + if os.path.isfile(dirpath): + return dirpath + else: + return False + + def get_useflags(self): + return self.portage.settings['USE'] + + def get_useflags_force(self): + return self.portage.settings.useforce + + def get_useflags_mask(self): + return self.portage.settings.usemask + + def get_package_setting(self, atom, setting): + myatom = atom[:] + if myatom.startswith("="): + myatom = myatom[1:] + return self.portage.portdb.aux_get(myatom,[setting])[0] + + def calculate_dependencies(self, my_iuse, my_use, my_license, my_depend, my_rdepend, my_pdepend, my_provide, my_src_uri): + metadata = {} + metadata['USE'] = my_use + metadata['IUSE'] = my_iuse + metadata['LICENSE'] = my_license + metadata['DEPEND'] = my_depend + metadata['PDEPEND'] = my_pdepend + metadata['RDEPEND'] = my_rdepend + metadata['PROVIDE'] = my_provide + metadata['SRC_URI'] = my_src_uri + use = metadata['USE'].split() + raw_use = use + iuse = set(metadata['IUSE'].split()) + use = [f for f in use if f in iuse] + use.sort() + metadata['USE'] = " ".join(use) + try: + from portage_dep import paren_reduce, use_reduce #, paren_enclose + p_normalize = self.entropyTools.paren_normalize + except ImportError: + from portage.dep import paren_reduce, use_reduce, paren_normalize as p_normalize #, paren_enclose + for k in "LICENSE", "RDEPEND", "DEPEND", "PDEPEND", "PROVIDE", "SRC_URI": + try: + deps = paren_reduce(metadata[k]) + deps = use_reduce(deps, uselist=raw_use) + deps = p_normalize(deps) + if k == "LICENSE": + deps = self.paren_license_choose(deps) + else: + deps = self.paren_choose(deps) + deps = ' '.join(deps) + except Exception, e: + self.updateProgress( + darkred("Error calculating Portage dependencies: %s: %s :: %s") % (str(Exception), k, str(e)) , + importance = 1, + type = "error", + header = red(" !!! ") + ) + deps = '' + continue + metadata[k] = deps + return metadata + + def paren_choose(self, dep_list): + + newlist = [] + do_skip = False + for idx in range(len(dep_list)): + + if do_skip: + do_skip = False + continue + + item = dep_list[idx] + if item == "||": # or + next_item = dep_list[idx+1] + if not next_item: # || ( asd? ( atom ) dsa? ( atom ) ) => [] if use asd and dsa are disabled + do_skip = True + continue + item = self.dep_or_select(next_item) # must be a list + if not item: + # no matches, transform to string and append, so reagent will fail + newlist.append(str(next_item)) + else: + newlist += item + do_skip = True + elif isinstance(item, list): # and + item = self.dep_and_select(item) + newlist += item + else: + newlist.append(item) + + return newlist + + def dep_and_select(self, and_list): + do_skip = False + newlist = [] + for idx in range(len(and_list)): + + if do_skip: + do_skip = False + continue + + x = and_list[idx] + if x == "||": + x = self.dep_or_select(and_list[idx+1]) + do_skip = True + if not x: + x = str(and_list[idx+1]) + else: + newlist += x + elif isinstance(x, list): + x = self.dep_and_select(x) + newlist += x + else: + newlist.append(x) + + # now verify if all are satisfied + for x in newlist: + match = self.get_installed_atom(x) + if match == None: + return [] + + return newlist + + def dep_or_select(self, or_list): + do_skip = False + for idx in range(len(or_list)): + if do_skip: + do_skip = False + continue + x = or_list[idx] + if x == "||": # or + x = self.dep_or_select(or_list[idx+1]) + do_skip = True + elif isinstance(x, list): # and + x = self.dep_and_select(x) + if not x: + continue + # found + return x + else: + x = [x] + + for y in x: + match = self.get_installed_atom(y) + if match != None: + return [y] + + return [] + + def paren_license_choose(self, dep_list): + + newlist = set() + for item in dep_list: + + if isinstance(item, list): + # match the first + data = set(self.paren_license_choose(item)) + newlist.update(data) + else: + if item not in ["||"]: + newlist.add(item) + + return list(newlist) + + def get_vdb_path(self): + rc = etpConst['systemroot']+"/"+self.portage_const.VDB_PATH + if (not rc.endswith("/")): + return rc+"/" + return rc + + def get_available_packages(self, categories = [], filter_reinstalls = True): + mypath = etpConst['systemroot']+"/" + mysettings = self._get_portage_config("/",mypath) + portdb = self.portage.portdbapi(mysettings["PORTDIR"], mysettings = mysettings) + cps = portdb.cp_all() + visibles = set() + for cp in cps: + if categories and cp.split("/")[0] not in categories: + continue + # get slots + slots = set() + atoms = self.get_best_atom(cp, "match-visible") + for atom in atoms: + slots.add(portdb.aux_get(atom, ["SLOT"])[0]) + for slot in slots: + visibles.add(cp+":"+slot) + del cps + + # now match visibles + available = set() + for visible in visibles: + match = self.get_best_atom(visible) + if filter_reinstalls: + installed = self.get_installed_atom(visible) + # if not installed, installed == None + if installed != match: + available.add(match) + else: + available.add(match) + del visibles + + return available + + # Collect installed packages + def get_installed_packages(self, dbdir = None): + if not dbdir: + appDbDir = self.get_vdb_path() + else: + appDbDir = dbdir + dbDirs = os.listdir(appDbDir) + installedAtoms = set() + for pkgsdir in dbDirs: + if os.path.isdir(appDbDir+pkgsdir): + pkgdir = os.listdir(appDbDir+pkgsdir) + for pdir in pkgdir: + pkgcat = pkgsdir.split("/")[len(pkgsdir.split("/"))-1] + pkgatom = pkgcat+"/"+pdir + if pkgatom.find("-MERGING-") == -1: + installedAtoms.add(pkgatom) + return list(installedAtoms), len(installedAtoms) + + def get_installed_packages_counter(self, dbdir = None): + if not dbdir: + appDbDir = self.get_vdb_path() + else: + appDbDir = dbdir + dbDirs = os.listdir(appDbDir) + installedAtoms = set() + for pkgsdir in dbDirs: + if not os.path.isdir(appDbDir+pkgsdir): + continue + pkgdir = os.listdir(appDbDir+pkgsdir) + for pdir in pkgdir: + pkgcat = pkgsdir.split("/")[len(pkgsdir.split("/"))-1] + pkgatom = pkgcat+"/"+pdir + if pkgatom.find("-MERGING-") == -1: + # get counter + try: + f = open(appDbDir+pkgsdir+"/"+pdir+"/"+dbCOUNTER,"r") + counter = int(f.readline().strip()) + f.close() + except IOError: + continue + except ValueError: + continue + installedAtoms.add((pkgatom,counter)) + return installedAtoms, len(installedAtoms) + + def refill_counter(self, dbdir = None): + if not dbdir: + appDbDir = self.get_vdb_path() + else: + appDbDir = dbdir + counters = set() + for catdir in os.listdir(appDbDir): + catdir = appDbDir+catdir + if not os.path.isdir(catdir): + continue + for pkgdir in os.listdir(catdir): + pkgdir = catdir+"/"+pkgdir + if not os.path.isdir(pkgdir): + continue + counterfile = pkgdir+"/"+dbCOUNTER + if not os.path.isfile(pkgdir+"/"+dbCOUNTER): + continue + try: + f = open(counterfile,"r") + counter = int(f.readline().strip()) + counters.add(counter) + f.close() + except: + continue + newcounter = max(counters) + if not os.path.isdir(os.path.dirname(etpConst['edbcounter'])): + os.makedirs(os.path.dirname(etpConst['edbcounter'])) + try: + f = open(etpConst['edbcounter'],"w") + except IOError, e: + if e[0] == 21: + shutil.rmtree(etpConst['edbcounter'],True) + try: + os.rmdir(etpConst['edbcounter']) + except: + pass + f = open(etpConst['edbcounter'],"w") + f.write(str(newcounter)) + f.flush() + f.close() + del counters + return newcounter + + + def spm_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []): + + rc = self.entropyTools.spawnFunction( self._portage_doebuild, + myebuild, + mydo, + tree, + cpv, + portage_tmpdir, + licenses + ) + return rc + + def _portage_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []): + # myebuild = path/to/ebuild.ebuild with a valid unpacked xpak metadata + # tree = "bintree" + # tree = "bintree" + # cpv = atom + ''' + # This is a demonstration that Sabayon team love Gentoo so much + [01:46] if you want something to stay in mysettings + [01:46] do mysettings.backup_changes("CFLAGS") for example + [01:46] otherwise your change can get lost inside doebuild() + [01:47] because it calls mysettings.reset() + # ^^^ this is DA MAN! + ''' + # mydbapi = portage.fakedbapi(settings=portage.settings) + # vartree = portage.vartree(root=myroot) + + oldsystderr = sys.stderr + f = open("/dev/null","w") + sys.stderr = f + + ### SETUP ENVIRONMENT + # if mute, supress portage output + domute = False + if etpUi['mute']: + domute = True + oldsysstdout = sys.stdout + sys.stdout = f + + mypath = etpConst['systemroot']+"/" + os.environ["SKIP_EQUO_SYNC"] = "1" + os.environ["CD_ROOT"] = "/tmp" # workaround for scripts asking for user intervention + os.environ["ROOT"] = mypath + + if licenses: + os.environ["ACCEPT_LICENSE"] = str(' '.join(licenses)) # we already do this early + + # load metadata + myebuilddir = os.path.dirname(myebuild) + keys = self.portage.auxdbkeys + metadata = {} + + for key in keys: + mykeypath = os.path.join(myebuilddir,key) + if os.path.isfile(mykeypath) and os.access(mykeypath,os.R_OK): + f = open(mykeypath,"r") + metadata[key] = f.readline().strip() + f.close() + + ### END SETUP ENVIRONMENT + + # find config + mysettings = self._get_portage_config("/",mypath) + mysettings['EBUILD_PHASE'] = mydo + + try: # this is a >portage-2.1.4_rc11 feature + mysettings._environ_whitelist = set(mysettings._environ_whitelist) + # put our vars into whitelist + mysettings._environ_whitelist.add("SKIP_EQUO_SYNC") + mysettings._environ_whitelist.add("ACCEPT_LICENSE") + mysettings._environ_whitelist.add("CD_ROOT") + mysettings._environ_whitelist.add("ROOT") + mysettings._environ_whitelist = frozenset(mysettings._environ_whitelist) + except: + pass + + cpv = str(cpv) + mysettings.setcpv(cpv) + portage_tmpdir_created = False # for pkg_postrm, pkg_prerm + if portage_tmpdir: + if not os.path.isdir(portage_tmpdir): + os.makedirs(portage_tmpdir) + portage_tmpdir_created = True + mysettings['PORTAGE_TMPDIR'] = str(portage_tmpdir) + mysettings.backup_changes("PORTAGE_TMPDIR") + + mydbapi = self.portage.fakedbapi(settings=mysettings) + mydbapi.cpv_inject(cpv, metadata = metadata) + + # cached vartree class + vartree = self._get_portage_vartree(mypath) + + rc = self.portage.doebuild(myebuild = str(myebuild), mydo = str(mydo), myroot = mypath, tree = tree, mysettings = mysettings, mydbapi = mydbapi, vartree = vartree, use_cache = 0) + + # if mute, restore old stdout/stderr + if domute: + sys.stdout = oldsysstdout + + sys.stderr = oldsystderr + f.close() + + if portage_tmpdir_created: + shutil.rmtree(portage_tmpdir,True) + + del mydbapi + del metadata + del keys + return rc + diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 9406077ef..67a3bdd0c 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -420,10 +420,6 @@ def const_resetCache(): repo_error_messages_cache.clear() maskingReasonsStorage.clear() -# Inside it you'll find instantiated vartree classes -portageRoots = {} -portageConfigs = {} - # Client packages/database repositories etpRepositories = {} etpRepositoriesExcluded = {} @@ -637,7 +633,10 @@ def initConfig_entropyConstants(rootdir): 'exec': "/usr/bin/emerge", # source package manager executable 'ask_cmd': "--ask", 'pretend_cmd': "--pretend", - 'verbose_cmd': "--verbose" + 'verbose_cmd': "--verbose", + 'backend': "portage", + 'available_backends': ["portage"], + 'cache': {} }, 'downloadspeedlimit': None, # equo packages download speed limit (in kb/sec) diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index a85906e70..865003312 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -1607,10 +1607,10 @@ def saveRepositorySettings(repodata, remove = False, disable = False, enable = F # inject new repodata keys = repolines_data.keys() keys.sort() - for c in keys: - repoid = repolines_data[c]['repoid'] + for cc in keys: + repoid = repolines_data[cc]['repoid'] # write the first - line = repolines_data[c]['line'] + line = repolines_data[cc]['line'] content.append(line) _saveRepositoriesContent(content) @@ -1729,12 +1729,14 @@ def quickpkg(pkgdata, dirpath, edb = True, portdbPath = None, fake = False, comp # appending xpak metadata if etpConst['gentoo-compat']: import etpXpak + from entropy import SpmInterface + SpmIntf = SpmInterface(None) + Spm = SpmIntf.intf gentoo_name = remove_tag(pkgname) gentoo_name = remove_entropy_revision(gentoo_name) if portdbPath == None: - from portageTools import getPortageAppDbPath - dbdir = getPortageAppDbPath()+"/"+pkgcat+"/"+gentoo_name+"/" + dbdir = Spm.get_vdb_path()+"/"+pkgcat+"/"+gentoo_name+"/" else: dbdir = portdbPath+"/"+pkgcat+"/"+gentoo_name+"/" if os.path.isdir(dbdir): @@ -1761,8 +1763,10 @@ def quickpkg(pkgdata, dirpath, edb = True, portdbPath = None, fake = False, comp def appendXpak(tbz2file, atom): import etpXpak - from portageTools import getPortageAppDbPath - dbdir = getPortageAppDbPath()+"/"+atom+"/" + from entropy import SpmInterface + SpmIntf = SpmInterface(None) + Spm = SpmIntf.intf + dbdir = Spm.get_vdb_path()+"/"+atom+"/" if os.path.isdir(dbdir): tbz2 = etpXpak.tbz2(tbz2file) tbz2.recompose(dbdir) @@ -1773,7 +1777,9 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje data = {} - from portageTools import calculate_dependencies, getPackagesInSystem, getConfigProtectAndMask, getThirdPartyMirrors + from entropy import SpmInterface + SpmIntf = SpmInterface(None) + Spm = SpmIntf.intf info_package = bold(os.path.basename(package))+": " @@ -1822,6 +1828,10 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje if not silent: print_info(yellow(" * ")+red(info_package+"Unpacking package data..."),back = True) # unpack file tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/" + if not os.path.isdir(tbz2TmpDir): + if os.path.lexists(tbz2TmpDir): + os.remove(tbz2TmpDir) + os.makedirs(tbz2TmpDir) extractXpak(tbz2File,tbz2TmpDir) if not silent: print_info(yellow(" * ")+red(info_package+"Getting package CHOST..."),back = True) @@ -2122,7 +2132,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje sources = "" if not silent: print_info(yellow(" * ")+red(info_package+"Getting package metadata information..."),back = True) - portage_metadata = calculate_dependencies(iuse, use, lics, depend, rdepend, pdepend, provide, sources) + portage_metadata = Spm.calculate_dependencies(iuse, use, lics, depend, rdepend, pdepend, provide, sources) data['provide'] = portage_metadata['PROVIDE'].split() data['license'] = portage_metadata['LICENSE'] @@ -2146,12 +2156,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje data['conflicts'].append(conflict) # Get License text if possible - licenses_dir = None - try: - from portageTools import getPortageEnv - licenses_dir = os.path.join(getPortageEnv('PORTDIR'),'licenses') - except: - pass + licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses') data['licensedata'] = {} if licenses_dir: licdata = [str(x.strip()) for x in data['license'].split() if str(x.strip()) and is_valid_string(x.strip())] @@ -2172,13 +2177,13 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje if i.startswith("mirror://"): # parse what mirror I need mirrorURI = i.split("/")[2] - mirrorlist = getThirdPartyMirrors(mirrorURI) + mirrorlist = Spm.get_third_party_mirrors(mirrorURI) data['mirrorlinks'].append([mirrorURI,mirrorlist]) # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] if not silent: print_info(yellow(" * ")+red(info_package+"Getting System Packages List..."),back = True) # write only if it's a systempackage data['systempackage'] = '' - systemPackages = getPackagesInSystem() + systemPackages = Spm.get_atoms_in_system() for x in systemPackages: x = dep_getkey(x) y = data['category']+"/"+data['name'] @@ -2189,7 +2194,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje if not silent: print_info(yellow(" * ")+red(info_package+"Getting CONFIG_PROTECT/CONFIG_PROTECT_MASK List..."),back = True) # write only if it's a systempackage - protect, mask = getConfigProtectAndMask() + protect, mask = Spm.get_config_protect_and_mask() data['config_protect'] = protect data['config_protect_mask'] = mask @@ -2276,41 +2281,6 @@ def collectPaths(): path |= paths return path -# this is especially used to try to guess portage bytecoded entries in CONTENTS -def string_to_utf8(string): - done = False - - # simple unicode? - try: - newstring = unicode(string) - done = True - except: - pass - if done: - return newstring - - # try utf8 - try: - newstring = string.decode("iso-8859-1") - done = True - except: - pass - if done: - return newstring - - # try latin1 + iso-8859-1 - try: - newstring = string.decode("latin1") - done = True - except: - pass - if done: - return newstring - - # otherwise return None - print "DEBUG: cannot encode into filesystem encoding -> "+unicode(string) - return None - def listToUtf8(mylist): mynewlist = [] for item in mylist: @@ -2322,3 +2292,40 @@ def listToUtf8(mylist): except: raise return mynewlist + +# used by PortageInterface +class paren_normalize(list): + """Take a dependency structure as returned by paren_reduce or use_reduce + and generate an equivalent structure that has no redundant lists.""" + def __init__(self, src): + list.__init__(self) + self._zap_parens(src, self) + + def _zap_parens(self, src, dest, disjunction=False): + if not src: + return dest + i = iter(src) + for x in i: + if isinstance(x, basestring): + if x == '||': + x = self._zap_parens(i.next(), [], disjunction=True) + if len(x) == 1: + dest.append(x[0]) + else: + dest.append("||") + dest.append(x) + elif x.endswith("?"): + dest.append(x) + dest.append(self._zap_parens(i.next(), [])) + else: + dest.append(x) + else: + if disjunction: + x = self._zap_parens(x, []) + if len(x) == 1: + dest.append(x[0]) + else: + dest.append(x) + else: + self._zap_parens(x, dest) + return dest \ No newline at end of file diff --git a/libraries/portageTools.py b/libraries/portageTools.py deleted file mode 100644 index bd6dff345..000000000 --- a/libraries/portageTools.py +++ /dev/null @@ -1,898 +0,0 @@ -#!/usr/bin/python -''' - # DESCRIPTION: - # Entropy Portage Interface - - Copyright (C) 2007-2008 Fabio Erculiani - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -''' - -############ -# Portage initialization -##################################################################################### - -from entropyConstants import * -import entropyTools -import portage - -############ -# Functions and Classes -##################################################################################### - -def getThirdPartyMirrors(mirrorname): - try: - return portage.thirdpartymirrors[mirrorname] - except KeyError: - return [] - -def getPortageEnv(var): - return portage.settings[var] - -# Packages in system (in the Portage language -> emerge system, remember?) -def getPackagesInSystem(): - system = portage.settings.packages - sysoutput = [] - for x in system: - y = getInstalledAtoms(x) - if (y != None): - for z in y: - sysoutput.append(z) - sysoutput.append("sys-kernel/linux-sabayon") # our kernel - sysoutput.append("dev-db/sqlite") # our interface - sysoutput.append("dev-python/pysqlite") # our python interface to our interface (lol) - sysoutput.append("virtual/cron") # our cron service - sysoutput.append("app-admin/equo") # our package manager (client) - sysoutput.append("sys-apps/entropy") # our package manager (server+client) - return sysoutput - -def getConfigProtectAndMask(): - import commands - config_protect = portage.settings['CONFIG_PROTECT'] - config_protect = config_protect.split() - config_protect_mask = portage.settings['CONFIG_PROTECT_MASK'] - config_protect_mask = config_protect_mask.split() - # explode - protect = [] - for x in config_protect: - if x.startswith("$"): #FIXME: small hack - x = commands.getoutput("echo "+x).split("\n")[0] - protect.append(x) - mask = [] - for x in config_protect_mask: - if x.startswith("$"): #FIXME: small hack - x = commands.getoutput("echo "+x).split("\n")[0] - mask.append(x) - return ' '.join(protect),' '.join(mask) - -# resolve atoms automagically (best, not current!) -# sys-libs/application --> sys-libs/application-1.2.3-r1 -def getBestAtom(atom, match = "bestmatch-visible"): - try: - rc = portage.portdb.xmatch(match,str(atom)) - return rc - except ValueError: - return "!!conflicts" - -# same as above but includes masked ebuilds -def getBestMaskedAtom(atom): - atoms = portage.portdb.xmatch("match-all",str(atom)) - # find the best - try: - from portage_versions import best - except ImportError: - from portage.versions import best - - rc = best(atoms) - return rc - -# should be only used when a pkgcat/pkgname <-- is not specified (example: db, amarok, AND NOT media-sound/amarok) -def getAtomCategory(atom): - try: - rc = portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0] - return rc - except: - return None - -# please always force =pkgcat/pkgname-ver if possible -def getInstalledAtom(atom): - mypath = etpConst['systemroot']+"/" - try: - cached = portageRoots.get(mypath) - if cached == None: - mytree = portage.vartree(root=mypath) - portageRoots[mypath] = mytree - else: - mytree = cached - except NameError: - mytree = portage.vartree(root=mypath) - rc = mytree.dep_match(str(atom)) - if (rc != []): - if (len(rc) == 1): - return rc[0] - else: - return rc[len(rc)-1] - else: - return None - -def getPackageSlot(atom): - mypath = etpConst['systemroot']+"/" - try: - cached = portageRoots.get(mypath) - if cached == None: - mytree = portage.vartree(root=mypath) - portageRoots[mypath] = mytree - else: - mytree = cached - except NameError: - mytree = portage.vartree(root=mypath) - if atom.startswith("="): - atom = atom[1:] - rc = mytree.getslot(atom) - if rc != "": - return rc - else: - return None - -def getInstalledAtoms(atom): - mypath = etpConst['systemroot']+"/" - try: - cached = portageRoots.get(mypath) - if cached == None: - mytree = portage.vartree(root=mypath) - portageRoots[mypath] = mytree - else: - mytree = cached - except NameError: - mytree = portage.vartree(root=mypath) - rc = mytree.dep_match(str(atom)) - if (rc != []): - return rc - else: - return None - -def parseElogFile(atom): - - import commands - - if atom.startswith("="): - atom = atom[1:] - if atom.startswith(">"): - atom = atom[1:] - if atom.startswith("<"): - atom = atom[1:] - if (atom.find("/") != -1): - pkgcat = atom.split("/")[0] - pkgnamever = atom.split("/")[1]+"*.log" - else: - pkgcat = "*" - pkgnamever = atom+"*.log" - elogfile = pkgcat+":"+pkgnamever - reallogfile = commands.getoutput("find "+etpConst['logdir']+"/elog/ -name '"+elogfile+"'").split("\n")[0].strip() - if os.path.isfile(reallogfile): - # FIXME: improve this part - logline = False - logoutput = [] - f = open(reallogfile,"r") - reallog = f.readlines() - f.close() - for line in reallog: - if line.startswith("INFO: postinst") or line.startswith("LOG: postinst"): - logline = True - continue - # disable all the others - elif line.startswith("INFO:") or line.startswith("LOG:"): - logline = False - continue - if (logline) and (line.strip() != ""): - # trap ! - logoutput.append(line.strip()) - return logoutput - else: - return [] - -# create a .tbz2 file in the specified path -def quickpkg(atom,dirpath): - - # getting package info - pkgname = atom.split("/")[1] - pkgcat = atom.split("/")[0] - #pkgfile = pkgname+".tbz2" - if not os.path.isdir(dirpath): - os.makedirs(dirpath) - dirpath += "/"+pkgname+".tbz2" - dbdir = getPortageAppDbPath()+"/"+pkgcat+"/"+pkgname+"/" - - import tarfile - import stat - trees = portage.db["/"] - vartree = trees["vartree"] - dblnk = portage.dblink(pkgcat, pkgname, "/", vartree.settings, treetype="vartree", vartree=vartree) - dblnk.lockdb() - tar = tarfile.open(dirpath,"w:bz2") - - contents = dblnk.getcontents() - id_strings = {} - paths = contents.keys() - paths.sort() - - for path in paths: - try: - exist = os.lstat(path) - except OSError: - continue # skip file - ftype = contents[path][0] - lpath = path - arcname = path[1:] - if 'dir' == ftype and \ - not stat.S_ISDIR(exist.st_mode) and \ - os.path.isdir(lpath): - lpath = os.path.realpath(lpath) - tarinfo = tar.gettarinfo(lpath, arcname) - tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) - tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) - - if stat.S_ISREG(exist.st_mode): - tarinfo.type = tarfile.REGTYPE - f = open(path) - try: - tar.addfile(tarinfo, f) - finally: - f.close() - else: - tar.addfile(tarinfo) - - tar.close() - - # appending xpak informations - import etpXpak - tbz2 = etpXpak.tbz2(dirpath) - tbz2.recompose(dbdir) - - dblnk.unlockdb() - - if os.path.isfile(dirpath): - return dirpath - else: - return False - -def getUSEFlags(): - return portage.settings['USE'] - -def getUSEForce(): - return portage.settings.useforce - -def getUSEMask(): - return portage.settings.usemask - -def getMAKEOPTS(): - return portage.settings['MAKEOPTS'] - -def getCFLAGS(): - return portage.settings['CFLAGS'] - -def getLDFLAGS(): - return portage.settings['LDFLAGS'] - -# you must provide a complete atom -def getPackageIUSE(atom): - return getPackageVar(atom,"IUSE") - -def getPackageVar(atom,var): - if atom.startswith("="): - atom = atom[1:] - # can't check - return error - if (atom.find("/") == -1): - return 1 - return portage.portdb.aux_get(atom,[var])[0] - -class paren_normalize(list): - """Take a dependency structure as returned by paren_reduce or use_reduce - and generate an equivalent structure that has no redundant lists.""" - def __init__(self, src): - list.__init__(self) - self._zap_parens(src, self) - - def _zap_parens(self, src, dest, disjunction=False): - if not src: - return dest - i = iter(src) - for x in i: - if isinstance(x, basestring): - if x == '||': - x = self._zap_parens(i.next(), [], disjunction=True) - if len(x) == 1: - dest.append(x[0]) - else: - dest.append("||") - dest.append(x) - elif x.endswith("?"): - dest.append(x) - dest.append(self._zap_parens(i.next(), [])) - else: - dest.append(x) - else: - if disjunction: - x = self._zap_parens(x, []) - if len(x) == 1: - dest.append(x[0]) - else: - dest.append(x) - else: - self._zap_parens(x, dest) - return dest - -def calculate_dependencies(my_iuse, my_use, my_license, my_depend, my_rdepend, my_pdepend, my_provide, my_src_uri): - metadata = {} - metadata['USE'] = my_use - metadata['IUSE'] = my_iuse - metadata['LICENSE'] = my_license - metadata['DEPEND'] = my_depend - metadata['PDEPEND'] = my_pdepend - metadata['RDEPEND'] = my_rdepend - metadata['PROVIDE'] = my_provide - metadata['SRC_URI'] = my_src_uri - use = metadata['USE'].split() - raw_use = use - iuse = set(metadata['IUSE'].split()) - use = [f for f in use if f in iuse] - use.sort() - metadata['USE'] = " ".join(use) - # FIXME: there's some portage trunk stuff - try: - from portage_dep import paren_reduce, use_reduce, paren_enclose - p_normalize = paren_normalize - except ImportError: - from portage.dep import paren_reduce, use_reduce, paren_normalize as p_normalize, paren_enclose - for k in "LICENSE", "RDEPEND", "DEPEND", "PDEPEND", "PROVIDE", "SRC_URI": - try: - deps = paren_reduce(metadata[k]) - deps = use_reduce(deps, uselist=raw_use) - deps = p_normalize(deps) - if k == "LICENSE": - deps = paren_license_choose(deps) - else: - deps = paren_choose(deps) - deps = ' '.join(deps) - except Exception, e: - print "%s: %s\n" % (k, str(e)) - deps = '' - continue - metadata[k] = deps - return metadata - -def paren_choose(dep_list): - - newlist = [] - do_skip = False - for idx in range(len(dep_list)): - - if do_skip: - do_skip = False - continue - - item = dep_list[idx] - if item == "||": # or - next_item = dep_list[idx+1] - if not next_item: # || ( asd? ( atom ) dsa? ( atom ) ) => [] if use asd and dsa are disabled - do_skip = True - continue - item = dep_or_select(next_item) # must be a list - if not item: - # no matches, transform to string and append, so reagent will fail - newlist.append(str(next_item)) - else: - newlist += item - do_skip = True - elif isinstance(item, list): # and - item = dep_and_select(item) - newlist += item - else: - newlist.append(item) - - return newlist - -def dep_and_select(and_list): - do_skip = False - newlist = [] - for idx in range(len(and_list)): - - if do_skip: - do_skip = False - continue - - x = and_list[idx] - if x == "||": - x = dep_or_select(and_list[idx+1]) - do_skip = True - if not x: - x = str(and_list[idx+1]) - else: - newlist += x - elif isinstance(x, list): - x = dep_and_select(x) - newlist += x - else: - newlist.append(x) - - # now verify if all are satisfied - for x in newlist: - match = getInstalledAtom(x) - if match == None: - return [] - - return newlist - -def dep_or_select(or_list): - do_skip = False - for idx in range(len(or_list)): - if do_skip: - do_skip = False - continue - x = or_list[idx] - if x == "||": # or - x = dep_or_select(or_list[idx+1]) - do_skip = True - elif isinstance(x, list): # and - x = dep_and_select(x) - if not x: - continue - # found - return x - else: - x = [x] - - for y in x: - match = getInstalledAtom(y) - if match != None: - return [y] - - return [] - -def paren_license_choose(dep_list): - - newlist = set() - for item in dep_list: - - if isinstance(item, list): - # match the first - data = set(paren_license_choose(item)) - newlist.update(data) - else: - if item not in ["||"]: - newlist.add(item) - - return list(newlist) - -## -## HIGHLY DEPRECATED, USE calculate_dependencies -## -def synthetizeRoughDependencies(roughDependencies, useflags = None): - if useflags is None: - useflags = getUSEFlags() - # returns dependencies, conflicts - - useMatch = False - openParenthesis = 0 - openParenthesisFromOr = 0 - openOr = False - useFlagQuestion = False - dependencies = "" - conflicts = "" - useflags = useflags.split() - - length = len(roughDependencies) - atomcount = -1 - - while atomcount < length: - - atomcount += 1 - try: - atom = roughDependencies[atomcount] - except: - break - - if atom.startswith("("): - if (openOr): - openParenthesisFromOr += 1 - openParenthesis += 1 - curparenthesis = openParenthesis # 2 - if (useFlagQuestion) and (not useMatch): - skip = True - while (skip): - atomcount += 1 - atom = roughDependencies[atomcount] - if atom.startswith("("): - curparenthesis += 1 - elif atom.startswith(")"): - if (curparenthesis == openParenthesis): - skip = False - curparenthesis -= 1 - useFlagQuestion = False - - elif atom.endswith("?"): - - #if (useFlagQuestion) and (not useMatch): # if we're already in a question and the question is not accepted, skip the cycle - # continue - # we need to see if that useflag is enabled - useFlag = atom.split("?")[0] - useFlagQuestion = True # V - #openParenthesisFromLastUseFlagQuestion = 0 - if useFlag.startswith("!"): - checkFlag = useFlag[1:] - try: - useflags.index(checkFlag) - useMatch = False - except: - useMatch = True - else: - try: - useflags.index(useFlag) - useMatch = True # V - except: - useMatch = False - - elif atom.startswith(")"): - - openParenthesis -= 1 - if (openParenthesis == 0): - useFlagQuestion = False - useMatch = False - - if (openOr): - # remove last "_or_" from dependencies - if (openParenthesisFromOr == 1): - openOr = False - if dependencies.endswith(dbOR): - dependencies = dependencies[:len(dependencies)-len(dbOR)] - dependencies += " " - elif (openParenthesisFromOr == 2): - if dependencies.endswith("|and|"): - dependencies = dependencies[:len(dependencies)-len("|and|")] - dependencies += dbOR - openParenthesisFromOr -= 1 - - elif atom.startswith("||"): - openOr = True # V - - elif (atom.find("/") != -1) and (not atom.startswith("!")) and (not atom.endswith("?")): - # it's a package name /-??? - if (useFlagQuestion == useMatch): - # check if there's an OR - if (openOr): - dependencies += atom - # check if the or is fucked up - if openParenthesisFromOr > 1: - dependencies += "|and|" # !! - else: - dependencies += dbOR - else: - dependencies += atom - dependencies += " " - - elif atom.startswith("!") and (not atom.endswith("?")): - if ((useFlagQuestion) and (useMatch)) or ((not useFlagQuestion) and (not useMatch)): - conflicts += atom - if (openOr): - conflicts += dbOR - else: - conflicts += " " - - - # format properly - tmpConflicts = list(set(conflicts.split())) - conflicts = '' - tmpData = [] - for i in tmpConflicts: - i = i[1:] # remove "!" - tmpData.append(i) - conflicts = ' '.join(tmpData) - - tmpData = [] - tmpDeps = list(set(dependencies.split())) - dependencies = '' - for i in tmpDeps: - tmpData.append(i) - - # now filter |or| and |and| - _tmpData = [] - for dep in tmpData: - - if dep.find("|or|") != -1: - deps = dep.split("|or|") - # find the best - results = [] - for x in deps: - if x.find("|and|") != -1: - anddeps = x.split("|and|") - results.append(anddeps) - else: - if x: - results.append([x]) - - # now parse results - for result in results: - outdeps = result[:] - for y in result: - yresult = getInstalledAtoms(y) - if (yresult != None): - outdeps.remove(y) - if (not outdeps): - # find it - for y in result: - _tmpData.append(y) - break - - else: - _tmpData.append(dep) - - dependencies = ' '.join(_tmpData) - - return dependencies, conflicts - -def getPortageAppDbPath(): - try: - import portage_const - except ImportError: - import portage.const as portage_const - rc = etpConst['systemroot']+"/"+portage_const.VDB_PATH - if (not rc.endswith("/")): - return rc+"/" - return rc - -def getAvailablePackages(categories = [], filter_reinstalls = True): - try: - import portage_const - except ImportError: - import portage.const as portage_const - mypath = etpConst['systemroot']+"/" - mysettings = portage.config(config_root="/", target_root=mypath, config_incrementals=portage_const.INCREMENTALS) - portdb = portage.portdbapi(mysettings["PORTDIR"], mysettings = mysettings) - cps = portdb.cp_all() - visibles = set() - for cp in cps: - if categories and cp.split("/")[0] not in categories: - continue - # get slots - slots = set() - atoms = getBestAtom(cp, "match-visible") - for atom in atoms: - slots.add(portdb.aux_get(atom, ["SLOT"])[0]) - for slot in slots: - visibles.add(cp+":"+slot) - del cps - - # now match visibles - available = set() - for visible in visibles: - match = getBestAtom(visible) - if filter_reinstalls: - installed = getInstalledAtom(visible) - # if not installed, installed == None - if installed != match: - available.add(match) - else: - available.add(match) - del visibles - - return available - - -# Collect installed packages -def getInstalledPackages(dbdir = None): - if not dbdir: - appDbDir = getPortageAppDbPath() - else: - appDbDir = dbdir - dbDirs = os.listdir(appDbDir) - installedAtoms = [] - for pkgsdir in dbDirs: - if os.path.isdir(appDbDir+pkgsdir): - pkgdir = os.listdir(appDbDir+pkgsdir) - for pdir in pkgdir: - pkgcat = pkgsdir.split("/")[len(pkgsdir.split("/"))-1] - pkgatom = pkgcat+"/"+pdir - if pkgatom.find("-MERGING-") == -1: - installedAtoms.append(pkgatom) - return installedAtoms, len(installedAtoms) - -def getInstalledPackagesCounters(): - appDbDir = getPortageAppDbPath() - dbDirs = os.listdir(appDbDir) - installedAtoms = set() - for pkgsdir in dbDirs: - if not os.path.isdir(appDbDir+pkgsdir): - continue - pkgdir = os.listdir(appDbDir+pkgsdir) - for pdir in pkgdir: - pkgcat = pkgsdir.split("/")[len(pkgsdir.split("/"))-1] - pkgatom = pkgcat+"/"+pdir - if pkgatom.find("-MERGING-") == -1: - # get counter - try: - f = open(appDbDir+pkgsdir+"/"+pdir+"/"+dbCOUNTER,"r") - counter = int(f.readline().strip()) - f.close() - except IOError: - continue - except ValueError: - continue - installedAtoms.add((pkgatom,counter)) - return installedAtoms, len(installedAtoms) - -def refillCounter(): - appDbDir = getPortageAppDbPath() - counters = set() - for catdir in os.listdir(appDbDir): - catdir = appDbDir+catdir - if not os.path.isdir(catdir): - continue - for pkgdir in os.listdir(catdir): - pkgdir = catdir+"/"+pkgdir - if not os.path.isdir(pkgdir): - continue - counterfile = pkgdir+"/"+dbCOUNTER - if not os.path.isfile(pkgdir+"/"+dbCOUNTER): - continue - try: - f = open(counterfile,"r") - counter = int(f.readline().strip()) - counters.add(counter) - f.close() - except: - continue - newcounter = max(counters) - if not os.path.isdir(os.path.dirname(etpConst['edbcounter'])): - os.makedirs(os.path.dirname(etpConst['edbcounter'])) - try: - f = open(etpConst['edbcounter'],"w") - except IOError, e: - if e[0] == 21: - import shutil - shutil.rmtree(etpConst['edbcounter'],True) - try: - os.rmdir(etpConst['edbcounter']) - except: - pass - f = open(etpConst['edbcounter'],"w") - f.write(str(newcounter)) - f.flush() - f.close() - del counters - return newcounter - -def portage_doebuild(myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []): - - rc = entropyTools.spawnFunction( _portage_doebuild, - myebuild, - mydo, - tree, - cpv, - portage_tmpdir, - licenses - ) - return rc - -def _portage_doebuild(myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []): - try: - import portage_const - except ImportError: - import portage.const as portage_const - # myebuild = path/to/ebuild.ebuild with a valid unpacked xpak metadata - # tree = "bintree" - # tree = "bintree" - # cpv = atom - ''' - # This is a demonstration that Sabayon team love Gentoo so much - [01:46] if you want something to stay in mysettings - [01:46] do mysettings.backup_changes("CFLAGS") for example - [01:46] otherwise your change can get lost inside doebuild() - [01:47] because it calls mysettings.reset() - # ^^^ this is DA MAN! - ''' - # mydbapi = portage.fakedbapi(settings=portage.settings) - # vartree = portage.vartree(root=myroot) - - oldsystderr = sys.stderr - f = open("/dev/null","w") - sys.stderr = f - - ### SETUP ENVIRONMENT - # if mute, supress portage output - domute = False - if etpUi['mute']: - domute = True - oldsysstdout = sys.stdout - sys.stdout = f - - mypath = etpConst['systemroot']+"/" - os.environ["SKIP_EQUO_SYNC"] = "1" - os.environ["CD_ROOT"] = "/tmp" # workaround for scripts asking for user intervention - os.environ["ROOT"] = mypath - - if licenses: - os.environ["ACCEPT_LICENSE"] = str(' '.join(licenses)) # we already do this early - - # load metadata - myebuilddir = os.path.dirname(myebuild) - keys = portage.auxdbkeys - metadata = {} - - for key in keys: - mykeypath = os.path.join(myebuilddir,key) - if os.path.isfile(mykeypath) and os.access(mykeypath,os.R_OK): - f = open(mykeypath,"r") - metadata[key] = f.readline().strip() - f.close() - - ### END SETUP ENVIRONMENT - - # find config - if portageConfigs.has_key(mypath): - mysettings = portageConfigs.get(mypath) - else: - mysettings = portage.config(config_root="/", target_root=mypath, config_incrementals=portage_const.INCREMENTALS) - #portageConfigs[mypath] = mysettings - mysettings['EBUILD_PHASE'] = mydo - - try: # this is a >portage-2.1.4_rc11 feature - mysettings._environ_whitelist = set(mysettings._environ_whitelist) - # put our vars into whitelist - mysettings._environ_whitelist.add("SKIP_EQUO_SYNC") - mysettings._environ_whitelist.add("ACCEPT_LICENSE") - mysettings._environ_whitelist.add("CD_ROOT") - mysettings._environ_whitelist.add("ROOT") - mysettings._environ_whitelist = frozenset(mysettings._environ_whitelist) - except: - pass - - cpv = str(cpv) - mysettings.setcpv(cpv) - portage_tmpdir_created = False # for pkg_postrm, pkg_prerm - if portage_tmpdir: - if not os.path.isdir(portage_tmpdir): - os.makedirs(portage_tmpdir) - portage_tmpdir_created = True - mysettings['PORTAGE_TMPDIR'] = str(portage_tmpdir) - mysettings.backup_changes("PORTAGE_TMPDIR") - - mydbapi = portage.fakedbapi(settings=mysettings) - mydbapi.cpv_inject(cpv, metadata = metadata) - - # cached vartree class - if portageRoots.has_key(mypath): - vartree = portageRoots.get(mypath) - else: - vartree = portage.vartree(root=mypath) - portageRoots[mypath] = vartree - - ### FIXME: add support for cache_overlay - rc = portage.doebuild(myebuild = str(myebuild), mydo = str(mydo), myroot = mypath, tree = tree, mysettings = mysettings, mydbapi = mydbapi, vartree = vartree, use_cache = 0) - - # if mute, restore old stdout/stderr - if domute: - sys.stdout = oldsysstdout - - sys.stderr = oldsystderr - f.close() - - if portage_tmpdir_created: - import shutil - shutil.rmtree(portage_tmpdir,True) - - del mydbapi - del metadata - del keys - return rc diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index dbf78b3a4..b2c2452b8 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -127,6 +127,7 @@ def update(options): continue _options.append(opt) options = _options + Spm = Entropy.Spm() if (not reagentRequestSeekStore): @@ -134,8 +135,7 @@ def update(options): if not reagentRequestRepackage: print_info(brown(" * ")+red("Scanning database for differences...")) - from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot - installedPackages = getInstalledPackagesCounters() + installedPackages = Spm.get_installed_packages_counter() installedCounters = set() toBeAdded = set() toBeRemoved = set() @@ -164,7 +164,7 @@ def update(options): add = True for pkgdata in toBeAdded: - addslot = getPackageSlot(pkgdata[0]) + addslot = Spm.get_package_slot(pkgdata[0]) addkey = Entropy.entropyTools.dep_getkey(pkgdata[0]) # workaround for ebuilds not having slot if addslot == None: @@ -249,9 +249,7 @@ def update(options): # then exit gracefully return 0 - from portageTools import getPortageAppDbPath,quickpkg - appdb = getPortageAppDbPath() - + appdb = Spm.get_vdb_path() packages = [] for item in repackageItems: match = dbconn.atomMatch(item) @@ -276,7 +274,7 @@ def update(options): print_info(brown(" @@ ")+blue("Compressing packages...")) for x in toBeAdded: print_info(brown(" # ")+red(x[0]+"...")) - rc = quickpkg(x[0],etpConst['packagesstoredir']) + rc = Spm.quickpkg(x[0],etpConst['packagesstoredir']) if (rc is None): reagentLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"update: "+str(x)+" -> quickpkg error. Cannot continue.") print_error(red(" *")+" quickpkg error for "+red(x)) @@ -402,7 +400,7 @@ def librariesTest(listfiles = False): return 1 if listfiles: - for x in brokenlibs: + for x in brokenexecs: print x return 0 @@ -412,7 +410,7 @@ def librariesTest(listfiles = False): atomsdata = set() - print_info(red(" @@ ")+blue("Matching libraries with Portage:")) + print_info(red(" @@ ")+blue("Matching libraries with Spm:")) qfile_exec = "/usr/bin/qfile" qfile_opts = " -qCe " packages = set() @@ -1352,7 +1350,7 @@ def spm(options): if len(options) < 2: return 0 - import portageTools + Spm = Entropy.Spm() options = options[1:] opts = [] @@ -1372,7 +1370,7 @@ def spm(options): return 0 categories = list(set(options[1:])) categories.sort() - packages = portageTools.getAvailablePackages(categories) + packages = Spm.get_available_packages(categories) packages = list(packages) packages.sort() if do_list: