68835280e4
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1149 cd1c1023-2f26-0410-ae45-c471fc1f0318
6683 lines
285 KiB
Python
6683 lines
285 KiB
Python
#!/usr/bin/python
|
|
'''
|
|
# DESCRIPTION:
|
|
# Entropy Object Oriented Interface
|
|
|
|
Copyright (C) 2007-2008 Fabio Erculiani
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
'''
|
|
|
|
import shutil
|
|
import commands
|
|
import urllib2
|
|
import socket
|
|
import random
|
|
import time
|
|
from entropyConstants import *
|
|
from outputTools import *
|
|
import exceptionTools
|
|
global _garbage_cycle
|
|
_garbage_cycle = 0
|
|
|
|
class matchContainer:
|
|
def __init__(self):
|
|
self.data = set()
|
|
|
|
def inside(self, match):
|
|
if match in self.data:
|
|
return True
|
|
return False
|
|
|
|
def add(self, match):
|
|
self.data.add(match)
|
|
|
|
def clear(self):
|
|
self.data.clear()
|
|
|
|
'''
|
|
Main Entropy (client side) package management class
|
|
'''
|
|
class EquoInterface(TextInterface):
|
|
|
|
'''
|
|
@input indexing(bool): enable/disable database tables indexing
|
|
@input noclientdb(int/bool): 0 (or False): normal operation, every check on the client db will be done
|
|
1 (or True): openClientDatabase won't raise an exception if client database does not exist
|
|
2: client database won't be opened at all
|
|
@input xcache(bool): enable/disable database caching
|
|
'''
|
|
def __init__(self, indexing = True, noclientdb = 0, xcache = True):
|
|
|
|
# Logging initialization
|
|
import logTools
|
|
self.equoLog = logTools.LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[Equo]")
|
|
|
|
import dumpTools
|
|
self.dumpTools = dumpTools
|
|
import databaseTools
|
|
self.databaseTools = databaseTools
|
|
import entropyTools
|
|
self.entropyTools = entropyTools
|
|
import gc
|
|
self.gcTool = gc
|
|
self.urlFetcher = urlFetcher # in this way, can be reimplemented (so you can override updateProgress)
|
|
self.progress = None # supporting external updateProgress stuff, you can point self.progress to your progress bar
|
|
# and reimplement updateProgress
|
|
self.FtpInterface = FtpInterface # for convenience
|
|
self.indexing = indexing
|
|
self.noclientdb = False
|
|
self.openclientdb = True
|
|
if noclientdb in (False,0):
|
|
self.noclientdb = False
|
|
elif noclientdb in (True,1):
|
|
self.noclientdb = True
|
|
elif noclientdb == 2:
|
|
self.noclientdb = True
|
|
self.openclientdb = False
|
|
self.xcache = xcache
|
|
if self.openclientdb:
|
|
self.openClientDatabase()
|
|
self.FileUpdates = self.__FileUpdates()
|
|
self.repoDbCache = {}
|
|
|
|
# security interface
|
|
self.Security = SecurityInterface(self)
|
|
|
|
def switchChroot(self, chroot = ""):
|
|
# clean caches
|
|
self.purge_cache()
|
|
const_resetCache()
|
|
if chroot.endswith("/"):
|
|
chroot = chroot[:-1]
|
|
etpSys['rootdir'] = chroot
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
initConfig_clientConstants()
|
|
|
|
def reopenClientDbconn(self):
|
|
self.clientDbconn.closeDB()
|
|
self.openClientDatabase()
|
|
|
|
def closeAllRepositoryDatabases(self):
|
|
for item in self.repoDbCache:
|
|
self.repoDbCache[item].closeDB()
|
|
self.repoDbCache.clear()
|
|
|
|
def openClientDatabase(self):
|
|
self.clientDbconn = self.databaseTools.openClientDatabase(indexing = self.indexing, generate = self.noclientdb, xcache = self.xcache)
|
|
return self.clientDbconn # just for reference
|
|
|
|
def clientDatabaseSanityCheck(self):
|
|
self.updateProgress(darkred("Sanity Check: system database"), importance = 2, type = "warning")
|
|
idpkgs = self.clientDbconn.listAllIdpackages()
|
|
length = len(idpkgs)
|
|
count = 0
|
|
errors = False
|
|
for x in idpkgs:
|
|
count += 1
|
|
self.updateProgress(
|
|
darkgreen("Scanning..."),
|
|
importance = 0,
|
|
type = "info",
|
|
back = True,
|
|
count = (count,length),
|
|
percent = True
|
|
)
|
|
try:
|
|
self.clientDbconn.getPackageData(x)
|
|
except Exception ,e:
|
|
errors = True
|
|
self.updateProgress(
|
|
darkred("Errors on idpackage %s, exception: %s, error: %s") % (str(x), str(Exception),str(e)),
|
|
importance = 0,
|
|
type = "warning"
|
|
)
|
|
|
|
if not errors:
|
|
self.updateProgress(darkred("Sanity Check: %s") % (bold("PASSED"),), importance = 2, type = "warning")
|
|
return 0
|
|
else:
|
|
self.updateProgress(darkred("Sanity Check: %s") % (bold("CORRUPUTED"),), importance = 2, type = "warning")
|
|
return -1
|
|
|
|
def openRepositoryDatabase(self, repoid):
|
|
if not self.repoDbCache.has_key((repoid,etpConst['systemroot'])):
|
|
dbconn = self.loadRepositoryDatabase(repoid, xcache = self.xcache, indexing = self.indexing)
|
|
self.repoDbCache[(repoid,etpConst['systemroot'])] = dbconn
|
|
return dbconn
|
|
else:
|
|
return self.repoDbCache.get((repoid,etpConst['systemroot']))
|
|
|
|
'''
|
|
@description: open the repository database
|
|
@input repositoryName: name of the client database
|
|
@input xcache: loads on-disk cache
|
|
@input indexing: indexes SQL tables
|
|
@output: database class instance
|
|
NOTE: DO NOT USE THIS DIRECTLY, BUT USE EquoInterface.openRepositoryDatabase
|
|
'''
|
|
def loadRepositoryDatabase(self, repositoryName, xcache = True, indexing = True):
|
|
dbfile = etpRepositories[repositoryName]['dbpath']+"/"+etpConst['etpdatabasefile']
|
|
if not os.path.isfile(dbfile):
|
|
if repositoryName not in repo_error_messages_cache:
|
|
self.updateProgress(darkred("Repository %s hasn't been downloaded yet !!!") % (repositoryName,), importance = 2, type = "warning")
|
|
repo_error_messages_cache.add(repositoryName)
|
|
raise exceptionTools.RepositoryError("RepositoryError: repository %s hasn't been downloaded yet." % (repositoryName,))
|
|
conn = self.databaseTools.etpDatabase(readOnly = True, dbFile = dbfile, clientDatabase = True, dbname = etpConst['dbnamerepoprefix']+repositoryName, xcache = xcache, indexing = indexing)
|
|
# initialize CONFIG_PROTECT
|
|
if (etpRepositories[repositoryName]['configprotect'] == None) or \
|
|
(etpRepositories[repositoryName]['configprotectmask'] == None):
|
|
|
|
etpRepositories[repositoryName]['configprotect'] = conn.listConfigProtectDirectories()
|
|
etpRepositories[repositoryName]['configprotectmask'] = conn.listConfigProtectDirectories(mask = True)
|
|
etpRepositories[repositoryName]['configprotect'] = [etpConst['systemroot']+x for x in etpRepositories[repositoryName]['configprotect']]
|
|
etpRepositories[repositoryName]['configprotectmask'] = [etpConst['systemroot']+x for x in etpRepositories[repositoryName]['configprotectmask']]
|
|
|
|
etpRepositories[repositoryName]['configprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpRepositories[repositoryName]['configprotect']]
|
|
etpRepositories[repositoryName]['configprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpRepositories[repositoryName]['configprotectmask']]
|
|
if not etpConst['treeupdatescalled'] and (etpConst['uid'] == 0) and (not repositoryName.endswith(".tbz2")):
|
|
conn.clientUpdatePackagesData()
|
|
return conn
|
|
|
|
def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False):
|
|
if xcache == None:
|
|
xcache = self.xcache
|
|
dbconn = self.databaseTools.openGenericDatabase(dbfile,
|
|
dbname = dbname,
|
|
xcache = xcache,
|
|
indexing = self.indexing,
|
|
readOnly = readOnly
|
|
)
|
|
return dbconn
|
|
|
|
def listAllAvailableBranches(self):
|
|
branches = set()
|
|
for repo in etpRepositories:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
branches.update(dbconn.listAllBranches())
|
|
return branches
|
|
|
|
|
|
'''
|
|
Cache stuff :: begin
|
|
'''
|
|
def purge_cache(self, showProgress = True):
|
|
const_resetCache()
|
|
dumpdir = etpConst['dumpstoragedir']
|
|
if not dumpdir.endswith("/"): dumpdir = dumpdir+"/"
|
|
for key in etpCache:
|
|
cachefile = dumpdir+etpCache[key]+"*.dmp"
|
|
if showProgress: self.updateProgress(darkred("Cleaning %s...") % (cachefile,), importance = 1, type = "warning", back = True)
|
|
try:
|
|
os.system("rm -f "+cachefile)
|
|
except:
|
|
pass
|
|
# reset dict cache
|
|
if showProgress: self.updateProgress(darkgreen("Cache is now empty."), importance = 2, type = "info")
|
|
|
|
def generate_cache(self, depcache = True, configcache = True):
|
|
# clean first of all
|
|
self.purge_cache()
|
|
if depcache:
|
|
self.do_depcache()
|
|
if configcache:
|
|
self.do_configcache()
|
|
|
|
def do_configcache(self):
|
|
self.updateProgress(darkred("Configuration files"), importance = 2, type = "warning")
|
|
self.updateProgress(red("Scanning hard disk"), importance = 1, type = "warning")
|
|
self.FileUpdates.scanfs(dcache = False)
|
|
self.updateProgress(darkred("Cache generation complete."), importance = 2, type = "info")
|
|
|
|
def do_depcache(self):
|
|
self.updateProgress(darkred("Dependencies"), importance = 2, type = "warning")
|
|
self.updateProgress(darkred("Scanning repositories"), importance = 2, type = "warning")
|
|
names = set()
|
|
keys = set()
|
|
depends = set()
|
|
atoms = set()
|
|
for reponame in etpRepositoriesOrder:
|
|
self.updateProgress(darkgreen("Scanning %s" % (etpRepositories[reponame]['description'],)) , importance = 1, type = "info", back = True)
|
|
# get all packages keys
|
|
try:
|
|
dbconn = self.openRepositoryDatabase(reponame)
|
|
except exceptionTools.RepositoryError:
|
|
self.updateProgress(darkred("Cannot download/access: %s" % (etpRepositories[reponame]['description'],)) , importance = 2, type = "error")
|
|
continue
|
|
pkgdata = dbconn.listAllPackages()
|
|
pkgdata = set(pkgdata)
|
|
for info in pkgdata:
|
|
key = self.entropyTools.dep_getkey(info[0])
|
|
keys.add(key)
|
|
names.add(key.split("/")[1])
|
|
atoms.add(info[0])
|
|
# dependencies
|
|
pkgdata = dbconn.listAllDependencies()
|
|
for info in pkgdata:
|
|
depends.add(info[1])
|
|
|
|
self.updateProgress(darkgreen("Resolving metadata"), importance = 1, type = "warning")
|
|
atomMatchCache.clear()
|
|
maxlen = len(names)
|
|
cnt = 0
|
|
for name in names:
|
|
cnt += 1
|
|
self.updateProgress(darkgreen("Resolving name: %s") % (
|
|
name
|
|
), importance = 0, type = "info", back = True, count = (cnt, maxlen) )
|
|
self.atomMatch(name)
|
|
maxlen = len(keys)
|
|
cnt = 0
|
|
for key in keys:
|
|
cnt += 1
|
|
self.updateProgress(darkgreen("Resolving key: %s") % (
|
|
key
|
|
), importance = 0, type = "info", back = True, count = (cnt, maxlen) )
|
|
self.atomMatch(key)
|
|
maxlen = len(atoms)
|
|
cnt = 0
|
|
for atom in atoms:
|
|
cnt += 1
|
|
self.updateProgress(darkgreen("Resolving atom: %s") % (
|
|
atom
|
|
), importance = 0, type = "info", back = True, count = (cnt, maxlen) )
|
|
self.atomMatch(atom)
|
|
maxlen = len(depends)
|
|
cnt = 0
|
|
for depend in depends:
|
|
cnt += 1
|
|
self.updateProgress(darkgreen("Resolving dependency: %s") % (
|
|
depend
|
|
), importance = 0, type = "info", back = True, count = (cnt, maxlen) )
|
|
self.atomMatch(depend)
|
|
|
|
# we can barely ignore any exception from here
|
|
# especially cases where client db does not exist
|
|
try:
|
|
self.calculate_world_updates()
|
|
except:
|
|
pass
|
|
|
|
self.updateProgress(darkred("Dependencies cache filled."), importance = 2, type = "warning")
|
|
self.save_cache()
|
|
|
|
def load_cache(self, showProgress = True):
|
|
|
|
if (etpConst['uid'] != 0) or (not self.xcache): # don't load cache as user
|
|
return
|
|
|
|
try:
|
|
if showProgress: self.updateProgress(blue("Loading On-Disk Cache..."), importance = 2, type = "info")
|
|
except:
|
|
pass
|
|
|
|
# atomMatch
|
|
try:
|
|
mycache = self.dumpTools.loadobj(etpCache['atomMatch'])
|
|
if isinstance(mycache, dict):
|
|
atomMatchCache.clear()
|
|
atomMatchCache.update(mycache)
|
|
del mycache
|
|
except:
|
|
atomMatchCache.clear()
|
|
self.dumpTools.dumpobj(etpCache['atomMatch'],{})
|
|
|
|
# removal dependencies
|
|
try:
|
|
mycache = self.dumpTools.loadobj(etpCache['generateDependsTree'])
|
|
if isinstance(mycache, dict):
|
|
generateDependsTreeCache.clear()
|
|
generateDependsTreeCache.update(mycache)
|
|
del mycache
|
|
except:
|
|
generateDependsTreeCache.clear()
|
|
self.dumpTools.dumpobj(etpCache['generateDependsTree'],{})
|
|
|
|
# check_package_update cache
|
|
try:
|
|
mycache = self.dumpTools.loadobj(etpCache['check_package_update'])
|
|
if isinstance(mycache, dict):
|
|
check_package_update_cache.clear()
|
|
check_package_update_cache.update(mycache)
|
|
del mycache
|
|
except:
|
|
check_package_update_cache.clear()
|
|
self.dumpTools.dumpobj(etpCache['check_package_update'],{})
|
|
|
|
def save_cache(self):
|
|
|
|
if (etpConst['uid'] != 0): # don't save cache as user
|
|
return
|
|
|
|
self.dumpTools.dumpobj(etpCache['check_package_update'],check_package_update_cache)
|
|
|
|
self.dumpTools.dumpobj(etpCache['atomMatch'],atomMatchCache)
|
|
if os.path.isfile(etpConst['dumpstoragedir']+"/"+etpCache['atomMatch']+".dmp"):
|
|
if os.stat(etpConst['dumpstoragedir']+"/"+etpCache['atomMatch']+".dmp")[6] > etpCacheSizes['atomMatch']:
|
|
# clean cache
|
|
self.dumpTools.dumpobj(etpCache['atomMatch'],{})
|
|
self.dumpTools.dumpobj(etpCache['generateDependsTree'],generateDependsTreeCache)
|
|
if os.path.isfile(etpConst['dumpstoragedir']+"/"+etpCache['generateDependsTree']+".dmp"):
|
|
if os.stat(etpConst['dumpstoragedir']+"/"+etpCache['generateDependsTree']+".dmp")[6] > etpCacheSizes['generateDependsTree']:
|
|
# clean cache
|
|
self.dumpTools.dumpobj(etpCache['generateDependsTree'],{})
|
|
for dbinfo in dbCacheStore:
|
|
self.dumpTools.dumpobj(dbinfo,dbCacheStore[dbinfo])
|
|
# check size
|
|
if os.path.isfile(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp"):
|
|
if dbinfo.startswith(etpCache['dbMatch']):
|
|
if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbMatch']:
|
|
# clean cache
|
|
self.dumpTools.dumpobj(dbinfo,{})
|
|
elif dbinfo.startswith(etpCache['dbInfo']):
|
|
if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbInfo']:
|
|
# clean cache
|
|
self.dumpTools.dumpobj(dbinfo,{})
|
|
elif dbinfo.startswith(etpCache['dbSearch']):
|
|
if os.stat(etpConst['dumpstoragedir']+"/"+dbinfo+".dmp")[6] > etpCacheSizes['dbSearch']:
|
|
# clean cache
|
|
self.dumpTools.dumpobj(dbinfo,{})
|
|
|
|
'''
|
|
Cache stuff :: end
|
|
'''
|
|
|
|
def dependencies_test(self, dbconn = None):
|
|
|
|
if dbconn == None:
|
|
dbconn = self.clientDbconn
|
|
# get all the installed packages
|
|
installedPackages = dbconn.listAllIdpackages()
|
|
|
|
depsNotSatisfied = {}
|
|
# now look
|
|
length = str((len(installedPackages)))
|
|
count = 0
|
|
for xidpackage in installedPackages:
|
|
count += 1
|
|
atom = dbconn.retrieveAtom(xidpackage)
|
|
self.updateProgress(
|
|
darkgreen(" Checking ")+bold(atom),
|
|
importance = 0,
|
|
type = "info",
|
|
back = True,
|
|
count = (count,length),
|
|
header = darkred(" @@ ")
|
|
)
|
|
|
|
xdeps = dbconn.retrieveDependenciesList(xidpackage)
|
|
needed_deps = set()
|
|
for xdep in xdeps:
|
|
if xdep[0] == "!": # filter conflicts
|
|
continue
|
|
xmatch = dbconn.atomMatch(xdep)
|
|
if xmatch[0] == -1:
|
|
needed_deps.add(xdep)
|
|
|
|
if needed_deps:
|
|
depsNotSatisfied[xidpackage] = set()
|
|
depsNotSatisfied[xidpackage].update(needed_deps)
|
|
|
|
depsNotMatched = set()
|
|
if (depsNotSatisfied):
|
|
for xidpackage in depsNotSatisfied:
|
|
for dep in depsNotSatisfied[xidpackage]:
|
|
match = dbconn.atomMatch(dep)
|
|
if match[0] == -1: # ????
|
|
depsNotMatched.add(dep)
|
|
continue
|
|
|
|
return depsNotMatched
|
|
|
|
def find_belonging_dependency(self, matched_atoms):
|
|
crying_atoms = set()
|
|
for atom in matched_atoms:
|
|
for repo in etpRepositories:
|
|
rdbconn = self.openRepositoryDatabase(repo)
|
|
riddep = rdbconn.searchDependency(atom)
|
|
if riddep != -1:
|
|
ridpackages = rdbconn.searchIdpackageFromIddependency(riddep)
|
|
for i in ridpackages:
|
|
iatom = rdbconn.retrieveAtom(i)
|
|
crying_atoms.add((iatom,repo))
|
|
return crying_atoms
|
|
|
|
def libraries_test(self, dbconn = None, reagent = False):
|
|
|
|
if dbconn == None:
|
|
dbconn = self.clientDbconn
|
|
|
|
self.updateProgress(
|
|
blue("Dependencies test"),
|
|
importance = 2,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
# run ldconfig first
|
|
os.system("ldconfig -r "+myroot+" &> /dev/null")
|
|
# open /etc/ld.so.conf
|
|
if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"):
|
|
self.updateProgress(
|
|
blue("Cannot find ")+red(etpConst['systemroot']+"/etc/ld.so.conf"),
|
|
importance = 1,
|
|
type = "error",
|
|
header = red(" @@ ")
|
|
)
|
|
return (),(),-1
|
|
|
|
ldpaths = self.entropyTools.collectLinkerPaths()
|
|
|
|
executables = set()
|
|
total = len(ldpaths)
|
|
count = 0
|
|
for ldpath in ldpaths:
|
|
count += 1
|
|
self.updateProgress(
|
|
blue("Tree: ")+red(etpConst['systemroot']+ldpath),
|
|
importance = 0,
|
|
type = "info",
|
|
count = (count,total),
|
|
back = True,
|
|
percent = True,
|
|
header = " "
|
|
)
|
|
ldpath = ldpath.encode(sys.getfilesystemencoding())
|
|
for currentdir,subdirs,files in os.walk(etpConst['systemroot']+ldpath):
|
|
for item in files:
|
|
filepath = currentdir+"/"+item
|
|
if os.access(filepath,os.X_OK):
|
|
executables.add(filepath[len(etpConst['systemroot']):])
|
|
|
|
self.updateProgress(
|
|
blue("Collecting broken executables"),
|
|
importance = 2,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
self.updateProgress(
|
|
red("Attention: ")+blue("don't worry about libraries that are shown here but not later."),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
brokenlibs = set()
|
|
brokenexecs = {}
|
|
total = len(executables)
|
|
count = 0
|
|
for executable in executables:
|
|
count += 1
|
|
self.updateProgress(
|
|
red(etpConst['systemroot']+executable),
|
|
importance = 0,
|
|
type = "info",
|
|
count = (count,total),
|
|
back = True,
|
|
percent = True,
|
|
header = " "
|
|
)
|
|
if not etpConst['systemroot']:
|
|
stdin, stdouterr = os.popen4("ldd "+executable)
|
|
else:
|
|
if not os.access(etpConst['systemroot']+"/bin/sh",os.X_OK):
|
|
raise exceptionTools.FileNotFound("FileNotFound: /bin/sh not found.")
|
|
stdin, stdouterr = os.popen4("echo 'ldd "+executable+"' | chroot "+etpConst['systemroot'])
|
|
output = stdouterr.readlines()
|
|
if '\n'.join(output).find("not found") != -1:
|
|
# investigate
|
|
mylibs = set()
|
|
for row in output:
|
|
if row.find("not found") != -1:
|
|
try:
|
|
row = row.strip().split("=>")[0].strip()
|
|
mylibs.add(row)
|
|
except:
|
|
continue
|
|
if mylibs:
|
|
alllibs = blue(' :: ').join(list(mylibs))
|
|
self.updateProgress(
|
|
red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]",
|
|
importance = 1,
|
|
type = "info",
|
|
percent = True,
|
|
count = (count,total),
|
|
header = " "
|
|
)
|
|
brokenlibs.update(mylibs)
|
|
brokenexecs[executable] = mylibs.copy()
|
|
del executables
|
|
|
|
self.updateProgress(
|
|
blue("Trying to match packages"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
|
|
packagesMatched = set()
|
|
# now search packages that contain the found libs
|
|
|
|
# match libraries
|
|
for repoid in etpRepositoriesOrder:
|
|
self.updateProgress(
|
|
blue("Repository: ")+darkgreen(etpRepositories[repoid]['description'])+" ["+red(repoid)+"]",
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
if reagent:
|
|
rdbconn = dbconn
|
|
else:
|
|
rdbconn = self.openRepositoryDatabase(repoid)
|
|
libsfound = set()
|
|
for lib in brokenlibs:
|
|
packages = rdbconn.searchBelongs(file = "%"+lib, like = True, branch = etpConst['branch'])
|
|
if packages:
|
|
for idpackage in packages:
|
|
# retrieve content and really look if library is in ldpath
|
|
mycontent = rdbconn.retrieveContent(idpackage)
|
|
matching_libs = [x for x in mycontent if x.endswith(lib) and (os.path.dirname(x) in ldpaths)]
|
|
libsfound.add(lib)
|
|
if matching_libs:
|
|
packagesMatched.add((idpackage,repoid,lib))
|
|
brokenlibs.difference_update(libsfound)
|
|
|
|
return packagesMatched,brokenlibs,0
|
|
|
|
def move_to_branch(self, branch, pretend = False):
|
|
availbranches = self.listAllAvailableBranches()
|
|
if branch not in availbranches:
|
|
return 1
|
|
if pretend:
|
|
return 0
|
|
if branch != etpConst['branch']:
|
|
# update configuration
|
|
self.entropyTools.writeNewBranch(branch)
|
|
# reset treeupdatesactions
|
|
self.clientDbconn.resetTreeupdatesDigests()
|
|
# clean cache
|
|
self.purge_cache(showProgress = False)
|
|
self.load_cache(showProgress = False)
|
|
# reopen Client Database, this will make treeupdates to be re-read
|
|
self.reopenClientDbconn()
|
|
self.closeAllRepositoryDatabases()
|
|
return 0
|
|
|
|
# tell if a new equo release is available, returns True or False
|
|
def check_equo_updates(self):
|
|
found, match = self.check_package_update("app-admin/equo")
|
|
return found
|
|
|
|
def check_package_update(self, atom):
|
|
|
|
if check_package_update_cache.has_key(atom):
|
|
return check_package_update_cache[atom]
|
|
|
|
found = False
|
|
match = self.clientDbconn.atomMatch(atom)
|
|
matched = None
|
|
if match[0] != -1:
|
|
myatom = self.clientDbconn.retrieveAtom(match[0])
|
|
pkg_match = ">="+myatom
|
|
pkg_unsatisfied,x = self.filterSatisfiedDependencies([pkg_match])
|
|
del x
|
|
if pkg_unsatisfied:
|
|
found = True
|
|
del pkg_unsatisfied
|
|
matched = self.atomMatch(pkg_match)
|
|
del match
|
|
|
|
# cache
|
|
check_package_update_cache[atom] = (found,matched)
|
|
return found, matched
|
|
|
|
|
|
# @returns -1 if the file does not exist or contains bad data
|
|
# @returns int>0 if the file exists
|
|
def get_repository_revision(self, reponame):
|
|
if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
|
|
f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r")
|
|
try:
|
|
revision = int(f.readline().strip())
|
|
except:
|
|
revision = -1
|
|
f.close()
|
|
else:
|
|
revision = -1
|
|
return revision
|
|
|
|
# @returns -1 if the file does not exist
|
|
# @returns int>0 if the file exists
|
|
def get_repository_db_file_checksum(self, reponame):
|
|
if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
|
|
f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
|
|
try:
|
|
mhash = f.readline().strip().split()[0]
|
|
except:
|
|
mhash = "-1"
|
|
f.close()
|
|
else:
|
|
mhash = "-1"
|
|
return mhash
|
|
|
|
|
|
def fetch_repository_if_not_available(self, reponame):
|
|
if fetch_repository_if_not_available_cache.has_key(reponame):
|
|
return fetch_repository_if_not_available_cache.get(reponame)
|
|
# open database
|
|
rc = 0
|
|
dbfile = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasefile']
|
|
if not os.path.isfile(dbfile):
|
|
# sync
|
|
repoConn = self.Repositories(reponames = [reponame], noEquoCheck = True)
|
|
rc = repoConn.sync()
|
|
del repoConn
|
|
if os.path.isfile(dbfile):
|
|
rc = 0
|
|
fetch_repository_if_not_available_cache[reponame] = rc
|
|
return rc
|
|
|
|
'''
|
|
@description: matches the package that user chose, using dbconnection.atomMatch searching in all available repositories.
|
|
@input atom: user choosen package name
|
|
@output: the matched selection, list: [package id,repository name] | if nothing found, returns: ( -1,1 )
|
|
@ exit errors:
|
|
-1 => repository cannot be fetched online
|
|
'''
|
|
def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = ()):
|
|
|
|
if self.xcache:
|
|
cached = atomMatchCache.get(atom)
|
|
if cached:
|
|
try:
|
|
if (cached['matchSlot'] == matchSlot) and (cached['matchBranches'] == matchBranches) and (cached['etpRepositories'] == etpRepositories) and (cached['caseSensitive'] == caseSensitive) and (cached['etpRepositoriesOrder'] == etpRepositoriesOrder):
|
|
return cached['result']
|
|
except KeyError:
|
|
pass
|
|
|
|
repoResults = {}
|
|
for repo in etpRepositoriesOrder:
|
|
|
|
# check if repo exists
|
|
if not repo.endswith(".tbz2"):
|
|
fetch = self.fetch_repository_if_not_available(repo)
|
|
if fetch != 0:
|
|
continue # cannot fetch repo, excluding
|
|
|
|
# open database
|
|
try:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
except exceptionTools.RepositoryError, e:
|
|
continue # repo not available
|
|
|
|
# search
|
|
query = dbconn.atomMatch(atom, caseSensitive = caseSensitive, matchSlot = matchSlot, matchBranches = matchBranches)
|
|
if query[1] == 0:
|
|
# package found, add to our dictionary
|
|
repoResults[repo] = query[0]
|
|
|
|
# handle repoResults
|
|
packageInformation = {}
|
|
|
|
# nothing found
|
|
if not repoResults:
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = -1,1
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return -1,1
|
|
|
|
elif len(repoResults) == 1:
|
|
# one result found
|
|
for repo in repoResults:
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[repo],repo
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[repo],repo
|
|
|
|
elif len(repoResults) > 1:
|
|
# we have to decide which version should be taken
|
|
|
|
# .tbz2 repos have always the precedence, so if we find them, we should second what user wants, installing his tbz2
|
|
tbz2repos = [x for x in repoResults if x.endswith(".tbz2")]
|
|
if tbz2repos:
|
|
del tbz2repos
|
|
newrepos = repoResults.copy()
|
|
for x in newrepos:
|
|
if not x.endswith(".tbz2"):
|
|
del repoResults[x]
|
|
|
|
# get package information for all the entries
|
|
for repo in repoResults:
|
|
|
|
# open database
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
# search
|
|
packageInformation[repo] = {}
|
|
packageInformation[repo]['version'] = dbconn.retrieveVersion(repoResults[repo])
|
|
packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(repoResults[repo])
|
|
packageInformation[repo]['revision'] = dbconn.retrieveRevision(repoResults[repo])
|
|
|
|
versions = []
|
|
repoNames = []
|
|
# compare versions
|
|
for repo in packageInformation:
|
|
repoNames.append(repo)
|
|
versions.append(packageInformation[repo]['version'])
|
|
|
|
# found duplicates, this mean that we have to look at the revision and then, at the version tag
|
|
# if all this shait fails, get the uppest repository
|
|
# if no duplicates, we're done
|
|
filteredVersions = self.entropyTools.filterDuplicatedEntries(versions)
|
|
if (len(versions) > len(filteredVersions)):
|
|
# there are duplicated results, fetch them
|
|
# get the newerVersion
|
|
newerVersion = self.entropyTools.getNewerVersion(versions)
|
|
newerVersion = newerVersion[0]
|
|
# is newerVersion, the duplicated one?
|
|
duplicatedEntries = self.entropyTools.extractDuplicatedEntries(versions)
|
|
needFiltering = False
|
|
if newerVersion in duplicatedEntries:
|
|
needFiltering = True
|
|
|
|
if (needFiltering):
|
|
# we have to decide which one is good
|
|
# we have newerVersion
|
|
conflictingEntries = {}
|
|
for repo in packageInformation:
|
|
if packageInformation[repo]['version'] == newerVersion:
|
|
conflictingEntries[repo] = {}
|
|
conflictingEntries[repo]['versiontag'] = packageInformation[repo]['versiontag']
|
|
conflictingEntries[repo]['revision'] = packageInformation[repo]['revision']
|
|
|
|
# at this point compare tags
|
|
tags = [conflictingEntries[x]['versiontag'] for x in conflictingEntries]
|
|
newerTag = self.entropyTools.getNewerVersionTag(tags)
|
|
newerTag = newerTag[0]
|
|
|
|
# is the chosen tag duplicated?
|
|
duplicatedTags = self.entropyTools.extractDuplicatedEntries(tags)
|
|
needFiltering = False
|
|
if newerTag in duplicatedTags:
|
|
needFiltering = True
|
|
|
|
if (needFiltering):
|
|
# yes, it is. we need to compare revisions
|
|
conflictingTags = {}
|
|
for repo in conflictingEntries:
|
|
if conflictingEntries[repo]['versiontag'] == newerTag:
|
|
conflictingTags[repo] = {}
|
|
conflictingTags[repo]['revision'] = conflictingEntries[repo]['revision']
|
|
|
|
revisions = []
|
|
for repo in conflictingTags:
|
|
revisions.append(str(conflictingTags[repo]['revision']))
|
|
newerRevision = max(revisions)
|
|
duplicatedRevisions = self.entropyTools.extractDuplicatedEntries(revisions)
|
|
needFiltering = False
|
|
if newerRevision in duplicatedRevisions:
|
|
needFiltering = True
|
|
|
|
if (needFiltering):
|
|
# ok, we must get the repository with the biggest priority
|
|
for repository in etpRepositoriesOrder:
|
|
if repository in conflictingTags:
|
|
# found it, WE ARE DOOONE!
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[repository],repository
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[repository],repository
|
|
else:
|
|
# we are done!!!
|
|
reponame = ''
|
|
for x in conflictingTags:
|
|
if str(conflictingTags[x]['revision']) == str(newerRevision):
|
|
reponame = x
|
|
break
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[reponame],reponame
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[reponame],reponame
|
|
else:
|
|
# we're finally done
|
|
reponame = ''
|
|
for x in conflictingEntries:
|
|
if conflictingEntries[x]['versiontag'] == newerTag:
|
|
reponame = x
|
|
break
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[reponame],reponame
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[reponame],reponame
|
|
else:
|
|
# we are fine, the newerVersion is not one of the duplicated ones
|
|
reponame = ''
|
|
for x in packageInformation:
|
|
if packageInformation[x]['version'] == newerVersion:
|
|
reponame = x
|
|
break
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[reponame],reponame
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[reponame],reponame
|
|
else:
|
|
# yeah, we're done, just return the info
|
|
newerVersion = self.entropyTools.getNewerVersion(versions)
|
|
# get the repository name
|
|
newerVersion = newerVersion[0]
|
|
reponame = ''
|
|
for x in packageInformation:
|
|
if packageInformation[x]['version'] == newerVersion:
|
|
reponame = x
|
|
break
|
|
atomMatchCache[atom] = {}
|
|
atomMatchCache[atom]['result'] = repoResults[reponame],reponame
|
|
atomMatchCache[atom]['matchSlot'] = matchSlot
|
|
atomMatchCache[atom]['matchBranches'] = matchBranches
|
|
atomMatchCache[atom]['caseSensitive'] = caseSensitive
|
|
atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy()
|
|
atomMatchCache[atom]['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
return repoResults[reponame],reponame
|
|
|
|
def __repository_move_clear_cache(self, repoid):
|
|
# clean world_available cache
|
|
self.dumpTools.dumpobj(etpCache['world_available'], {})
|
|
# clean world_update cache
|
|
self.dumpTools.dumpobj(etpCache['world_update'], {})
|
|
# clean check_update_package_cache
|
|
check_package_update_cache.clear()
|
|
self.dumpTools.dumpobj(etpCache['check_package_update'],{})
|
|
# clear atomMatchCache
|
|
atomMatchCache.clear()
|
|
self.dumpTools.dumpobj(etpCache['atomMatch'],{})
|
|
dbCacheStore[etpCache['dbMatch']+etpConst['dbnamerepoprefix']+repoid] = {}
|
|
dbCacheStore[etpCache['dbSearch']+etpConst['dbnamerepoprefix']+repoid] = {}
|
|
dbCacheStore[etpCache['dbInfo']+etpConst['dbnamerepoprefix']+repoid] = {}
|
|
self.dumpTools.dumpobj(etpCache['dbMatch']+etpConst['dbnamerepoprefix']+repoid,{})
|
|
self.dumpTools.dumpobj(etpCache['dbSearch']+etpConst['dbnamerepoprefix']+repoid,{})
|
|
self.dumpTools.dumpobj(etpCache['dbInfo']+etpConst['dbnamerepoprefix']+repoid,{})
|
|
|
|
|
|
def addRepository(self, repodata):
|
|
# update etpRepositories
|
|
try:
|
|
etpRepositories[repodata['repoid']] = {}
|
|
etpRepositories[repodata['repoid']]['description'] = repodata['description']
|
|
etpRepositories[repodata['repoid']]['configprotect'] = None
|
|
etpRepositories[repodata['repoid']]['configprotectmask'] = None
|
|
except KeyError:
|
|
raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted")
|
|
|
|
if repodata['repoid'].endswith(".tbz2"): # dynamic repository
|
|
try:
|
|
etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:]
|
|
etpRepositories[repodata['repoid']]['smartpackage'] = repodata['smartpackage']
|
|
etpRepositories[repodata['repoid']]['dbpath'] = repodata['dbpath']
|
|
etpRepositories[repodata['repoid']]['pkgpath'] = repodata['pkgpath']
|
|
except KeyError:
|
|
raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted")
|
|
# put at top priority, shift others
|
|
etpRepositoriesOrder.insert(0,repodata['repoid'])
|
|
else:
|
|
# XXX it's boring to keep this in sync with entropyConstants stuff, solutions?
|
|
etpRepositories[repodata['repoid']]['packages'] = [x+"/"+etpConst['product'] for x in repodata['packages']]
|
|
etpRepositories[repodata['repoid']]['database'] = repodata['database'] + "/" + etpConst['product'] + "/database/" + etpConst['currentarch']
|
|
etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat']
|
|
etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch']
|
|
# set dbrevision
|
|
myrev = self.get_repository_revision(repodata['repoid'])
|
|
if myrev == -1:
|
|
myrev = 0
|
|
etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev)
|
|
if repodata.has_key("position"):
|
|
etpRepositoriesOrder.insert(repodata['position'],repodata['repoid'])
|
|
else:
|
|
etpRepositoriesOrder.append(repodata['repoid'])
|
|
self.__repository_move_clear_cache(repodata['repoid'])
|
|
# save new etpRepositories to file
|
|
self.entropyTools.saveRepositorySettings(repodata)
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
|
|
def removeRepository(self, repoid):
|
|
|
|
done = False
|
|
try:
|
|
del etpRepositories[repoid]
|
|
done = True
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
del etpRepositoriesExcluded[repoid]
|
|
done = True
|
|
except:
|
|
pass
|
|
|
|
if done:
|
|
try:
|
|
etpRepositoriesOrder.remove(repoid)
|
|
except:
|
|
pass
|
|
# it's not vital to reset etpRepositoriesOrder counters
|
|
|
|
self.__repository_move_clear_cache(repoid)
|
|
# save new etpRepositories to file
|
|
repodata = {}
|
|
repodata['repoid'] = repoid
|
|
self.entropyTools.saveRepositorySettings(repodata, remove = True)
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
|
|
def shiftRepository(self, repoid, toidx):
|
|
# update etpRepositoriesOrder
|
|
etpRepositoriesOrder.remove(repoid)
|
|
etpRepositoriesOrder.insert(toidx,repoid)
|
|
self.entropyTools.writeOrderedRepositoriesEntries()
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
self.__repository_move_clear_cache(repoid)
|
|
|
|
def enableRepository(self, repoid):
|
|
self.__repository_move_clear_cache(repoid)
|
|
# save new etpRepositories to file
|
|
repodata = {}
|
|
repodata['repoid'] = repoid
|
|
self.entropyTools.saveRepositorySettings(repodata, enable = True)
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
|
|
def disableRepository(self, repoid):
|
|
# update etpRepositories
|
|
done = False
|
|
try:
|
|
del etpRepositories[repoid]
|
|
done = True
|
|
except:
|
|
pass
|
|
|
|
if done:
|
|
try:
|
|
etpRepositoriesOrder.remove(repoid)
|
|
except:
|
|
pass
|
|
# it's not vital to reset etpRepositoriesOrder counters
|
|
|
|
self.__repository_move_clear_cache(repoid)
|
|
# save new etpRepositories to file
|
|
repodata = {}
|
|
repodata['repoid'] = repoid
|
|
self.entropyTools.saveRepositorySettings(repodata, disable = True)
|
|
initConfig_entropyConstants(etpSys['rootdir'])
|
|
|
|
|
|
'''
|
|
@description: filter the already installed dependencies
|
|
@input dependencies: list of dependencies to check
|
|
@output: filtered list, aka the needed ones and the ones satisfied
|
|
'''
|
|
def filterSatisfiedDependencies(self, dependencies, deep_deps = False):
|
|
|
|
unsatisfiedDeps = set()
|
|
satisfiedDeps = set()
|
|
|
|
for dependency in dependencies:
|
|
|
|
depsatisfied = set()
|
|
depunsatisfied = set()
|
|
|
|
''' caching '''
|
|
cached = filterSatisfiedDependenciesCache.get(dependency)
|
|
if cached:
|
|
if (cached['deep_deps'] == deep_deps):
|
|
unsatisfiedDeps.update(cached['depunsatisfied'])
|
|
satisfiedDeps.update(cached['depsatisfied'])
|
|
continue
|
|
|
|
### conflict
|
|
if dependency[0] == "!":
|
|
testdep = dependency[1:]
|
|
xmatch = self.clientDbconn.atomMatch(testdep)
|
|
if xmatch[0] != -1:
|
|
unsatisfiedDeps.add(dependency)
|
|
else:
|
|
satisfiedDeps.add(dependency)
|
|
continue
|
|
|
|
repoMatch = self.atomMatch(dependency)
|
|
if repoMatch[0] != -1:
|
|
dbconn = self.openRepositoryDatabase(repoMatch[1])
|
|
repo_pkgver = dbconn.retrieveVersion(repoMatch[0])
|
|
repo_pkgtag = dbconn.retrieveVersionTag(repoMatch[0])
|
|
repo_pkgrev = dbconn.retrieveRevision(repoMatch[0])
|
|
else:
|
|
# dependency does not exist in our database
|
|
unsatisfiedDeps.add(dependency)
|
|
continue
|
|
|
|
clientMatch = self.clientDbconn.atomMatch(dependency)
|
|
if clientMatch[0] != -1:
|
|
|
|
installedVer = self.clientDbconn.retrieveVersion(clientMatch[0])
|
|
installedTag = self.clientDbconn.retrieveVersionTag(clientMatch[0])
|
|
installedRev = self.clientDbconn.retrieveRevision(clientMatch[0])
|
|
if installedRev == 9999: # any revision is fine
|
|
repo_pkgrev = 9999
|
|
|
|
if (deep_deps):
|
|
vcmp = self.entropyTools.entropyCompareVersions((repo_pkgver,repo_pkgtag,repo_pkgrev),(installedVer,installedTag,installedRev))
|
|
if vcmp != 0:
|
|
filterSatisfiedDependenciesCmpResults[dependency] = vcmp
|
|
depunsatisfied.add(dependency)
|
|
else:
|
|
# check if needed is the same?
|
|
depsatisfied.add(dependency)
|
|
else:
|
|
depsatisfied.add(dependency)
|
|
else:
|
|
# not the same version installed
|
|
filterSatisfiedDependenciesCmpResults[dependency] = 10
|
|
depunsatisfied.add(dependency)
|
|
|
|
if depsatisfied:
|
|
# check if it's really satisfied by looking at needed
|
|
installedNeeded = self.clientDbconn.retrieveNeeded(clientMatch[0])
|
|
repo_needed = dbconn.retrieveNeeded(repoMatch[0])
|
|
if installedNeeded != repo_needed:
|
|
depunsatisfied.update(depsatisfied)
|
|
depsatisfied.clear()
|
|
else:
|
|
# also check useflags
|
|
installedUseflags = self.clientDbconn.retrieveUseflags(clientMatch[0])
|
|
repo_useflags = dbconn.retrieveUseflags(repoMatch[0])
|
|
if installedUseflags != repo_useflags:
|
|
depunsatisfied.update(depsatisfied)
|
|
depsatisfied.clear()
|
|
|
|
unsatisfiedDeps.update(depunsatisfied)
|
|
satisfiedDeps.update(depsatisfied)
|
|
|
|
''' caching '''
|
|
filterSatisfiedDependenciesCache[dependency] = {}
|
|
filterSatisfiedDependenciesCache[dependency]['depunsatisfied'] = depunsatisfied
|
|
filterSatisfiedDependenciesCache[dependency]['depsatisfied'] = depsatisfied
|
|
filterSatisfiedDependenciesCache[dependency]['deep_deps'] = deep_deps
|
|
|
|
return unsatisfiedDeps, satisfiedDeps
|
|
|
|
|
|
'''
|
|
@description: generates a dependency tree using unsatisfied dependencies
|
|
@input package: atomInfo (idpackage,reponame)
|
|
@output: dependency tree dictionary, plus status code
|
|
'''
|
|
def generate_dependency_tree(self, atomInfo, empty_deps = False, deep_deps = False, matchfilter = None):
|
|
|
|
usefilter = False
|
|
if matchfilter != None:
|
|
usefilter = True
|
|
|
|
mydbconn = self.openRepositoryDatabase(atomInfo[1])
|
|
myatom = mydbconn.retrieveAtom(atomInfo[0])
|
|
|
|
# caches
|
|
treecache = set()
|
|
matchcache = set()
|
|
keyslotcache = set()
|
|
# special events
|
|
dependenciesNotFound = set()
|
|
conflicts = set()
|
|
|
|
mydep = (1,myatom)
|
|
mybuffer = self.entropyTools.lifobuffer()
|
|
deptree = set()
|
|
if usefilter:
|
|
if not matchfilter.inside(atomInfo):
|
|
deptree.add((1,atomInfo))
|
|
else:
|
|
deptree.add((1,atomInfo))
|
|
|
|
''' these ones ? not needed anymore?
|
|
#mybuffer.push((1,myatom))
|
|
#mytree.append((1,myatom))
|
|
'''
|
|
|
|
while mydep != None:
|
|
|
|
# already analyzed in this call
|
|
if mydep[1] in treecache:
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
|
|
# conflicts
|
|
if mydep[1][0] == "!":
|
|
xmatch = self.clientDbconn.atomMatch(mydep[1][1:])
|
|
if xmatch[0] != -1:
|
|
conflicts.add(xmatch[0])
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
|
|
# atom found?
|
|
match = self.atomMatch(mydep[1])
|
|
if match[0] == -1:
|
|
dependenciesNotFound.add(mydep[1])
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
|
|
# check if atom has been already pulled in
|
|
matchdb = self.openRepositoryDatabase(match[1])
|
|
matchatom = matchdb.retrieveAtom(match[0])
|
|
matchslot = matchdb.retrieveSlot(match[0]) # used later
|
|
if matchatom in treecache:
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
else:
|
|
treecache.add(matchatom)
|
|
|
|
treecache.add(mydep[1])
|
|
|
|
# check if key + slot has been already pulled in
|
|
key = self.entropyTools.dep_getkey(matchatom)
|
|
if (matchslot,key) in keyslotcache:
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
else:
|
|
keyslotcache.add((matchslot,key))
|
|
|
|
# already analyzed by the calling function
|
|
if usefilter:
|
|
if matchfilter.inside(match):
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
matchfilter.add(match)
|
|
|
|
# result already analyzed?
|
|
if match in matchcache:
|
|
mydep = mybuffer.pop()
|
|
continue
|
|
|
|
treedepth = mydep[0]+1
|
|
|
|
# all checks passed, well done
|
|
matchcache.add(match)
|
|
deptree.add((mydep[0],match)) # add match
|
|
|
|
matchdb = self.openRepositoryDatabase(match[1])
|
|
myundeps = matchdb.retrieveDependenciesList(match[0])
|
|
if (not empty_deps):
|
|
myundeps, xxx = self.filterSatisfiedDependencies(myundeps, deep_deps = deep_deps)
|
|
del xxx
|
|
for x in myundeps:
|
|
mybuffer.push((treedepth,x))
|
|
|
|
mydep = mybuffer.pop()
|
|
|
|
newdeptree = {}
|
|
for x in deptree:
|
|
key = x[0]
|
|
item = x[1]
|
|
try:
|
|
newdeptree[key].add(item)
|
|
except:
|
|
newdeptree[key] = set()
|
|
newdeptree[key].add(item)
|
|
del deptree
|
|
|
|
if (dependenciesNotFound):
|
|
# Houston, we've got a problem
|
|
flatview = list(dependenciesNotFound)
|
|
return flatview,-2
|
|
|
|
# conflicts
|
|
newdeptree[0] = conflicts
|
|
|
|
treecache.clear()
|
|
matchcache.clear()
|
|
|
|
return newdeptree,0 # note: newtree[0] contains possible conflicts
|
|
|
|
|
|
def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False):
|
|
|
|
deptree = {}
|
|
deptree[0] = set()
|
|
|
|
if not etpUi['quiet']: atomlen = len(matched_atoms); count = 0
|
|
matchfilter = matchContainer()
|
|
|
|
for atomInfo in matched_atoms:
|
|
|
|
if not etpUi['quiet']: count += 1; self.updateProgress(":: "+str(round((float(count)/atomlen)*100,1))+"% ::", importance = 0, type = "info", back = True)
|
|
|
|
# check if atomInfo is in matchfilter
|
|
|
|
newtree, result = self.generate_dependency_tree(atomInfo, empty_deps, deep_deps, matchfilter = matchfilter)
|
|
|
|
if (result != 0):
|
|
return newtree, result
|
|
elif (newtree):
|
|
parent_keys = deptree.keys()
|
|
# add conflicts
|
|
max_parent_key = parent_keys[-1]
|
|
deptree[0].update(newtree[0])
|
|
# reverse dict
|
|
levelcount = 0
|
|
reversetree = {}
|
|
for key in newtree.keys()[::-1]:
|
|
if key == 0:
|
|
continue
|
|
levelcount += 1
|
|
reversetree[levelcount] = newtree[key]
|
|
del newtree
|
|
for mylevel in reversetree.keys():
|
|
deptree[max_parent_key+mylevel] = reversetree[mylevel].copy()
|
|
del reversetree
|
|
|
|
matchfilter.clear()
|
|
del matchfilter
|
|
return deptree,0
|
|
|
|
'''
|
|
@description: generates a depends tree using provided idpackages (from client database)
|
|
!!! you can see it as the function that generates the removal tree
|
|
@input package: idpackages list
|
|
@output: depends tree dictionary, plus status code
|
|
'''
|
|
def generate_depends_tree(self, idpackages, deep = False):
|
|
|
|
''' caching '''
|
|
cached = generateDependsTreeCache.get(tuple(idpackages))
|
|
if cached:
|
|
if (cached['deep'] == deep):
|
|
return cached['result']
|
|
|
|
dependscache = set()
|
|
dependsOk = False
|
|
treeview = set(idpackages)
|
|
treelevel = idpackages[:]
|
|
tree = {}
|
|
treedepth = 0 # I start from level 1 because level 0 is idpackages itself
|
|
tree[treedepth] = set(idpackages)
|
|
monotree = set(idpackages) # monodimensional tree
|
|
|
|
# check if dependstable is sane before beginning
|
|
self.clientDbconn.retrieveDepends(idpackages[0])
|
|
count = 0
|
|
|
|
while (not dependsOk):
|
|
treedepth += 1
|
|
tree[treedepth] = set()
|
|
for idpackage in treelevel:
|
|
|
|
count += 1
|
|
p_atom = self.clientDbconn.retrieveAtom(idpackage)
|
|
self.updateProgress(blue("Calculating removable depends of %s") % (red(p_atom),), importance = 0, type = "info", back = True, header = '|/-\\'[count%4]+" ")
|
|
|
|
systempkg = self.clientDbconn.isSystemPackage(idpackage)
|
|
if (idpackage in dependscache) or systempkg:
|
|
try:
|
|
while 1: treeview.remove(idpackage)
|
|
except:
|
|
pass
|
|
continue
|
|
|
|
# obtain its depends
|
|
depends = self.clientDbconn.retrieveDepends(idpackage)
|
|
# filter already satisfied ones
|
|
depends = [x for x in depends if x not in list(monotree) and not self.clientDbconn.isSystemPackage(x)]
|
|
if (depends): # something depends on idpackage
|
|
for x in depends:
|
|
if x not in tree[treedepth]:
|
|
tree[treedepth].add(x)
|
|
monotree.add(x)
|
|
treeview.add(x)
|
|
elif deep: # if deep, grab its dependencies and check
|
|
mydeps = set(self.clientDbconn.retrieveDependencies(idpackage))
|
|
_mydeps = set()
|
|
for x in mydeps:
|
|
match = self.clientDbconn.atomMatch(x)
|
|
if match and match[1] == 0:
|
|
_mydeps.add(match[0])
|
|
mydeps = _mydeps
|
|
# now filter them
|
|
mydeps = [x for x in mydeps if x not in list(monotree) and not self.clientDbconn.isSystemPackage(x)]
|
|
for x in mydeps:
|
|
mydepends = self.clientDbconn.retrieveDepends(x)
|
|
mydepends = [y for y in mydepends if y not in list(monotree)]
|
|
if (not mydepends):
|
|
tree[treedepth].add(x)
|
|
monotree.add(x)
|
|
treeview.add(x)
|
|
|
|
dependscache.add(idpackage)
|
|
try:
|
|
while 1: treeview.remove(idpackage)
|
|
except:
|
|
pass
|
|
|
|
treelevel = list(treeview)[:]
|
|
if (not treelevel):
|
|
if not tree[treedepth]:
|
|
del tree[treedepth] # probably the last one is empty then
|
|
dependsOk = True
|
|
|
|
newtree = tree.copy() # tree list
|
|
if (tree):
|
|
# now filter newtree
|
|
treelength = len(newtree)
|
|
for count in range(treelength)[::-1]:
|
|
x = 0
|
|
while x < count:
|
|
# remove dups in this list
|
|
for z in newtree[count]:
|
|
try:
|
|
while 1:
|
|
newtree[x].remove(z)
|
|
except:
|
|
pass
|
|
x += 1
|
|
|
|
del tree
|
|
|
|
''' caching '''
|
|
generateDependsTreeCache[tuple(idpackages)] = {}
|
|
generateDependsTreeCache[tuple(idpackages)]['result'] = newtree,0
|
|
generateDependsTreeCache[tuple(idpackages)]['deep'] = deep
|
|
return newtree,0 # treeview is used to show deps while tree is used to run the dependency code.
|
|
|
|
def list_repo_categories(self):
|
|
categories = set()
|
|
for repo in etpRepositories:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
catsdata = dbconn.listAllCategories()
|
|
categories.update(set([x[1] for x in catsdata]))
|
|
return categories
|
|
|
|
def list_repo_packages_in_category(self, category):
|
|
pkg_matches = set()
|
|
for repo in etpRepositories:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
catsdata = dbconn.searchPackagesByCategory(category, branch = etpConst['branch'])
|
|
pkg_matches.update(set([(x[1],repo) for x in catsdata]))
|
|
return pkg_matches
|
|
|
|
def list_installed_packages_in_category(self, category):
|
|
pkg_matches = set([x[1] for x in self.clientDbconn.searchPackagesByCategory(category)])
|
|
return pkg_matches
|
|
|
|
def all_repositories_checksum(self):
|
|
sum_hashes = ''
|
|
for repo in etpRepositories:
|
|
try:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
except exceptionTools.RepositoryError:
|
|
continue # repo not available
|
|
sum_hashes += dbconn.tablesChecksum()
|
|
return sum_hashes
|
|
|
|
# this function searches all the not installed packages available in the repositories
|
|
def calculate_available_packages(self, branch = etpConst['branch']):
|
|
|
|
if self.xcache:
|
|
repo_digest = self.all_repositories_checksum()
|
|
client_digest = self.clientDbconn.tablesChecksum()
|
|
disk_cache = self.dumpTools.loadobj(etpCache['world_available'])
|
|
try:
|
|
if disk_cache != None:
|
|
if disk_cache['repo_digest'] == repo_digest and \
|
|
disk_cache['branch'] == branch and \
|
|
disk_cache['client_digest'] == client_digest and \
|
|
disk_cache['etpRepositories_keys'] == etpRepositories.keys() and \
|
|
disk_cache['etpRepositoriesOrder'] == etpRepositoriesOrder:
|
|
return disk_cache['available']
|
|
except:
|
|
try:
|
|
self.dumpTools.dumpobj(etpCache['world_available'], {})
|
|
except IOError:
|
|
pass
|
|
|
|
available = set()
|
|
self.setTotalCycles(len(etpRepositories))
|
|
for repo in etpRepositories:
|
|
try:
|
|
dbconn = self.openRepositoryDatabase(repo)
|
|
except exceptionTools.RepositoryError:
|
|
self.cycleDone()
|
|
continue
|
|
idpackages = dbconn.listAllIdpackages(branch = etpConst['branch'])
|
|
count = 0
|
|
maxlen = len(idpackages)
|
|
for idpackage in idpackages:
|
|
count += 1
|
|
self.updateProgress("Calculating updates for %s" % (repo,), importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = "::")
|
|
# ignore masked packages
|
|
idpackage = dbconn.idpackageValidator(idpackage)
|
|
if idpackage == -1:
|
|
continue
|
|
# get key + slot
|
|
key, slot = dbconn.retrieveKeySlot(idpackage)
|
|
matches = self.clientDbconn.searchKeySlot(key, slot)
|
|
if not matches:
|
|
available.add((idpackage,repo))
|
|
self.cycleDone()
|
|
|
|
if self.xcache:
|
|
try:
|
|
mycache = {}
|
|
mycache['repo_digest'] = repo_digest
|
|
mycache['client_digest'] = client_digest
|
|
mycache['available'] = available.copy()
|
|
mycache['branch'] = branch
|
|
mycache['etpRepositories_keys'] = etpRepositories.keys()[:]
|
|
mycache['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
# save cache
|
|
self.dumpTools.dumpobj(etpCache['world_available'], mycache)
|
|
mycache.clear()
|
|
del mycache
|
|
except:
|
|
self.dumpTools.dumpobj(etpCache['world_available'], {})
|
|
|
|
return available
|
|
|
|
def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None):
|
|
if self.xcache:
|
|
if db_digest == None:
|
|
db_digest = self.clientDbconn.tablesChecksum()
|
|
disk_cache = self.dumpTools.loadobj(etpCache['world_update'])
|
|
try:
|
|
if disk_cache != None:
|
|
if disk_cache['db_digest'] == db_digest and \
|
|
disk_cache['empty_deps'] == empty_deps and \
|
|
disk_cache['etpRepositories_keys'] == etpRepositories.keys() and \
|
|
disk_cache['etpRepositoriesOrder'] == etpRepositoriesOrder and \
|
|
disk_cache['branch'] == branch:
|
|
return disk_cache['update'],disk_cache['remove'],disk_cache['fine']
|
|
except:
|
|
try:
|
|
self.dumpTools.dumpobj(etpCache['world_update'], {})
|
|
except IOError:
|
|
pass
|
|
|
|
def calculate_world_updates(self, empty_deps = False, branch = etpConst['branch']):
|
|
|
|
update = set()
|
|
remove = set()
|
|
fine = set()
|
|
|
|
db_digest = self.clientDbconn.tablesChecksum()
|
|
cached = self.get_world_update_cache(empty_deps = empty_deps, branch = branch, db_digest = db_digest)
|
|
if cached != None:
|
|
return cached
|
|
|
|
# get all the installed packages
|
|
idpackages = self.clientDbconn.listAllIdpackages()
|
|
maxlen = len(idpackages)
|
|
count = 0
|
|
for idpackage in idpackages:
|
|
count += 1
|
|
self.updateProgress("Calculating world dependencies", importance = 0, type = "info", back = True, header = "::", count = (count,maxlen), percent = True, footer = " ::")
|
|
tainted = False
|
|
myscopedata = self.clientDbconn.getScopeData(idpackage)
|
|
#atom = myscopedata[0]
|
|
#category = myscopedata[1]
|
|
#name = myscopedata[2]
|
|
#slot = myscopedata[4]
|
|
#revision = myscopedata[6]
|
|
#atomkey = myscopedata[1]+"/"+myscopedata[2]
|
|
# search in the packages
|
|
match = self.atomMatch(myscopedata[0])
|
|
if match[0] == -1: # atom has been changed, or removed?
|
|
tainted = True
|
|
else: # not changed, is the revision changed?
|
|
adbconn = self.openRepositoryDatabase(match[1])
|
|
arevision = adbconn.retrieveRevision(match[0])
|
|
# if revision is 9999, then any revision is fine
|
|
if myscopedata[6] == 9999: arevision = 9999
|
|
if myscopedata[6] != arevision:
|
|
tainted = True
|
|
elif (myscopedata[6] == arevision):
|
|
# check if "needed" are the same, otherwise, pull
|
|
# this will avoid having old packages installed just because user ran equo database generate (migrating from gentoo)
|
|
# also this helps in environments with multiple repositories, to avoid messing with libraries
|
|
aneeded = adbconn.retrieveNeeded(match[0])
|
|
needed = self.clientDbconn.retrieveNeeded(idpackage)
|
|
if needed != aneeded:
|
|
tainted = True
|
|
else:
|
|
# check use flags too
|
|
# it helps for the same reason above and when doing upgrades to different branches
|
|
auseflags = adbconn.retrieveUseflags(match[0])
|
|
useflags = self.clientDbconn.retrieveUseflags(idpackage)
|
|
if auseflags != useflags:
|
|
tainted = True
|
|
elif (empty_deps):
|
|
tainted = True
|
|
if (tainted):
|
|
# Alice! use the key! ... and the slot
|
|
matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchSlot = myscopedata[4], matchBranches = (branch,))
|
|
if matchresults[0] != -1:
|
|
mdbconn = self.openRepositoryDatabase(matchresults[1])
|
|
#matchatom = mdbconn.retrieveAtom(matchresults[0])
|
|
update.add(matchresults)
|
|
else:
|
|
remove.add(idpackage)
|
|
# look for packages that would match key with any slot (for eg, gcc updates), slot changes handling
|
|
matchresults = self.atomMatch(myscopedata[1]+"/"+myscopedata[2], matchBranches = (branch,))
|
|
if matchresults[0] != -1:
|
|
mdbconn = self.openRepositoryDatabase(matchresults[1])
|
|
matchatom = mdbconn.retrieveAtom(matchresults[0])
|
|
# compare versions
|
|
unsatisfied, satisfied = self.filterSatisfiedDependencies((matchatom,))
|
|
if unsatisfied:
|
|
update.add(matchresults)
|
|
else:
|
|
fine.add(myscopedata[0])
|
|
|
|
del idpackages
|
|
|
|
if self.xcache:
|
|
try:
|
|
mycache = {}
|
|
mycache['db_digest'] = db_digest
|
|
mycache['update'] = update.copy()
|
|
mycache['remove'] = remove.copy()
|
|
mycache['fine'] = fine.copy()
|
|
mycache['empty_deps'] = empty_deps
|
|
mycache['branch'] = branch
|
|
mycache['etpRepositories_keys'] = etpRepositories.keys()[:]
|
|
mycache['etpRepositoriesOrder'] = etpRepositoriesOrder[:]
|
|
# save cache
|
|
self.dumpTools.dumpobj(etpCache['world_update'], mycache)
|
|
mycache.clear()
|
|
del mycache
|
|
except:
|
|
self.dumpTools.dumpobj(etpCache['world_update'], {})
|
|
return update, remove, fine
|
|
|
|
# every tbz2 file that would be installed must pass from here
|
|
def add_tbz2_to_repos(self, tbz2file):
|
|
atoms_contained = []
|
|
basefile = os.path.basename(tbz2file)
|
|
if os.path.isdir(etpConst['entropyunpackdir']+"/"+basefile[:-5]):
|
|
shutil.rmtree(etpConst['entropyunpackdir']+"/"+basefile[:-5])
|
|
os.makedirs(etpConst['entropyunpackdir']+"/"+basefile[:-5])
|
|
dbfile = self.entropyTools.extractEdb(tbz2file, dbpath = etpConst['entropyunpackdir']+"/"+basefile[:-5]+"/packages.db")
|
|
if dbfile == None:
|
|
return -1,atoms_contained
|
|
etpSys['dirstoclean'].add(os.path.dirname(dbfile))
|
|
# add dbfile
|
|
repodata = {}
|
|
repodata['repoid'] = basefile
|
|
repodata['description'] = "Dynamic database from "+basefile
|
|
repodata['packages'] = []
|
|
repodata['dbpath'] = os.path.dirname(dbfile)
|
|
repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added
|
|
repodata['smartpackage'] = False # extra info added
|
|
self.addRepository(repodata)
|
|
mydbconn = self.openGenericDatabase(dbfile)
|
|
# read all idpackages
|
|
try:
|
|
myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files
|
|
except:
|
|
del etpRepositories[basefile]
|
|
return -2,atoms_contained
|
|
if len(myidpackages) > 1:
|
|
etpRepositories[basefile]['smartpackage'] = True
|
|
for myidpackage in myidpackages:
|
|
compiled_arch = mydbconn.retrieveDownloadURL(myidpackage)
|
|
if compiled_arch.find("/"+etpSys['arch']+"/") == -1:
|
|
return -3,atoms_contained
|
|
atoms_contained.append((int(myidpackage),basefile))
|
|
mydbconn.closeDB()
|
|
del mydbconn
|
|
return 0,atoms_contained
|
|
|
|
# This is the function that should be used by third party applications
|
|
# to retrieve a list of available updates, along with conflicts (removalQueue) and obsoletes
|
|
# (removed)
|
|
def retrieveWorldQueue(self, empty_deps = False, branch = etpConst['branch']):
|
|
update, remove, fine = self.calculate_world_updates(empty_deps = empty_deps, branch = branch)
|
|
del fine
|
|
data = {}
|
|
data['removed'] = list(remove)
|
|
data['runQueue'] = []
|
|
data['removalQueue'] = []
|
|
status = -1
|
|
if update:
|
|
# calculate install+removal queues
|
|
install, removal, status = self.retrieveInstallQueue(update, empty_deps, deep_deps = False)
|
|
# update data['removed']
|
|
data['removed'] = [x for x in data['removed'] if x not in removal]
|
|
data['runQueue'] += install
|
|
data['removalQueue'] += removal
|
|
return data,status
|
|
|
|
def validatePackageRemoval(self, idpackage):
|
|
system_pkg = self.clientDbconn.isSystemPackage(idpackage)
|
|
if not system_pkg:
|
|
return True # valid
|
|
|
|
pkgatom = self.clientDbconn.retrieveAtom(idpackage)
|
|
# check if the package is slotted and exist more than one installed first
|
|
sysresults = self.clientDbconn.atomMatch(self.entropyTools.dep_getkey(pkgatom), multiMatch = True)
|
|
slots = set()
|
|
if sysresults[1] == 0:
|
|
for x in sysresults[0]:
|
|
slots.add(self.clientDbconn.retrieveSlot(x))
|
|
if len(slots) < 2:
|
|
return False
|
|
return True # valid
|
|
else:
|
|
return False
|
|
|
|
|
|
def retrieveRemovalQueue(self, idpackages, deep = False):
|
|
queue = []
|
|
treeview = self.generate_depends_tree(idpackages, deep = deep)
|
|
for x in range(len(treeview[0]))[::-1]:
|
|
for y in treeview[0][x]:
|
|
queue.append(y)
|
|
return queue
|
|
|
|
def retrieveInstallQueue(self, matched_atoms, empty_deps, deep_deps):
|
|
|
|
install = []
|
|
removal = []
|
|
treepackages, result = self.get_required_packages(matched_atoms, empty_deps, deep_deps)
|
|
|
|
if result == -2:
|
|
return treepackages,removal,result
|
|
|
|
# format
|
|
for x in range(len(treepackages)):
|
|
if x == 0:
|
|
# conflicts
|
|
for a in treepackages[x]:
|
|
removal.append(a)
|
|
else:
|
|
for a in treepackages[x]:
|
|
install.append(a)
|
|
|
|
# filter out packages that are in actionQueue comparing key + slot
|
|
if install and removal:
|
|
myremmatch = {}
|
|
for x in removal:
|
|
myremmatch.update({(self.entropyTools.dep_getkey(self.clientDbconn.retrieveAtom(x)),self.clientDbconn.retrieveSlot(x)): x})
|
|
for packageInfo in install:
|
|
dbconn = self.openRepositoryDatabase(packageInfo[1])
|
|
testtuple = (self.entropyTools.dep_getkey(dbconn.retrieveAtom(packageInfo[0])),dbconn.retrieveSlot(packageInfo[0]))
|
|
if testtuple in myremmatch:
|
|
# remove from removalQueue
|
|
if myremmatch[testtuple] in removal:
|
|
removal.remove(myremmatch[testtuple])
|
|
del testtuple
|
|
del myremmatch
|
|
|
|
del treepackages
|
|
return install, removal, 0
|
|
|
|
# this function searches into client database for a package matching provided key + slot
|
|
# and returns its idpackage or -1 if none found
|
|
def retrieveInstalledIdPackage(self, pkgkey, pkgslot):
|
|
match = self.clientDbconn.atomMatch(pkgkey, matchSlot = pkgslot)
|
|
if match[1] == 0:
|
|
return match[0]
|
|
return -1
|
|
|
|
'''
|
|
Package interface :: begin
|
|
'''
|
|
|
|
# Get checksum of a package by running md5sum remotely (using php helpers)
|
|
# @returns hex: if the file exists
|
|
# @returns None: if the server does not support HTTP handlers
|
|
# @returns None: if the file is not found
|
|
# mainly used server side
|
|
def get_remote_package_checksum(self, servername, filename, branch):
|
|
|
|
# etpHandlers['md5sum'] is the command
|
|
# create the request
|
|
try:
|
|
url = etpRemoteSupport[servername]
|
|
except:
|
|
# not found, does not support HTTP handlers
|
|
return None
|
|
|
|
# does the package has "#" (== tag) ? hackish thing that works
|
|
filename = filename.replace("#","%23")
|
|
# "+"
|
|
filename = filename.replace("+","%2b")
|
|
|
|
request = url+etpHandlers['md5sum']+filename+"&branch="+branch
|
|
|
|
# now pray the server
|
|
try:
|
|
if etpConst['proxy']:
|
|
proxy_support = urllib2.ProxyHandler(etpConst['proxy'])
|
|
opener = urllib2.build_opener(proxy_support)
|
|
urllib2.install_opener(opener)
|
|
item = urllib2.urlopen(request)
|
|
result = item.readline().strip()
|
|
item.close()
|
|
del item
|
|
return result
|
|
except: # no HTTP support?
|
|
return None
|
|
|
|
'''
|
|
@description: check if Equo has to download the given package
|
|
@input package: filename to check inside the packages directory -> file, checksum of the package -> checksum
|
|
@output: -1 = should be downloaded, -2 = digest broken (not mandatory), remove & download, 0 = all fine, we don't need to download it
|
|
'''
|
|
def check_needed_package_download(self, filepath, checksum = None):
|
|
# is the file available
|
|
if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath):
|
|
if checksum is None:
|
|
return 0
|
|
else:
|
|
# check digest
|
|
md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum)
|
|
if (md5res):
|
|
return 0
|
|
else:
|
|
return -2
|
|
else:
|
|
return -1
|
|
|
|
'''
|
|
@description: download a package into etpConst['packagesbindir'] and check for digest if digest is not False
|
|
@input package: url -> HTTP/FTP url, digest -> md5 hash of the file
|
|
@output: -1 = download error (cannot find the file), -2 = digest error, 0 = all fine
|
|
'''
|
|
def fetch_file(self, url, digest = None, resume = True):
|
|
# remove old
|
|
filename = os.path.basename(url)
|
|
filepath = etpConst['packagesbindir']+"/"+etpConst['branch']+"/"+filename
|
|
|
|
# load class
|
|
fetchConn = self.urlFetcher(url, filepath, resume = resume)
|
|
fetchConn.progress = self.progress
|
|
|
|
# start to download
|
|
data_transfer = 0
|
|
resumed = False
|
|
try:
|
|
fetchChecksum = fetchConn.download()
|
|
data_transfer = fetchConn.datatransfer
|
|
resumed = fetchConn.resumed
|
|
except KeyboardInterrupt:
|
|
return -4, data_transfer, resumed
|
|
except NameError:
|
|
raise
|
|
except:
|
|
return -1, data_transfer, resumed
|
|
if fetchChecksum == "-3":
|
|
return -3, data_transfer, resumed
|
|
|
|
del fetchConn
|
|
if (digest):
|
|
if (fetchChecksum != digest):
|
|
# not properly downloaded
|
|
return -2, data_transfer, resumed
|
|
else:
|
|
return 0, data_transfer, resumed
|
|
return 0, data_transfer, resumed
|
|
|
|
def add_failing_mirror(self, mirrorname,increment = 1):
|
|
item = etpRemoteFailures.get(mirrorname)
|
|
if item == None:
|
|
etpRemoteFailures[mirrorname] = increment
|
|
else:
|
|
etpRemoteFailures[mirrorname] += increment # add a failure
|
|
return etpRemoteFailures[mirrorname]
|
|
|
|
def get_failing_mirror_status(self, mirrorname):
|
|
item = etpRemoteFailures.get(mirrorname)
|
|
if item == None:
|
|
return 0
|
|
else:
|
|
return item
|
|
|
|
'''
|
|
@description: download a package into etpConst['packagesbindir'] passing all the available mirrors
|
|
@input package: repository -> name of the repository, filename -> name of the file to download, digest -> md5 hash of the file
|
|
@output: 0 = all fine, !=0 = error on all the available mirrors
|
|
'''
|
|
def fetch_file_on_mirrors(self, repository, filename, digest = False, verified = False):
|
|
|
|
uris = etpRepositories[repository]['packages'][::-1]
|
|
remaining = set(uris[:])
|
|
|
|
if verified: # file is already in place, match_checksum set infoDict['verified'] to True
|
|
return 0
|
|
|
|
mirrorcount = 0
|
|
for uri in uris:
|
|
|
|
if not remaining:
|
|
# tried all the mirrors, quitting for error
|
|
return 3
|
|
|
|
mirrorcount += 1
|
|
mirrorCountText = "( mirror #"+str(mirrorcount)+" ) "
|
|
url = uri+"/"+filename
|
|
|
|
# check if uri is sane
|
|
if self.get_failing_mirror_status(uri) >= 30:
|
|
# ohohoh!
|
|
etpRemoteFailures[uri] = 30 # set to 30 for convenience
|
|
self.updateProgress(
|
|
mirrorCountText+blue(" Mirror: ")+red(self.entropyTools.spliturl(url)[1])+" - maximum failure threshold reached.",
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" ## ")
|
|
)
|
|
|
|
if self.get_failing_mirror_status(uri) == 30:
|
|
self.add_failing_mirror(uri,45) # put to 75 then decrement by 4 so we won't reach 30 anytime soon ahahaha
|
|
else:
|
|
# now decrement each time this point is reached, if will be back < 30, then equo will try to use it again
|
|
if self.get_failing_mirror_status(uri) > 31:
|
|
self.add_failing_mirror(uri,-4)
|
|
else:
|
|
# put to 0 - reenable mirror, welcome back uri!
|
|
etpRemoteFailures[uri] = 0
|
|
|
|
remaining.remove(uri)
|
|
continue
|
|
|
|
do_resume = True
|
|
while 1:
|
|
try:
|
|
# now fetch the new one
|
|
self.updateProgress(
|
|
mirrorCountText+blue("Downloading from: ")+red(self.entropyTools.spliturl(url)[1]),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" ## ")
|
|
)
|
|
rc, data_transfer, resumed = self.fetch_file(url, digest, do_resume)
|
|
if rc == 0:
|
|
self.updateProgress(
|
|
mirrorCountText+blue("Successfully downloaded from: ")+red(self.entropyTools.spliturl(url)[1])+blue(" at "+str(self.entropyTools.bytesIntoHuman(data_transfer))+"/sec"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
return 0
|
|
elif resumed:
|
|
do_resume = False
|
|
continue
|
|
else:
|
|
error_message = mirrorCountText+blue("Error downloading from: ")+red(self.entropyTools.spliturl(url)[1])
|
|
# something bad happened
|
|
if rc == -1:
|
|
error_message += " - file not available on this mirror."
|
|
elif rc == -2:
|
|
self.add_failing_mirror(uri,1)
|
|
error_message += " - wrong checksum."
|
|
elif rc == -3:
|
|
self.add_failing_mirror(uri,2)
|
|
error_message += " - not found."
|
|
elif rc == -4:
|
|
error_message += blue(" - discarded download.")
|
|
else:
|
|
self.add_failing_mirror(uri, 5)
|
|
error_message += " - unknown reason."
|
|
self.updateProgress(
|
|
error_message,
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" ## ")
|
|
)
|
|
if rc == -4: # user discarded fetch
|
|
return 1
|
|
remaining.remove(uri)
|
|
break
|
|
except KeyboardInterrupt:
|
|
break
|
|
except:
|
|
raise
|
|
return 0
|
|
|
|
def quickpkg(self, atomstring, savedir = None):
|
|
if savedir == None:
|
|
savedir = etpConst['packagestmpdir']
|
|
if not os.path.isdir(etpConst['packagestmpdir']):
|
|
os.makedirs(etpConst['packagestmpdir'])
|
|
# match package
|
|
match = self.clientDbconn.atomMatch(atomstring)
|
|
if match[0] == -1:
|
|
return -1,None,None
|
|
atom = self.clientDbconn.atomMatch(match[0])
|
|
pkgdata = self.clientDbconn.getPackageData(match[0])
|
|
resultfile = self.entropyTools.quickpkg(pkgdata = pkgdata, dirpath = savedir)
|
|
if resultfile == None:
|
|
return -1,atom,None
|
|
else:
|
|
return 0,atom,resultfile
|
|
|
|
def Package(self):
|
|
conn = PackageInterface(EquoInstance = self)
|
|
return conn
|
|
|
|
def instanceTest(self):
|
|
return
|
|
|
|
'''
|
|
Package interface :: end
|
|
'''
|
|
|
|
'''
|
|
Triggers interface :: begin
|
|
'''
|
|
def Triggers(self, phase, pkgdata):
|
|
conn = TriggerInterface(EquoInstance = self, phase = phase, pkgdata = pkgdata)
|
|
return conn
|
|
'''
|
|
Triggers interface :: end
|
|
'''
|
|
|
|
'''
|
|
Repository interface :: begin
|
|
'''
|
|
def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False):
|
|
conn = RepoInterface(EquoInstance = self, reponames = reponames, forceUpdate = forceUpdate, noEquoCheck = noEquoCheck)
|
|
return conn
|
|
'''
|
|
Repository interface :: end
|
|
'''
|
|
|
|
'''
|
|
Configuration files (updates, not entropy related) interface :: begin
|
|
'''
|
|
def __FileUpdates(self):
|
|
conn = FileUpdatesInterface(EquoInstance = self)
|
|
return conn
|
|
'''
|
|
Configuration files (updates, not entropy related) interface :: end
|
|
'''
|
|
|
|
'''
|
|
Real package actions (install/remove) interface
|
|
'''
|
|
class PackageInterface:
|
|
|
|
def __init__(self, EquoInstance):
|
|
self.Entropy = EquoInstance
|
|
try:
|
|
self.Entropy.instanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
|
|
self.infoDict = {}
|
|
self.prepared = False
|
|
self.matched_atom = ()
|
|
self.valid_actions = ("fetch","remove","install")
|
|
self.action = None
|
|
|
|
def kill(self):
|
|
self.infoDict.clear()
|
|
self.matched_atom = ()
|
|
self.valid_actions = ()
|
|
self.action = None
|
|
self.prepared = False
|
|
|
|
def error_on_prepared(self):
|
|
if self.prepared:
|
|
raise exceptionTools.PermissionDenied("PermissionDenied: already prepared")
|
|
|
|
def error_on_not_prepared(self):
|
|
if not self.prepared:
|
|
raise exceptionTools.PermissionDenied("PermissionDenied: not yet prepared")
|
|
|
|
def check_action_validity(self, action):
|
|
if action not in self.valid_actions:
|
|
raise exceptionTools.InvalidData("InvalidData: action must be in %s" % (str(self.valid_actions),))
|
|
|
|
def match_checksum(self):
|
|
self.error_on_not_prepared()
|
|
dlcount = 0
|
|
match = False
|
|
while dlcount <= 5:
|
|
self.Entropy.updateProgress(
|
|
blue("Checking package checksum..."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = red(" ## "),
|
|
back = True
|
|
)
|
|
dlcheck = self.Entropy.check_needed_package_download(self.infoDict['download'], checksum = self.infoDict['checksum'])
|
|
if dlcheck == 0:
|
|
self.Entropy.updateProgress(
|
|
blue("Package checksum matches."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
self.infoDict['verified'] = True
|
|
match = True
|
|
break # file downloaded successfully
|
|
else:
|
|
dlcount += 1
|
|
self.Entropy.updateProgress(
|
|
blue("Package checksum does not match. Redownloading... attempt #"+str(dlcount)),
|
|
importance = 0,
|
|
type = "info",
|
|
header = red(" ## "),
|
|
back = True
|
|
)
|
|
fetch = self.Entropy.fetch_file_on_mirrors(
|
|
self.infoDict['repository'],
|
|
self.infoDict['download'],
|
|
self.infoDict['checksum']
|
|
)
|
|
if fetch != 0:
|
|
self.Entropy.updateProgress(
|
|
blue("Cannot properly fetch package! Quitting."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
return fetch
|
|
else:
|
|
self.infoDict['verified'] = True
|
|
if (not match):
|
|
self.Entropy.updateProgress(
|
|
blue("Cannot properly fetch package or checksum does not match. Try download latest repositories."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
return 1
|
|
return 0
|
|
|
|
'''
|
|
@description: unpack the given package file into the unpack dir
|
|
@input infoDict: dictionary containing package information
|
|
@output: 0 = all fine, >0 = error!
|
|
'''
|
|
def __unpack_package(self):
|
|
self.error_on_not_prepared()
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Unpacking package: "+str(self.infoDict['atom']))
|
|
|
|
if os.path.isdir(self.infoDict['unpackdir']):
|
|
shutil.rmtree(self.infoDict['unpackdir'].encode('raw_unicode_escape'))
|
|
elif os.path.isfile(self.infoDict['unpackdir']):
|
|
os.remove(self.infoDict['unpackdir'].encode('raw_unicode_escape'))
|
|
os.makedirs(self.infoDict['imagedir'])
|
|
|
|
rc = self.Entropy.entropyTools.uncompressTarBz2(
|
|
self.infoDict['pkgpath'],
|
|
self.infoDict['imagedir'],
|
|
catchEmpty = True
|
|
)
|
|
if rc != 0:
|
|
return rc
|
|
if not os.path.isdir(self.infoDict['imagedir']):
|
|
return 2
|
|
|
|
# unpack xpak ?
|
|
if etpConst['gentoo-compat']:
|
|
if os.path.isdir(self.infoDict['xpakpath']):
|
|
shutil.rmtree(self.infoDict['xpakpath'])
|
|
try:
|
|
os.rmdir(self.infoDict['xpakpath'])
|
|
except OSError:
|
|
pass
|
|
os.makedirs(self.infoDict['xpakpath'])
|
|
# create data dir where we'll unpack the xpak
|
|
os.mkdir(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'])
|
|
# now unpack for real
|
|
xpakPath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakfilename']
|
|
|
|
if (self.infoDict['smartpackage']):
|
|
# we need to get the .xpak from database
|
|
xdbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository'])
|
|
xpakdata = xdbconn.retrieveXpakMetadata(self.infoDict['idpackage'])
|
|
if xpakdata:
|
|
# save into a file
|
|
f = open(xpakPath,"wb")
|
|
f.write(xpakdata)
|
|
f.flush()
|
|
f.close()
|
|
self.infoDict['xpakstatus'] = self.Entropy.entropyTools.unpackXpak(
|
|
xpakPath,
|
|
self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
|
|
)
|
|
else:
|
|
self.infoDict['xpakstatus'] = None
|
|
del xpakdata
|
|
else:
|
|
self.infoDict['xpakstatus'] = self.Entropy.entropyTools.extractXpak(
|
|
self.infoDict['pkgpath'],
|
|
self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
|
|
)
|
|
|
|
# create fake portage ${D} linking it to imagedir
|
|
portage_db_fakedir = os.path.join(
|
|
self.infoDict['unpackdir'],
|
|
"portage/"+self.infoDict['category'] + "/" + self.infoDict['name'] + "-" + self.infoDict['version']
|
|
)
|
|
|
|
os.makedirs(portage_db_fakedir)
|
|
# now link it to self.infoDict['imagedir']
|
|
os.symlink(self.infoDict['imagedir'],os.path.join(portage_db_fakedir,"image"))
|
|
|
|
return 0
|
|
|
|
def __remove_package(self):
|
|
|
|
# clear on-disk cache
|
|
generateDependsTreeCache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],generateDependsTreeCache)
|
|
check_package_update_cache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['check_package_update'],{})
|
|
# remove security advisories cache
|
|
self.Entropy.dumpTools.dumpobj(etpCache['advisories'],{})
|
|
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: "+str(self.infoDict['removeatom']))
|
|
|
|
# remove from database
|
|
if self.infoDict['removeidpackage'] != -1:
|
|
self.Entropy.updateProgress(
|
|
blue("Removing from Entropy: ")+red(self.infoDict['removeatom']),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
self.__remove_package_from_database()
|
|
|
|
# Handle gentoo database
|
|
if (etpConst['gentoo-compat']): # FIXME: remove dep_striptag asap
|
|
gentooAtom = self.Entropy.entropyTools.dep_striptag(
|
|
self.Entropy.entropyTools.remove_tag(self.infoDict['removeatom'])
|
|
)
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing from Portage: "+str(gentooAtom))
|
|
self.__remove_package_from_gentoo_database(gentooAtom)
|
|
del gentooAtom
|
|
|
|
self.__remove_content_from_system()
|
|
return 0
|
|
|
|
def __remove_content_from_system(self):
|
|
|
|
# load CONFIG_PROTECT and its mask - client database at this point has been surely opened, so our dicts are already filled
|
|
protect = etpConst['dbconfigprotect']
|
|
mask = etpConst['dbconfigprotectmask']
|
|
|
|
# remove files from system
|
|
directories = set()
|
|
for item in self.infoDict['removecontent']:
|
|
# collision check
|
|
if etpConst['collisionprotect'] > 0:
|
|
|
|
if self.Entropy.clientDbconn.isFileAvailable(item) and os.path.isfile(etpConst['systemroot']+item):
|
|
# in this way we filter out directories
|
|
self.Entropy.updateProgress(
|
|
red("Collision found during remove of ")+etpConst['systemroot']+item+red(" - cannot overwrite"),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" ## ")
|
|
)
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Collision found during remove of "+etpConst['systemroot']+item+" - cannot overwrite")
|
|
continue
|
|
|
|
protected = False
|
|
if (not self.infoDict['removeconfig']) and (not self.infoDict['diffremoval']):
|
|
try:
|
|
# -- CONFIGURATION FILE PROTECTION --
|
|
if os.access(etpConst['systemroot']+item,os.R_OK):
|
|
for x in protect:
|
|
if etpConst['systemroot']+item.startswith(x):
|
|
protected = True
|
|
break
|
|
if (protected):
|
|
for x in mask:
|
|
if etpConst['systemroot']+item.startswith(x):
|
|
protected = False
|
|
break
|
|
if (protected) and os.path.isfile(etpConst['systemroot']+item):
|
|
protected = self.Entropy.entropyTools.istextfile(etpConst['systemroot']+item)
|
|
else:
|
|
protected = False # it's not a file
|
|
# -- CONFIGURATION FILE PROTECTION --
|
|
except:
|
|
pass # some filenames are buggy encoded
|
|
|
|
|
|
if (protected):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"[remove] Protecting config file: "+etpConst['systemroot']+item)
|
|
self.Entropy.updateProgress(
|
|
red("[remove] Protecting config file: ")+etpConst['systemroot']+item,
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" ## ")
|
|
)
|
|
else:
|
|
try:
|
|
os.lstat(etpConst['systemroot']+item)
|
|
except OSError:
|
|
continue # skip file, does not exist
|
|
except UnicodeEncodeError:
|
|
self.Entropy.updateProgress(
|
|
red("QA: ")+brown("this package contains a badly encoded file"),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
continue # file has a really bad encoding
|
|
|
|
if os.path.isdir(etpConst['systemroot']+item) and os.path.islink(etpConst['systemroot']+item): # S_ISDIR returns False for directory symlinks, so using os.path.isdir
|
|
# valid directory symlink
|
|
directories.add((etpConst['systemroot']+item,"link"))
|
|
elif os.path.isdir(etpConst['systemroot']+item):
|
|
# plain directory
|
|
directories.add((etpConst['systemroot']+item,"dir"))
|
|
else: # files, symlinks or not
|
|
# just a file or symlink or broken directory symlink (remove now)
|
|
try:
|
|
os.remove(etpConst['systemroot']+item)
|
|
# add its parent directory
|
|
dirfile = os.path.dirname(etpConst['systemroot']+item)
|
|
if os.path.isdir(dirfile) and os.path.islink(dirfile):
|
|
directories.add((dirfile,"link"))
|
|
elif os.path.isdir(dirfile):
|
|
directories.add((dirfile,"dir"))
|
|
except OSError:
|
|
pass
|
|
|
|
# now handle directories
|
|
directories = list(directories)
|
|
directories.reverse()
|
|
while 1:
|
|
taint = False
|
|
for directory in directories:
|
|
mydir = etpConst['systemroot']+directory[0]
|
|
if directory[1] == "link":
|
|
try:
|
|
mylist = os.listdir(mydir)
|
|
if not mylist:
|
|
try:
|
|
os.remove(mydir)
|
|
taint = True
|
|
except OSError:
|
|
pass
|
|
except OSError:
|
|
pass
|
|
elif directory[1] == "dir":
|
|
try:
|
|
mylist = os.listdir(mydir)
|
|
if not mylist:
|
|
try:
|
|
os.rmdir(mydir)
|
|
taint = True
|
|
except OSError:
|
|
pass
|
|
except OSError:
|
|
pass
|
|
|
|
if not taint:
|
|
break
|
|
del directories
|
|
|
|
|
|
'''
|
|
@description: remove package entry from Gentoo database
|
|
@input gentoo package atom (cat/name+ver):
|
|
@output: 0 = all fine, <0 = error!
|
|
'''
|
|
def __remove_package_from_gentoo_database(self, atom):
|
|
|
|
# handle gentoo-compat
|
|
_portage_avail = False
|
|
try:
|
|
from portageTools import getPortageAppDbPath as _portage_getPortageAppDbPath, getInstalledAtoms as _portage_getInstalledAtoms
|
|
_portage_avail = True
|
|
except:
|
|
return -1 # no Portage support
|
|
|
|
if (_portage_avail):
|
|
portDbDir = _portage_getPortageAppDbPath()
|
|
removePath = portDbDir+atom
|
|
try:
|
|
shutil.rmtree(removePath,True)
|
|
except:
|
|
pass
|
|
key = self.Entropy.entropyTools.dep_getkey(atom)
|
|
othersInstalled = _portage_getInstalledAtoms(key) #FIXME: really slow
|
|
if othersInstalled == None: # FIXME: beautify this shit (skippedKey)
|
|
# safest way (error free) is to use sed without loading the file
|
|
# escape /
|
|
skippedKey = ''
|
|
for x in key:
|
|
if x == "/":
|
|
x = "\/"
|
|
skippedKey += x
|
|
os.system("sed -i '/"+skippedKey+"/d' "+etpConst['systemroot']+"/var/lib/portage/world")
|
|
return 0
|
|
|
|
'''
|
|
@description: function that runs at the end of the package installation process, just removes data left by other steps
|
|
@output: 0 = all fine, >0 = error!
|
|
'''
|
|
def __cleanup_package(self, data):
|
|
# remove unpack dir
|
|
shutil.rmtree(data['unpackdir'],True)
|
|
try:
|
|
os.rmdir(data['unpackdir'])
|
|
except OSError:
|
|
pass
|
|
return 0
|
|
|
|
'''
|
|
@description: remove the package from the installed packages database..
|
|
This function is a wrapper around databaseTools.removePackage that will let us to add our custom things
|
|
@output: 0 = all fine, >0 = error!
|
|
'''
|
|
def __remove_package_from_database(self):
|
|
self.error_on_not_prepared()
|
|
|
|
find = False
|
|
disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_update'])
|
|
if disk_cache != None:
|
|
try:
|
|
slot = self.Entropy.clientDbconn.retrieveSlot(self.infoDict['removeidpackage'])
|
|
key = self.Entropy.entropyTools.dep_getkey(self.infoDict['removeatom'])
|
|
matched = self.Entropy.atomMatch(key, matchSlot = slot)
|
|
if matched[0] != -1:
|
|
dbconn = self.Entropy.openRepositoryDatabase(matched[1])
|
|
atom = dbconn.retrieveAtom(matched[0])
|
|
cached_item = (atom,matched)
|
|
if cached_item in disk_cache['update']:
|
|
find = True
|
|
except:
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world_update'], {})
|
|
|
|
self.Entropy.clientDbconn.removePackage(self.infoDict['removeidpackage'])
|
|
|
|
if find:
|
|
disk_cache['update'].remove(cached_item)
|
|
db_digest = self.Entropy.clientDbconn.tablesChecksum()
|
|
disk_cache['db_digest'] = db_digest
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache)
|
|
|
|
return 0
|
|
|
|
'''
|
|
@description: install unpacked files, update database and also update gentoo db if requested
|
|
@output: 0 = all fine, >0 = error!
|
|
'''
|
|
def __install_package(self):
|
|
|
|
# clear on-disk cache
|
|
generateDependsTreeCache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],{})
|
|
check_package_update_cache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['check_package_update'],{})
|
|
# clear advisories cache
|
|
self.Entropy.dumpTools.dumpobj(etpCache['advisories'],{})
|
|
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing package: "+str(self.infoDict['atom']))
|
|
|
|
# copy files over - install
|
|
rc = self.__move_image_to_system()
|
|
if rc != 0:
|
|
return rc
|
|
|
|
# inject into database
|
|
self.Entropy.updateProgress(
|
|
blue("Updating database with: ")+red(self.infoDict['atom']),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
newidpackage = self.__install_package_into_database()
|
|
|
|
# remove old files and gentoo stuff
|
|
if (self.infoDict['removeidpackage'] != -1):
|
|
# doing a diff removal
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Remove old package: "+str(self.infoDict['removeatom']))
|
|
self.infoDict['removeidpackage'] = -1 # disabling database removal
|
|
|
|
compatstring = ''
|
|
if etpConst['gentoo-compat']:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing Entropy and Gentoo database entry for "+str(self.infoDict['removeatom']))
|
|
compatstring = " ## w/Gentoo compatibility"
|
|
else:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing Entropy (only) database entry for "+str(self.infoDict['removeatom']))
|
|
|
|
self.Entropy.updateProgress(
|
|
blue("Cleaning old package files...")+compatstring,
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
self.__remove_package()
|
|
|
|
rc = 0
|
|
if (etpConst['gentoo-compat']):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing new Gentoo database entry: "+str(self.infoDict['atom']))
|
|
rc = self.__install_package_into_gentoo_database(newidpackage)
|
|
|
|
return rc
|
|
|
|
'''
|
|
@description: inject the database information into the Gentoo database
|
|
@output: 0 = all fine, !=0 = error!
|
|
'''
|
|
def __install_package_into_gentoo_database(self, newidpackage):
|
|
|
|
# handle gentoo-compat
|
|
_portage_avail = False
|
|
try:
|
|
import portageTools
|
|
_portage_avail = True
|
|
except:
|
|
return -1 # no Portage support
|
|
if (_portage_avail):
|
|
portDbDir = portageTools.getPortageAppDbPath()
|
|
# extract xpak from unpackDir+etpConst['packagecontentdir']+"/"+package
|
|
key = self.infoDict['category']+"/"+self.infoDict['name']
|
|
atomsfound = set()
|
|
dbdirs = os.listdir(portDbDir)
|
|
if self.infoDict['category'] in dbdirs:
|
|
catdirs = os.listdir(portDbDir+"/"+self.infoDict['category'])
|
|
dirsfound = set([self.infoDict['category']+"/"+x for x in catdirs if key == self.Entropy.entropyTools.dep_getkey(self.infoDict['category']+"/"+x)])
|
|
atomsfound.update(dirsfound)
|
|
|
|
### REMOVE
|
|
# parse slot and match and remove
|
|
if atomsfound:
|
|
pkgToRemove = ''
|
|
for atom in atomsfound:
|
|
atomslot = portageTools.getPackageSlot(atom)
|
|
# get slot from gentoo db
|
|
if atomslot == self.infoDict['slot']:
|
|
pkgToRemove = atom
|
|
break
|
|
if (pkgToRemove):
|
|
removePath = portDbDir+pkgToRemove
|
|
shutil.rmtree(removePath,True)
|
|
try:
|
|
os.rmdir(removePath)
|
|
except OSError:
|
|
pass
|
|
del atomsfound
|
|
|
|
# xpakstatus is perpared by unpackPackage()
|
|
# we now install it
|
|
if self.infoDict['xpakstatus'] != None and \
|
|
os.path.isdir(
|
|
self.infoDict['xpakpath'] + "/" + etpConst['entropyxpakdatarelativepath']
|
|
):
|
|
if not os.path.isdir(portDbDir+self.infoDict['category']):
|
|
os.makedirs(portDbDir+self.infoDict['category'])
|
|
destination = portDbDir+self.infoDict['category']+"/"+self.infoDict['name']+"-"+self.infoDict['version']
|
|
if os.path.isdir(destination):
|
|
shutil.rmtree(destination)
|
|
|
|
shutil.copytree(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'],destination)
|
|
|
|
# test if /var/cache/edb/counter is fine
|
|
if os.path.isfile(etpConst['edbcounter']):
|
|
try:
|
|
f = open(etpConst['edbcounter'],"r")
|
|
counter = int(f.readline().strip())
|
|
f.close()
|
|
except:
|
|
# need file recreation, parse gentoo tree
|
|
counter = portageTools.refillCounter()
|
|
else:
|
|
counter = portageTools.refillCounter()
|
|
|
|
# write new counter to file
|
|
if os.path.isdir(destination):
|
|
counter += 1
|
|
f = open(destination+"/"+dbCOUNTER,"w")
|
|
f.write(str(counter))
|
|
f.flush()
|
|
f.close()
|
|
f = open(etpConst['edbcounter'],"w")
|
|
f.write(str(counter))
|
|
f.flush()
|
|
f.close()
|
|
# update counter inside clientDatabase
|
|
self.Entropy.clientDbconn.insertCounter(newidpackage,counter)
|
|
else:
|
|
self.Entropy.updateProgress(
|
|
red("QA: ")+brown("cannot update Gentoo counter, destination %s does not exist." %
|
|
(destination,)
|
|
),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
|
|
return 0
|
|
|
|
'''
|
|
@description: injects package info into the installed packages database
|
|
@output: 0 = all fine, >0 = error!
|
|
'''
|
|
def __install_package_into_database(self):
|
|
|
|
# fetch info
|
|
dbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository'])
|
|
data = dbconn.getPackageData(self.infoDict['idpackage'])
|
|
# open client db
|
|
# always set data['injected'] to False
|
|
# installed packages database SHOULD never have more than one package for scope (key+slot)
|
|
data['injected'] = False
|
|
data['counter'] = -1 # gentoo counter will be set in self.__install_package_into_gentoo_database()
|
|
|
|
idpk, rev, x, status = self.Entropy.clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision'])
|
|
del x
|
|
del data
|
|
del status # if operation isn't successful, an error will be surely raised
|
|
|
|
# update datecreation
|
|
ctime = self.Entropy.entropyTools.getCurrentUnixTime()
|
|
self.Entropy.clientDbconn.setDateCreation(idpk, str(ctime))
|
|
|
|
# update world calculation cache
|
|
disk_cache = self.Entropy.dumpTools.loadobj(etpCache['world_update'])
|
|
if disk_cache != None:
|
|
try:
|
|
atom = self.Entropy.clientDbconn.retrieveAtom(idpk)
|
|
slot = self.Entropy.clientDbconn.retrieveSlot(idpk)
|
|
cached_item = (atom,(self.infoDict['idpackage'],self.infoDict['repository']))
|
|
if cached_item in disk_cache['update']:
|
|
disk_cache['update'].remove(cached_item)
|
|
db_digest = self.Entropy.clientDbconn.tablesChecksum()
|
|
disk_cache['db_digest'] = db_digest
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache)
|
|
else:
|
|
# add it, because can happen that a user firstly removes the package, then
|
|
# decides to install a specific version which is not the latest
|
|
key = self.Entropy.entropyTools.dep_getkey(atom)
|
|
match = self.Entropy.atomMatch(key, matchSlot = slot)
|
|
if (match[0] != -1) and (match != cached_item[1]):
|
|
ldbconn = self.Entropy.openRepositoryDatabase(match[1])
|
|
latest_atom = ldbconn.retrieveAtom(match[0])
|
|
disk_cache['update'].add(latest_atom,match)
|
|
# if you came from databaseTools.validateDatabase, this is the reason:
|
|
# db_digest is the checksum of baseinfo and extrainfo, if you add dependstable
|
|
# then you'll have to move this at the bottom of this function because, if you
|
|
# read below, dependstable will be updated, so checksum won't never match,
|
|
# the same for the condition above
|
|
db_digest = self.Entropy.clientDbconn.tablesChecksum()
|
|
disk_cache['db_digest'] = db_digest
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world_update'], disk_cache)
|
|
except Exception,e:
|
|
# invalidate cache
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world_update'],{})
|
|
print "DEBUG: please REPORT %s: %s" % (str(Exception),str(e),)
|
|
|
|
# add idpk to the installedtable
|
|
self.Entropy.clientDbconn.removePackageFromInstalledTable(idpk)
|
|
self.Entropy.clientDbconn.addPackageToInstalledTable(idpk,self.infoDict['repository'])
|
|
# update dependstable
|
|
try:
|
|
depends = self.Entropy.clientDbconn.listIdpackageDependencies(idpk)
|
|
for depend in depends:
|
|
atom = depend[1]
|
|
iddep = depend[0]
|
|
match = self.Entropy.clientDbconn.atomMatch(atom)
|
|
if (match[0] != -1):
|
|
self.Entropy.clientDbconn.removeDependencyFromDependsTable(iddep)
|
|
self.Entropy.clientDbconn.addDependRelationToDependsTable(iddep,match[0])
|
|
del depends
|
|
except:
|
|
self.Entropy.clientDbconn.regenerateDependsTable()
|
|
|
|
return idpk
|
|
|
|
def __move_image_to_system(self):
|
|
|
|
# load CONFIG_PROTECT and its mask
|
|
protect = etpRepositories[self.infoDict['repository']]['configprotect']
|
|
mask = etpRepositories[self.infoDict['repository']]['configprotectmask']
|
|
# setup imageDir properly
|
|
imageDir = self.infoDict['imagedir']
|
|
|
|
# merge data into system
|
|
for currentdir,subdirs,files in os.walk(imageDir):
|
|
# create subdirs
|
|
for subdir in subdirs:
|
|
|
|
imagepathDir = currentdir + "/" + subdir
|
|
rootdir = etpConst['systemroot']+imagepathDir[len(imageDir):]
|
|
|
|
# handle broken symlinks
|
|
if os.path.islink(rootdir) and not os.path.exists(rootdir):# broken symlink
|
|
os.remove(rootdir)
|
|
|
|
# if our directory is a file on the live system
|
|
elif os.path.isfile(rootdir): # really weird...!
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! "+rootdir+" is a file when it should be a directory !! Removing in 20 seconds...")
|
|
self.Entropy.updateProgress(
|
|
bold(rootdir)+red(" is a file when it should be a directory !! Removing in 20 seconds..."),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" *** ")
|
|
)
|
|
self.Entropy.entropyTools.ebeep(10)
|
|
time.sleep(20)
|
|
os.remove(rootdir)
|
|
|
|
# if our directory is a symlink instead, then copy the symlink
|
|
if os.path.islink(imagepathDir) and not os.path.isdir(rootdir): # for security we skip live items that are dirs
|
|
tolink = os.readlink(imagepathDir)
|
|
if os.path.islink(rootdir):
|
|
os.remove(rootdir)
|
|
os.symlink(tolink,rootdir)
|
|
elif (not os.path.isdir(rootdir)) and (not os.access(rootdir,os.R_OK)):
|
|
os.makedirs(rootdir)
|
|
|
|
if not os.path.islink(rootdir) and os.access(rootdir,os.W_OK):
|
|
# symlink don't need permissions, also until os.walk ends they might be broken
|
|
# XXX also, added os.access() check because there might be directories/files unwriteable
|
|
# what to do otherwise?
|
|
user = os.stat(imagepathDir)[4]
|
|
group = os.stat(imagepathDir)[5]
|
|
os.chown(rootdir,user,group)
|
|
shutil.copystat(imagepathDir,rootdir)
|
|
|
|
for item in files:
|
|
|
|
fromfile = currentdir+"/"+item
|
|
tofile = etpConst['systemroot']+fromfile[len(imageDir):]
|
|
fromfile_encoded = fromfile
|
|
tofile_encoded = tofile
|
|
# redecode to bytestring
|
|
fromfile = fromfile.encode('raw_unicode_escape')
|
|
tofile = tofile.encode('raw_unicode_escape')
|
|
|
|
if etpConst['collisionprotect'] > 1:
|
|
todbfile = fromfile[len(imageDir):]
|
|
if self.Entropy.clientDbconn.isFileAvailable(todbfile):
|
|
self.Entropy.updateProgress(
|
|
red("Collision found during install for ")+tofile+" - cannot overwrite",
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! Collision found during install for "+tofile+" - cannot overwrite")
|
|
continue
|
|
|
|
# -- CONFIGURATION FILE PROTECTION --
|
|
protected = False
|
|
tofile_before_protect = tofile
|
|
|
|
try:
|
|
for x in protect:
|
|
x = x.encode('raw_unicode_escape')
|
|
if tofile.startswith(x):
|
|
protected = True
|
|
break
|
|
if (protected): # check if perhaps, file is masked, so unprotected
|
|
for x in mask:
|
|
x = x.encode('raw_unicode_escape')
|
|
if tofile.startswith(x):
|
|
protected = False
|
|
break
|
|
|
|
if not os.path.lexists(tofile):
|
|
protected = False # file doesn't exist
|
|
|
|
# check if it's a text file
|
|
if (protected) and os.path.isfile(tofile):
|
|
protected = self.Entropy.entropyTools.istextfile(tofile)
|
|
else:
|
|
protected = False # it's not a file
|
|
|
|
# request new tofile then
|
|
if (protected):
|
|
if tofile not in etpConst['configprotectskip']:
|
|
tofile, prot_status = self.Entropy.entropyTools.allocateMaskedFile(tofile, fromfile)
|
|
if not prot_status:
|
|
protected = False
|
|
else:
|
|
oldtofile = tofile
|
|
if oldtofile.find("._cfg") != -1:
|
|
oldtofile = os.path.dirname(oldtofile)+"/"+os.path.basename(oldtofile)[10:]
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Protecting config file: "+oldtofile)
|
|
self.Entropy.updateProgress(
|
|
red("Protecting config file: ")+oldtofile,
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
else:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Skipping config file installation, as stated in equo.conf: "+tofile)
|
|
self.Entropy.updateProgress(
|
|
red("Skipping file installation: ")+tofile,
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
continue
|
|
|
|
# -- CONFIGURATION FILE PROTECTION --
|
|
|
|
except Exception, e:
|
|
protected = False # safely revert to false
|
|
tofile = tofile_before_protect
|
|
self.Entropy.updateProgress(
|
|
red("QA: ")+brown("cannot check CONFIG PROTECTION. Exception: %s :: error: %s" % (
|
|
str(Exception),
|
|
str(e),
|
|
)
|
|
),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" ## ")
|
|
)
|
|
pass # some files are buggy encoded
|
|
|
|
try:
|
|
|
|
if os.path.realpath(fromfile) == os.path.realpath(tofile) and os.path.islink(tofile):
|
|
# there is a serious issue here, better removing tofile, happened to someone:
|
|
'''
|
|
File \"/usr/lib/python2.4/shutil.py\", line 42, in copyfile
|
|
raise Error, \"`%s` and `%s` are the same file\" % (src, dst)
|
|
Error: `/var/tmp/entropy/packages/x86/3.5/mozilla-firefox-2.0.0.8.tbz2/image/usr/lib/mozilla-firefox/include/nsIURI.h` and `/usr/lib/mozilla-firefox/include/nsIURI.h` are the same file
|
|
'''
|
|
try: # try to cope...
|
|
os.remove(tofile)
|
|
except:
|
|
pass
|
|
|
|
# if our file is a dir on the live system
|
|
if os.path.isdir(tofile) and not os.path.islink(tofile): # really weird...!
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! "+tofile+" is a directory when it should be a file !! Removing in 20 seconds...")
|
|
self.Entropy.updateProgress(
|
|
bold(tofile)+red(" is a directory when it should be a file !! Removing in 20 seconds..."),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" *** ")
|
|
)
|
|
self.Entropy.entropyTools.ebeep(10)
|
|
time.sleep(20)
|
|
try:
|
|
shutil.rmtree(tofile, True)
|
|
os.rmdir(tofile)
|
|
except:
|
|
pass
|
|
try: # if it was a link
|
|
os.remove(tofile)
|
|
except OSError:
|
|
pass
|
|
|
|
# this also handles symlinks
|
|
# XXX
|
|
# XXX moving file using the raw format like portage does
|
|
# XXX
|
|
shutil.move(fromfile_encoded,tofile)
|
|
|
|
except IOError,(errno,strerror):
|
|
if errno == 2:
|
|
# better to pass away, sometimes gentoo packages are fucked up and contain broken things
|
|
pass
|
|
else:
|
|
rc = os.system("mv "+fromfile+" "+tofile)
|
|
if (rc != 0):
|
|
return 4
|
|
if (protected):
|
|
# add to disk cache
|
|
self.Entropy.FileUpdates.add_to_cache(tofile)
|
|
|
|
return 0
|
|
|
|
|
|
def fetch_step(self):
|
|
self.error_on_not_prepared()
|
|
self.Entropy.updateProgress(
|
|
blue("Fetching archive: ")+red(os.path.basename(self.infoDict['download'])),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
rc = self.Entropy.fetch_file_on_mirrors(
|
|
self.infoDict['repository'],
|
|
self.infoDict['download'],
|
|
self.infoDict['checksum'],
|
|
self.infoDict['verified']
|
|
)
|
|
if rc != 0:
|
|
self.Entropy.updateProgress(
|
|
red("Package cannot be fetched. Try to update repositories and retry. Error: %s" % (str(rc),)),
|
|
importance = 1,
|
|
type = "error",
|
|
header = darkred(" ## ")
|
|
)
|
|
return rc
|
|
return 0
|
|
|
|
def checksum_step(self):
|
|
self.error_on_not_prepared()
|
|
rc = self.match_checksum()
|
|
return rc
|
|
|
|
def unpack_step(self):
|
|
self.error_on_not_prepared()
|
|
self.Entropy.updateProgress(
|
|
blue("Unpacking package: ")+red(os.path.basename(self.infoDict['download'])),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
rc = self.__unpack_package()
|
|
if rc != 0:
|
|
if rc == 512:
|
|
errormsg = red("You are running out of disk space. I bet, you're probably Michele. Error 512")
|
|
else:
|
|
errormsg = red("An error occured while trying to unpack the package. Check if your system is healthy. Error "+str(rc))
|
|
self.Entropy.updateProgress(
|
|
errormsg,
|
|
importance = 1,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return rc
|
|
|
|
def install_step(self):
|
|
self.error_on_not_prepared()
|
|
compatstring = ''
|
|
if etpConst['gentoo-compat']:
|
|
compatstring = " ## w/Gentoo compatibility"
|
|
self.Entropy.updateProgress(
|
|
blue("Installing package: ")+red(self.infoDict['atom'])+compatstring,
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
rc = self.__install_package()
|
|
if rc != 0:
|
|
self.Entropy.updateProgress(
|
|
red("An error occured while trying to install the package. Check if your system is healthy. Error %s" % (str(rc),)),
|
|
importance = 1,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return rc
|
|
|
|
def remove_step(self):
|
|
self.error_on_not_prepared()
|
|
compatstring = ''
|
|
if etpConst['gentoo-compat']:
|
|
compatstring = " ## w/Gentoo compatibility"
|
|
self.Entropy.updateProgress(
|
|
blue("Removing data: ")+red(self.infoDict['removeatom'])+compatstring,
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
rc = self.__remove_package()
|
|
if rc != 0:
|
|
errormsg = red("An error occured while trying to remove the package. Check if you have enough disk space on your hard disk. Error %s " % (str(rc),))
|
|
self.Entropy.updateProgress(
|
|
errormsg,
|
|
importance = 1,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return rc
|
|
|
|
def cleanup_step(self):
|
|
self.error_on_not_prepared()
|
|
self.Entropy.updateProgress(
|
|
blue('Cleaning temporary files for archive: ')+red(self.infoDict['atom']),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" ## ")
|
|
)
|
|
tdict = {}
|
|
tdict['unpackdir'] = self.infoDict['unpackdir']
|
|
task = self.Entropy.entropyTools.parallelTask(self.__cleanup_package, tdict)
|
|
task.start()
|
|
# we don't care if cleanupPackage fails since it's not critical
|
|
return 0
|
|
|
|
def messages_step(self):
|
|
self.error_on_not_prepared()
|
|
# get messages
|
|
if self.infoDict['messages']:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Message from "+self.infoDict['atom']+" :")
|
|
self.Entropy.updateProgress(
|
|
darkgreen("Gentoo ebuild messages:"),
|
|
importance = 0,
|
|
type = "warning",
|
|
header = brown(" ## ")
|
|
)
|
|
for msg in self.infoDict['messages']:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,msg)
|
|
self.Entropy.updateProgress(
|
|
msg,
|
|
importance = 0,
|
|
type = "warning",
|
|
header = brown(" ## ")
|
|
)
|
|
if self.infoDict['messages']:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"End message.")
|
|
|
|
def postinstall_step(self):
|
|
self.error_on_not_prepared()
|
|
pkgdata = self.infoDict['triggers'].get('install')
|
|
if pkgdata:
|
|
Trigger = self.Entropy.Triggers('postinstall',pkgdata)
|
|
Trigger.prepare()
|
|
Trigger.run()
|
|
Trigger.kill()
|
|
del Trigger
|
|
del pkgdata
|
|
return 0
|
|
|
|
def preinstall_step(self):
|
|
self.error_on_not_prepared()
|
|
pkgdata = self.infoDict['triggers'].get('install')
|
|
if pkgdata:
|
|
|
|
Trigger = self.Entropy.Triggers('preinstall',pkgdata)
|
|
Trigger.prepare()
|
|
if (self.infoDict.get("diffremoval") != None): # diffremoval is true only when the remove action is triggered by installPackages()
|
|
if self.infoDict['diffremoval']:
|
|
remdata = self.infoDict['triggers'].get('remove')
|
|
if remdata:
|
|
rTrigger = self.Entropy.Triggers('preremove',remdata)
|
|
rTrigger.prepare()
|
|
Trigger.triggers = Trigger.triggers - rTrigger.triggers
|
|
rTrigger.kill()
|
|
del rTrigger
|
|
del remdata
|
|
Trigger.run()
|
|
Trigger.kill()
|
|
del Trigger
|
|
|
|
del pkgdata
|
|
return 0
|
|
|
|
def preremove_step(self):
|
|
self.error_on_not_prepared()
|
|
remdata = self.infoDict['triggers'].get('remove')
|
|
if remdata:
|
|
Trigger = self.Entropy.Triggers('preremove',remdata)
|
|
Trigger.prepare()
|
|
Trigger.run()
|
|
Trigger.kill()
|
|
del Trigger
|
|
del remdata
|
|
return 0
|
|
|
|
def postremove_step(self):
|
|
self.error_on_not_prepared()
|
|
remdata = self.infoDict['triggers'].get('remove')
|
|
if remdata:
|
|
|
|
Trigger = self.Entropy.Triggers('postremove',remdata)
|
|
Trigger.prepare()
|
|
if self.infoDict['diffremoval'] and (self.infoDict.get("atom") != None):
|
|
# diffremoval is true only when the remove action is triggered by installPackages()
|
|
pkgdata = self.infoDict['triggers'].get('install')
|
|
if pkgdata:
|
|
iTrigger = self.Entropy.Triggers('postinstall',pkgdata)
|
|
iTrigger.prepare()
|
|
Trigger.triggers = Trigger.triggers - iTrigger.triggers
|
|
iTrigger.kill()
|
|
del iTrigger
|
|
del pkgdata
|
|
Trigger.run()
|
|
Trigger.kill()
|
|
del Trigger
|
|
|
|
del remdata
|
|
return 0
|
|
|
|
|
|
'''
|
|
@description: execute the requested steps
|
|
@input xterm_header: purely optional
|
|
'''
|
|
def run(self, xterm_header = None):
|
|
self.error_on_not_prepared()
|
|
|
|
if xterm_header == None:
|
|
xterm_header = ""
|
|
|
|
rc = 0
|
|
for step in self.infoDict['steps']:
|
|
self.xterm_title = xterm_header+' '
|
|
|
|
if step == "fetch":
|
|
self.xterm_title += 'Fetching: '+os.path.basename(self.infoDict['download'])
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.fetch_step()
|
|
|
|
elif step == "checksum":
|
|
self.xterm_title += 'Verifying: '+os.path.basename(self.infoDict['download'])
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.checksum_step()
|
|
|
|
elif step == "unpack":
|
|
self.xterm_title += 'Unpacking: '+os.path.basename(self.infoDict['download'])
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.unpack_step()
|
|
|
|
elif step == "install":
|
|
self.xterm_title += 'Installing: '+self.infoDict['atom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.install_step()
|
|
|
|
elif step == "remove":
|
|
self.xterm_title += 'Removing: '+self.infoDict['removeatom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.remove_step()
|
|
|
|
elif step == "showmessages":
|
|
rc = self.messages_step()
|
|
|
|
elif step == "cleanup":
|
|
self.xterm_title += 'Cleaning: '+self.infoDict['atom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.cleanup_step()
|
|
|
|
elif step == "postinstall":
|
|
self.xterm_title += 'Postinstall: '+self.infoDict['atom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.postinstall_step()
|
|
|
|
elif step == "preinstall":
|
|
self.xterm_title += 'Preinstall: '+self.infoDict['atom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.preinstall_step()
|
|
|
|
elif step == "preremove":
|
|
self.xterm_title += 'Preremove: '+self.infoDict['removeatom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.preremove_step()
|
|
|
|
elif step == "postremove":
|
|
self.xterm_title += 'Postremove: '+self.infoDict['removeatom']
|
|
xtermTitle(self.xterm_title)
|
|
rc = self.postremove_step()
|
|
|
|
if rc != 0:
|
|
break
|
|
|
|
if rc != 0:
|
|
self.Entropy.updateProgress(
|
|
blue("An error occured. Action aborted."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = darkred(" ## ")
|
|
)
|
|
return rc
|
|
|
|
# XXX workaround for portage memleak - clear garbage
|
|
global _garbage_cycle
|
|
_garbage_cycle += 1
|
|
if _garbage_cycle > 15:
|
|
self.Entropy.gcTool.collect()
|
|
_garbage_cycle = 0
|
|
return rc
|
|
|
|
'''
|
|
Install/Removal process preparation function
|
|
- will generate all the metadata needed to run the action steps, creating infoDict automatically
|
|
@input matched_atom(tuple): is what is returned by EquoInstance.atomMatch:
|
|
(idpackage,repoid):
|
|
(2000,u'sabayonlinux.org')
|
|
NOTE: in case of remove action, matched_atom must be:
|
|
(idpackage,)
|
|
@input action(string): is an action to take, which must be one in self.valid_actions
|
|
'''
|
|
def prepare(self, matched_atom, action, metaopts = {}):
|
|
self.error_on_prepared()
|
|
self.check_action_validity(action)
|
|
|
|
self.action = action
|
|
self.matched_atom = matched_atom
|
|
self.metaopts = metaopts
|
|
# generate metadata dictionary
|
|
self.generate_metadata()
|
|
|
|
def generate_metadata(self):
|
|
self.error_on_prepared()
|
|
self.check_action_validity(self.action)
|
|
|
|
if self.action == "fetch":
|
|
self.__generate_fetch_metadata()
|
|
elif self.action == "remove":
|
|
self.__generate_remove_metadata()
|
|
elif self.action == "install":
|
|
self.__generate_install_metadata()
|
|
self.prepared = True
|
|
|
|
def __generate_remove_metadata(self):
|
|
idpackage = self.matched_atom[0]
|
|
self.infoDict.clear()
|
|
self.infoDict['triggers'] = {}
|
|
self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage)
|
|
self.infoDict['removeidpackage'] = idpackage
|
|
self.infoDict['diffremoval'] = False
|
|
removeConfig = False
|
|
if self.metaopts.has_key('removeconfig'):
|
|
removeConfig = self.metaopts.get('removeconfig')
|
|
self.infoDict['removeconfig'] = removeConfig
|
|
self.infoDict['removecontent'] = self.Entropy.clientDbconn.retrieveContent(idpackage)
|
|
self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getPackageData(idpackage)
|
|
self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent']
|
|
self.infoDict['steps'] = []
|
|
self.infoDict['steps'].append("preremove")
|
|
self.infoDict['steps'].append("remove")
|
|
self.infoDict['steps'].append("postremove")
|
|
return 0
|
|
|
|
def __generate_install_metadata(self):
|
|
self.infoDict.clear()
|
|
|
|
idpackage = self.matched_atom[0]
|
|
repository = self.matched_atom[1]
|
|
self.infoDict['idpackage'] = idpackage
|
|
self.infoDict['repository'] = repository
|
|
# get package atom
|
|
dbconn = self.Entropy.openRepositoryDatabase(repository)
|
|
self.infoDict['triggers'] = {}
|
|
self.infoDict['atom'] = dbconn.retrieveAtom(idpackage)
|
|
self.infoDict['slot'] = dbconn.retrieveSlot(idpackage)
|
|
self.infoDict['version'] = dbconn.retrieveVersion(idpackage)
|
|
self.infoDict['versiontag'] = dbconn.retrieveVersionTag(idpackage)
|
|
self.infoDict['revision'] = dbconn.retrieveRevision(idpackage)
|
|
self.infoDict['category'] = dbconn.retrieveCategory(idpackage)
|
|
self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage)
|
|
self.infoDict['name'] = dbconn.retrieveName(idpackage)
|
|
self.infoDict['messages'] = dbconn.retrieveMessages(idpackage)
|
|
self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage)
|
|
# fill action queue
|
|
self.infoDict['removeidpackage'] = -1
|
|
removeConfig = False
|
|
if self.metaopts.has_key('removeconfig'):
|
|
removeConfig = self.metaopts.get('removeconfig')
|
|
self.infoDict['removeconfig'] = removeConfig
|
|
self.infoDict['removeidpackage'] = self.Entropy.retrieveInstalledIdPackage(
|
|
self.Entropy.entropyTools.dep_getkey(self.infoDict['atom']),
|
|
self.infoDict['slot']
|
|
)
|
|
if self.infoDict['removeidpackage'] != -1:
|
|
self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage'])
|
|
|
|
# smartpackage ?
|
|
self.infoDict['smartpackage'] = False
|
|
# set unpack dir and image dir
|
|
if self.infoDict['repository'].endswith(".tbz2"):
|
|
# do arch check
|
|
compiled_arch = dbconn.retrieveDownloadURL(idpackage)
|
|
if compiled_arch.find("/"+etpSys['arch']+"/") == -1:
|
|
self.infoDict.clear()
|
|
self.prepared = False
|
|
return -1
|
|
self.infoDict['smartpackage'] = etpRepositories[self.infoDict['repository']]['smartpackage']
|
|
self.infoDict['pkgpath'] = etpRepositories[self.infoDict['repository']]['pkgpath']
|
|
else:
|
|
self.infoDict['pkgpath'] = etpConst['entropyworkdir']+"/"+self.infoDict['download']
|
|
self.infoDict['unpackdir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']
|
|
self.infoDict['imagedir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyimagerelativepath']
|
|
|
|
# gentoo xpak data
|
|
if etpConst['gentoo-compat']:
|
|
self.infoDict['xpakstatus'] = None
|
|
self.infoDict['xpakpath'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyxpakrelativepath']
|
|
self.infoDict['xpakdir'] = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
|
|
|
|
# compare both versions and if they match, disable removeidpackage
|
|
if self.infoDict['removeidpackage'] != -1:
|
|
installedVer = self.Entropy.clientDbconn.retrieveVersion(self.infoDict['removeidpackage'])
|
|
installedTag = self.Entropy.clientDbconn.retrieveVersionTag(self.infoDict['removeidpackage'])
|
|
installedRev = self.Entropy.clientDbconn.retrieveRevision(self.infoDict['removeidpackage'])
|
|
pkgcmp = self.Entropy.entropyTools.entropyCompareVersions(
|
|
(self.infoDict['version'],self.infoDict['versiontag'],self.infoDict['revision']),
|
|
(installedVer,installedTag,installedRev)
|
|
)
|
|
if pkgcmp == 0:
|
|
self.infoDict['removeidpackage'] = -1
|
|
del pkgcmp
|
|
|
|
# differential remove list
|
|
if self.infoDict['removeidpackage'] != -1:
|
|
# is it still available?
|
|
if self.Entropy.clientDbconn.isIDPackageAvailable(self.infoDict['removeidpackage']):
|
|
self.infoDict['diffremoval'] = True
|
|
self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage'])
|
|
newcontent = dbconn.retrieveContent(idpackage)
|
|
content_to_remove = self.Entropy.clientDbconn.contentDiff(self.infoDict['removeidpackage'],newcontent)
|
|
del newcontent
|
|
self.infoDict['removecontent'] = content_to_remove.copy()
|
|
del content_to_remove
|
|
# XXX: too much memory
|
|
self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getPackageData(self.infoDict['removeidpackage'])
|
|
self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent']
|
|
else:
|
|
self.infoDict['removeidpackage'] = -1
|
|
|
|
# set steps
|
|
self.infoDict['steps'] = []
|
|
# install
|
|
if (self.infoDict['removeidpackage'] != -1):
|
|
self.infoDict['steps'].append("preremove")
|
|
self.infoDict['steps'].append("unpack")
|
|
self.infoDict['steps'].append("preinstall")
|
|
self.infoDict['steps'].append("install")
|
|
if (self.infoDict['removeidpackage'] != -1):
|
|
self.infoDict['steps'].append("postremove")
|
|
self.infoDict['steps'].append("postinstall")
|
|
if not etpConst['gentoo-compat']: # otherwise gentoo triggers will show that
|
|
self.infoDict['steps'].append("showmessages")
|
|
self.infoDict['steps'].append("cleanup")
|
|
|
|
# XXX: too much memory used for this
|
|
self.infoDict['triggers']['install'] = dbconn.getPackageData(idpackage)
|
|
self.infoDict['triggers']['install']['unpackdir'] = self.infoDict['unpackdir']
|
|
if etpConst['gentoo-compat']:
|
|
self.infoDict['triggers']['install']['xpakpath'] = self.infoDict['xpakpath']
|
|
self.infoDict['triggers']['install']['xpakdir'] = self.infoDict['xpakdir']
|
|
|
|
return 0
|
|
|
|
|
|
def __generate_fetch_metadata(self):
|
|
self.infoDict.clear()
|
|
|
|
idpackage = self.matched_atom[0]
|
|
repository = self.matched_atom[1]
|
|
dochecksum = True
|
|
if self.metaopts.has_key('dochecksum'):
|
|
dochecksum = self.metaopts.get('dochecksum')
|
|
self.infoDict['repository'] = repository
|
|
self.infoDict['idpackage'] = idpackage
|
|
dbconn = self.Entropy.openRepositoryDatabase(repository)
|
|
self.infoDict['atom'] = dbconn.retrieveAtom(idpackage)
|
|
self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage)
|
|
self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage)
|
|
self.infoDict['verified'] = False
|
|
self.infoDict['steps'] = []
|
|
if not repository.endswith(".tbz2"):
|
|
if self.Entropy.check_needed_package_download(self.infoDict['download'], None) < 0:
|
|
self.infoDict['steps'].append("fetch")
|
|
if dochecksum:
|
|
self.infoDict['steps'].append("checksum")
|
|
# if file exists, first checksum then fetch
|
|
if os.path.isfile(os.path.join(etpConst['entropyworkdir'],self.infoDict['download'])):
|
|
# check size first
|
|
repo_size = dbconn.retrieveSize(idpackage)
|
|
f = open(os.path.join(etpConst['entropyworkdir'],self.infoDict['download']),"r")
|
|
f.seek(0,2)
|
|
disk_size = f.tell()
|
|
f.close()
|
|
if repo_size == disk_size:
|
|
self.infoDict['steps'].reverse()
|
|
return 0
|
|
|
|
class FileUpdatesInterface:
|
|
|
|
def __init__(self, EquoInstance = None):
|
|
|
|
if EquoInstance == None:
|
|
self.Entropy = TextInterface()
|
|
import dumpTools
|
|
self.Entropy.dumpTools = dumpTools
|
|
else:
|
|
self.Entropy = EquoInstance
|
|
try:
|
|
self.Entropy.instanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
|
|
|
|
'''
|
|
@description: scan for files that need to be merged
|
|
@output: dictionary using filename as key
|
|
'''
|
|
def scanfs(self, dcache = True):
|
|
|
|
if (dcache):
|
|
# can we load cache?
|
|
try:
|
|
z = self.load_cache()
|
|
if z != None:
|
|
return z
|
|
except:
|
|
pass
|
|
|
|
# open client database to fill etpConst['dbconfigprotect']
|
|
scandata = {}
|
|
counter = 0
|
|
for path in etpConst['dbconfigprotect']:
|
|
# it's a file?
|
|
scanfile = False
|
|
if os.path.isfile(path):
|
|
# find inside basename
|
|
path = os.path.dirname(path)
|
|
scanfile = True
|
|
|
|
for currentdir,subdirs,files in os.walk(path):
|
|
for item in files:
|
|
|
|
if (scanfile):
|
|
if path != item:
|
|
continue
|
|
|
|
filepath = currentdir+"/"+item
|
|
if item.startswith("._cfg"):
|
|
|
|
# further check then
|
|
number = item[5:9]
|
|
try:
|
|
int(number)
|
|
except:
|
|
continue # not a valid etc-update file
|
|
if item[9] != "_": # no valid format provided
|
|
continue
|
|
|
|
mydict = self.generate_dict(filepath)
|
|
if mydict['automerge']:
|
|
self.Entropy.updateProgress(
|
|
darkred("Automerging file: %s") % ( darkgreen(etpConst['systemroot']+mydict['source']) ),
|
|
importance = 0,
|
|
type = "info"
|
|
)
|
|
if os.path.isfile(etpConst['systemroot']+mydict['source']):
|
|
try:
|
|
shutil.move(etpConst['systemroot']+mydict['source'],etpConst['systemroot']+mydict['destination'])
|
|
except IOError:
|
|
self.Entropy.updateProgress(
|
|
darkred("I/O Error :: Cannot automerge file: %s") % ( darkgreen(etpConst['systemroot']+mydict['source']) ),
|
|
importance = 1,
|
|
type = "warning"
|
|
)
|
|
continue
|
|
else:
|
|
counter += 1
|
|
scandata[counter] = mydict.copy()
|
|
|
|
try:
|
|
self.Entropy.updateProgress(
|
|
"("+blue(str(counter))+") "+red(" file: ")+os.path.dirname(filepath)+"/"+os.path.basename(filepath)[10:],
|
|
importance = 1,
|
|
type = "info"
|
|
)
|
|
except:
|
|
pass # possible encoding issues
|
|
# store data
|
|
try:
|
|
self.Entropy.dumpTools.dumpobj(etpCache['configfiles'],scandata)
|
|
except:
|
|
pass
|
|
return scandata
|
|
|
|
def load_cache(self):
|
|
try:
|
|
sd = self.Entropy.dumpTools.loadobj(etpCache['configfiles'])
|
|
# check for corruption?
|
|
if isinstance(sd, dict):
|
|
# quick test if data is reliable
|
|
try:
|
|
taint = False
|
|
for x in sd:
|
|
if not os.path.isfile(etpConst['systemroot']+sd[x]['source']):
|
|
taint = True
|
|
break
|
|
if (not taint):
|
|
return sd
|
|
else:
|
|
raise exceptionTools.CacheCorruptionError("CacheCorruptionError: cache is corrupted.")
|
|
except:
|
|
raise exceptionTools.CacheCorruptionError("CacheCorruptionError: cache is corrupted.")
|
|
else:
|
|
raise exceptionTools.CacheCorruptionError("CacheCorruptionError: cache is corrupted.")
|
|
except:
|
|
raise exceptionTools.CacheCorruptionError("CacheCorruptionError: cache is corrupted.")
|
|
|
|
'''
|
|
@description: prints information about config files that should be updated
|
|
@attention: please be sure that filepath is properly formatted before using this function
|
|
'''
|
|
def add_to_cache(self, filepath):
|
|
try:
|
|
scandata = self.load_cache()
|
|
except:
|
|
scandata = self.scanfs(dcache = False)
|
|
keys = scandata.keys()
|
|
try:
|
|
for key in keys:
|
|
if scandata[key]['source'] == filepath[len(etpConst['systemroot']):]:
|
|
del scandata[key]
|
|
except:
|
|
pass
|
|
# get next counter
|
|
if keys:
|
|
keys.sort()
|
|
index = keys[-1]
|
|
else:
|
|
index = 0
|
|
index += 1
|
|
mydata = self.generate_dict(filepath)
|
|
scandata[index] = mydata.copy()
|
|
try:
|
|
self.Entropy.dumpTools.dumpobj(etpCache['configfiles'],scandata)
|
|
except:
|
|
pass
|
|
|
|
def remove_from_cache(self, sd, key):
|
|
try:
|
|
del sd[key]
|
|
except:
|
|
pass
|
|
self.Entropy.dumpTools.dumpobj(etpCache['configfiles'],sd)
|
|
return sd
|
|
|
|
def generate_dict(self, filepath):
|
|
|
|
item = os.path.basename(filepath)
|
|
currentdir = os.path.dirname(filepath)
|
|
tofile = item[10:]
|
|
number = item[5:9]
|
|
try:
|
|
int(number)
|
|
except:
|
|
raise exceptionTools.InvalidDataType("InvalidDataType: invalid config file number '0000->9999'.")
|
|
tofilepath = currentdir+"/"+tofile
|
|
mydict = {}
|
|
mydict['revision'] = number
|
|
mydict['destination'] = tofilepath[len(etpConst['systemroot']):]
|
|
mydict['source'] = filepath[len(etpConst['systemroot']):]
|
|
mydict['automerge'] = False
|
|
if not os.path.isfile(tofilepath):
|
|
mydict['automerge'] = True
|
|
if (not mydict['automerge']):
|
|
# is it trivial?
|
|
try:
|
|
if not os.path.lexists(filepath): # if file does not even exist
|
|
return mydict
|
|
if os.path.islink(filepath):
|
|
# if it's broken, skip diff and automerge
|
|
if not os.path.exists(filepath):
|
|
return mydict
|
|
result = commands.getoutput('diff -Nua '+filepath+' '+tofilepath+' | grep "^[+-][^+-]" | grep -v \'# .Header:.*\'')
|
|
if not result:
|
|
mydict['automerge'] = True
|
|
except:
|
|
pass
|
|
# another test
|
|
if (not mydict['automerge']):
|
|
try:
|
|
if not os.path.lexists(filepath): # if file does not even exist
|
|
return mydict
|
|
if os.path.islink(filepath):
|
|
# if it's broken, skip diff and automerge
|
|
if not os.path.exists(filepath):
|
|
return mydict
|
|
result = os.system('diff -Bbua '+filepath+' '+tofilepath+' | egrep \'^[+-]\' | egrep -v \'^[+-][\t ]*#|^--- |^\+\+\+ \' | egrep -qv \'^[-+][\t ]*$\'')
|
|
if result == 1:
|
|
mydict['automerge'] = True
|
|
except:
|
|
pass
|
|
return mydict
|
|
|
|
#
|
|
# repository control class, that's it
|
|
#
|
|
class RepoInterface:
|
|
|
|
def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False):
|
|
|
|
self.Entropy = EquoInstance
|
|
try:
|
|
self.Entropy.instanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
|
|
|
|
self.reponames = reponames
|
|
self.forceUpdate = forceUpdate
|
|
self.syncErrors = False
|
|
self.dbupdated = False
|
|
self.newEquo = False
|
|
self.noEquoCheck = noEquoCheck
|
|
self.alreadyUpdated = 0
|
|
self.notAvailable = 0
|
|
|
|
# check if I am root
|
|
if (not self.Entropy.entropyTools.isRoot()):
|
|
raise exceptionTools.PermissionDenied("PermissionDenied: not allowed as user.")
|
|
|
|
# check etpRepositories
|
|
if not etpRepositories:
|
|
raise exceptionTools.MissingParameter("MissingParameter: no repositories specified in %s" % (etpConst['repositoriesconf'],))
|
|
|
|
# Test network connectivity
|
|
conntest = self.Entropy.entropyTools.get_remote_data(etpConst['conntestlink'])
|
|
if not conntest:
|
|
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: Cannot connect to %s" % (etpConst['conntestlink'],))
|
|
|
|
if not self.reponames:
|
|
for x in etpRepositories:
|
|
self.reponames.append(x)
|
|
|
|
def __validate_repository_id(self, repoid):
|
|
if repoid not in self.reponames:
|
|
raise exceptionTools.InvalidData("InvalidData: repository is not listed in self.reponames")
|
|
|
|
def __validate_compression_method(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat'])
|
|
if cmethod == None:
|
|
raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
|
|
return cmethod
|
|
|
|
def __ensure_repository_path(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
# create dir if it doesn't exist
|
|
if not os.path.isdir(etpRepositories[repo]['dbpath']):
|
|
os.makedirs(etpRepositories[repo]['dbpath'])
|
|
|
|
def __construct_paths(self, item, repo, cmethod):
|
|
|
|
if item not in ("db","rev","ck", "lock"):
|
|
raise exceptionTools.InvalidData("InvalidData: supported db, rev, ck, lock")
|
|
|
|
if item == "db":
|
|
if cmethod == None:
|
|
raise exceptionTools.InvalidData("InvalidData: for db, cmethod can't be None")
|
|
url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[2]]
|
|
filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[2]]
|
|
elif item == "rev":
|
|
url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile']
|
|
filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile']
|
|
elif item == "ck":
|
|
url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile']
|
|
filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile']
|
|
elif item == "lock":
|
|
url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile']
|
|
filepath = "/dev/null"
|
|
|
|
return url, filepath
|
|
|
|
def __remove_repository_files(self, repo, dbfilenameid):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
|
|
os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'])
|
|
if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]):
|
|
os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid])
|
|
if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
|
|
os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
|
|
|
|
def __unpack_downloaded_database(self, repo, cmethod):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
path = eval("self.Entropy.entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]])
|
|
return path
|
|
|
|
def __verify_database_checksum(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
try:
|
|
f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
|
|
md5hash = f.readline().strip()
|
|
md5hash = md5hash.split()[0]
|
|
f.close()
|
|
except:
|
|
return -1
|
|
rc = self.Entropy.entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash)
|
|
return rc
|
|
|
|
# @returns -1 if the file is not available
|
|
# @returns int>0 if the revision has been retrieved
|
|
def get_online_repository_revision(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile']
|
|
status = self.Entropy.entropyTools.get_remote_data(url)
|
|
if (status):
|
|
status = status[0].strip()
|
|
return int(status)
|
|
else:
|
|
return -1
|
|
|
|
def is_repository_updatable(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
onlinestatus = self.get_online_repository_revision(repo)
|
|
if (onlinestatus != -1):
|
|
localstatus = self.Entropy.get_repository_revision(repo)
|
|
if (localstatus == onlinestatus) and (not self.forceUpdate):
|
|
return False
|
|
return True
|
|
|
|
def is_repository_unlocked(self, repo):
|
|
|
|
self.__validate_repository_id(repo)
|
|
|
|
rc = self.download_item("lock", repo)
|
|
if rc: # cannot download database
|
|
self.syncErrors = True
|
|
return False
|
|
return True
|
|
|
|
def clear_repository_cache(self, repo):
|
|
self.__validate_repository_id(repo)
|
|
self.Entropy.dumpTools.dumpobj(etpCache['dbInfo']+repo,{})
|
|
|
|
# this function can be reimplemented
|
|
def download_item(self, item, repo, cmethod = None):
|
|
|
|
self.__validate_repository_id(repo)
|
|
url, filepath = self.__construct_paths(item, repo, cmethod)
|
|
|
|
fetchConn = self.Entropy.urlFetcher(url, filepath, resume = False)
|
|
fetchConn.progress = self.Entropy.progress
|
|
rc = fetchConn.download()
|
|
del fetchConn
|
|
if rc in ("-1","-2","-3"):
|
|
return False
|
|
return True
|
|
|
|
def close_transactions(self):
|
|
|
|
if not self.dbupdated:
|
|
return
|
|
|
|
# safely clean ram caches
|
|
atomMatchCache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['atomMatch'],{})
|
|
generateDependsTreeCache.clear()
|
|
self.Entropy.dumpTools.dumpobj(etpCache['generateDependsTree'],{})
|
|
for dbinfo in dbCacheStore:
|
|
dbCacheStore[dbinfo].clear()
|
|
self.Entropy.dumpTools.dumpobj(dbinfo,{})
|
|
|
|
# clean resume caches
|
|
self.Entropy.dumpTools.dumpobj(etpCache['install'],{})
|
|
self.Entropy.dumpTools.dumpobj(etpCache['world'],{})
|
|
self.Entropy.dumpTools.dumpobj(etpCache['remove'],[])
|
|
|
|
def sync(self):
|
|
|
|
# close them
|
|
self.Entropy.closeAllRepositoryDatabases()
|
|
|
|
# let's dance!
|
|
self.Entropy.updateProgress( darkgreen("Repositories syncronization..."),
|
|
importance = 2,
|
|
type = "info",
|
|
header = darkred(" @@ ")
|
|
)
|
|
|
|
self.dbupdated = False
|
|
repocount = 0
|
|
repolength = len(self.reponames)
|
|
for repo in self.reponames:
|
|
|
|
repocount += 1
|
|
|
|
self.Entropy.updateProgress( bold("%s") % ( etpRepositories[repo]['description'] ),
|
|
importance = 2,
|
|
type = "info",
|
|
count = (repocount, repolength),
|
|
header = blue(" # ")
|
|
)
|
|
self.Entropy.updateProgress( red("Database URL: ") + darkgreen(etpRepositories[repo]['database']),
|
|
importance = 1,
|
|
type = "info",
|
|
header = blue(" # ")
|
|
)
|
|
self.Entropy.updateProgress( red("Database local path: ") + darkgreen(etpRepositories[repo]['dbpath']),
|
|
importance = 0,
|
|
type = "info",
|
|
header = blue(" # ")
|
|
)
|
|
|
|
# check if database is already updated to the latest revision
|
|
update = self.is_repository_updatable(repo)
|
|
if not update:
|
|
self.Entropy.updateProgress( bold("Attention: ") + red("database is already up to date."),
|
|
importance = 1,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
self.Entropy.cycleDone()
|
|
self.alreadyUpdated += 1
|
|
continue
|
|
|
|
# get database lock
|
|
unlocked = self.is_repository_unlocked(repo)
|
|
if not unlocked:
|
|
self.Entropy.updateProgress( bold("Attention: ") + red("repository is being updated. Try again in a few minutes."),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = "\t"
|
|
)
|
|
self.Entropy.cycleDone()
|
|
continue
|
|
|
|
# clear database interface cache belonging to this repository
|
|
self.clear_repository_cache(repo)
|
|
cmethod = self.__validate_compression_method(repo)
|
|
self.__ensure_repository_path(repo)
|
|
|
|
# starting to download
|
|
self.Entropy.updateProgress( red("Downloading database ") + darkgreen(etpConst[cmethod[2]])+red(" ..."),
|
|
importance = 1,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
down_status = self.download_item("db", repo, cmethod)
|
|
if not down_status:
|
|
self.Entropy.updateProgress( bold("Attention: ") + red("database does not exist online."),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = "\t"
|
|
)
|
|
self.Entropy.cycleDone()
|
|
self.notAvailable += 1
|
|
continue
|
|
|
|
# database is going to be updated
|
|
self.dbupdated = True
|
|
|
|
# unpack database
|
|
self.Entropy.updateProgress( red("Unpacking database to ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
# unpack database
|
|
self.__unpack_downloaded_database(repo, cmethod)
|
|
|
|
# download checksum
|
|
self.Entropy.updateProgress( red("Downloading checksum ") + darkgreen(etpConst['etpdatabasehashfile'])+red(" ..."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
down_status = self.download_item("ck", repo)
|
|
if not down_status:
|
|
self.Entropy.updateProgress( red("Cannot fetch checksum. Cannot verify database integrity !"),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = "\t"
|
|
)
|
|
else:
|
|
# verify checksum
|
|
self.Entropy.updateProgress( red("Checking downloaded database ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."),
|
|
importance = 0,
|
|
back = True,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
db_status = self.__verify_database_checksum(repo)
|
|
if db_status == -1:
|
|
self.Entropy.updateProgress( red("Cannot open digest. Cannot verify database integrity !"),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = "\t"
|
|
)
|
|
elif db_status:
|
|
self.Entropy.updateProgress( red("Downloaded database status: ")+bold("OK"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
else:
|
|
self.Entropy.updateProgress( red("Downloaded database status: ")+darkred("ERROR"),
|
|
importance = 1,
|
|
type = "error",
|
|
header = "\t"
|
|
)
|
|
self.Entropy.updateProgress( red("An error occured while checking database integrity. Giving up."),
|
|
importance = 1,
|
|
type = "error",
|
|
header = "\t"
|
|
)
|
|
# delete all
|
|
self.__remove_repository_files(repo, cmethod[2])
|
|
self.syncErrors = True
|
|
self.Entropy.cycleDone()
|
|
continue
|
|
|
|
# download revision
|
|
self.Entropy.updateProgress( red("Downloading revision ")+darkgreen(etpConst['etpdatabaserevisionfile'])+red(" ..."),
|
|
importance = 0,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
rev_status = self.download_item("rev", repo)
|
|
if not rev_status:
|
|
self.Entropy.updateProgress( red("Cannot download repository revision, don't ask me why !"),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = "\t"
|
|
)
|
|
else:
|
|
self.Entropy.updateProgress( red("Updated repository revision: ")+bold(str(self.Entropy.get_repository_revision(repo))),
|
|
importance = 1,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
self.Entropy.updateProgress(
|
|
red("Indexing Repository metadata ..."),
|
|
importance = 1,
|
|
type = "info",
|
|
header = "\t"
|
|
)
|
|
dbconn = self.Entropy.openRepositoryDatabase(repo)
|
|
dbconn.createAllIndexes()
|
|
|
|
self.Entropy.cycleDone()
|
|
|
|
self.close_transactions()
|
|
|
|
# keep them closed
|
|
self.Entropy.closeAllRepositoryDatabases()
|
|
|
|
# clean caches
|
|
if self.dbupdated:
|
|
self.Entropy.generate_cache(depcache = True, configcache = False)
|
|
# update Security Advisories
|
|
self.Entropy.Security.fetch_advisories()
|
|
|
|
if self.syncErrors:
|
|
self.Entropy.updateProgress( red("Something bad happened. Please have a look."),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = darkred(" @@ ")
|
|
)
|
|
self.syncErrors = True
|
|
return 128
|
|
|
|
rc = False
|
|
if not self.noEquoCheck:
|
|
try:
|
|
rc = self.Entropy.check_equo_updates()
|
|
except:
|
|
pass
|
|
|
|
if rc:
|
|
self.newEquo = True
|
|
self.Entropy.updateProgress( blue("A new ")+bold("Equo")+blue(" release is available. Please ")+bold("install it")+blue(" before any other package."),
|
|
importance = 1,
|
|
type = "info",
|
|
header = darkred(" !! ")
|
|
)
|
|
|
|
if (self.notAvailable >= len(self.reponames)):
|
|
return 2
|
|
elif (self.notAvailable > 0):
|
|
return 1
|
|
|
|
return 0
|
|
|
|
'''
|
|
Entropy FTP interface
|
|
'''
|
|
class FtpInterface:
|
|
|
|
# this must be run before calling the other functions
|
|
def __init__(self, ftpuri, EntropyInterface):
|
|
|
|
self.Entropy = EntropyInterface
|
|
try:
|
|
self.Entropy.outputInstanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based Instance is needed")
|
|
|
|
import entropyTools
|
|
self.entropyTools = entropyTools
|
|
import ftplib
|
|
self.ftplib = ftplib
|
|
|
|
self.oldprogress = 0.0
|
|
|
|
# import FTP modules
|
|
socket.setdefaulttimeout(60)
|
|
|
|
self.ftpuri = ftpuri
|
|
self.ftphost = self.entropyTools.extractFTPHostFromUri(self.ftpuri)
|
|
|
|
self.ftpuser = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1].split(":")[0]
|
|
if (self.ftpuser == ""):
|
|
self.ftpuser = "anonymous@"
|
|
self.ftppassword = "anonymous"
|
|
else:
|
|
self.ftppassword = ftpuri.split("@")[:len(ftpuri.split("@"))-1]
|
|
if len(self.ftppassword) > 1:
|
|
self.ftppassword = '@'.join(self.ftppassword)
|
|
self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1]
|
|
if (self.ftppassword == ""):
|
|
self.ftppassword = "anonymous"
|
|
else:
|
|
self.ftppassword = self.ftppassword[0]
|
|
self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1]
|
|
if (self.ftppassword == ""):
|
|
self.ftppassword = "anonymous"
|
|
|
|
self.ftpport = ftpuri.split(":")[len(ftpuri.split(":"))-1]
|
|
try:
|
|
self.ftpport = int(self.ftpport)
|
|
except:
|
|
self.ftpport = 21
|
|
|
|
self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1]
|
|
self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1]
|
|
self.ftpdir = self.ftpdir.split(":")[0]
|
|
if self.ftpdir.endswith("/"):
|
|
self.ftpdir = self.ftpdir[:len(self.ftpdir)-1]
|
|
if self.ftpdir == "":
|
|
self.ftpdir = "/"
|
|
|
|
count = 10
|
|
while 1:
|
|
count -= 1
|
|
try:
|
|
self.ftpconn = self.ftplib.FTP(self.ftphost)
|
|
break
|
|
except:
|
|
if not count:
|
|
raise
|
|
continue
|
|
|
|
self.ftpconn.login(self.ftpuser,self.ftppassword)
|
|
# change to our dir
|
|
self.ftpconn.cwd(self.ftpdir)
|
|
self.currentdir = self.ftpdir
|
|
|
|
# this can be used in case of exceptions
|
|
def reconnectHost(self):
|
|
# import FTP modules
|
|
socket.setdefaulttimeout(60)
|
|
counter = 10
|
|
while 1:
|
|
counter -= 1
|
|
try:
|
|
self.ftpconn = self.ftplib.FTP(self.ftphost)
|
|
break
|
|
except:
|
|
if not counter:
|
|
raise
|
|
continue
|
|
self.ftpconn.login(self.ftpuser,self.ftppassword)
|
|
# save curr dir
|
|
#cur = self.currentdir
|
|
#self.setCWD(self.ftpdir)
|
|
self.setCWD(self.currentdir)
|
|
|
|
def getHost(self):
|
|
return self.ftphost
|
|
|
|
def getPort(self):
|
|
return self.ftpport
|
|
|
|
def getDir(self):
|
|
return self.ftpdir
|
|
|
|
def getCWD(self):
|
|
pwd = self.ftpconn.pwd()
|
|
return pwd
|
|
|
|
def setCWD(self,dir):
|
|
self.ftpconn.cwd(dir)
|
|
self.currentdir = self.getCWD()
|
|
|
|
def setPASV(self,bool):
|
|
self.ftpconn.set_pasv(bool)
|
|
|
|
def setChmod(self,chmodvalue,file):
|
|
return self.ftpconn.voidcmd("SITE CHMOD "+str(chmodvalue)+" "+str(file))
|
|
|
|
def getFileMtime(self,path):
|
|
rc = self.ftpconn.sendcmd("mdtm "+path)
|
|
return rc.split()[len(rc.split())-1]
|
|
|
|
def spawnCommand(self,cmd):
|
|
return self.ftpconn.sendcmd(cmd)
|
|
|
|
# list files and directory of a FTP
|
|
# @returns a list
|
|
def listDir(self):
|
|
# directory is: self.ftpdir
|
|
try:
|
|
rc = self.ftpconn.nlst()
|
|
_rc = []
|
|
for i in rc:
|
|
_rc.append(i.split("/")[len(i.split("/"))-1])
|
|
rc = _rc
|
|
except:
|
|
return []
|
|
return rc
|
|
|
|
# list if the file is available
|
|
# @returns True or False
|
|
def isFileAvailable(self,filename):
|
|
# directory is: self.ftpdir
|
|
try:
|
|
rc = self.ftpconn.nlst()
|
|
_rc = []
|
|
for i in rc:
|
|
_rc.append(i.split("/")[len(i.split("/"))-1])
|
|
rc = _rc
|
|
for i in rc:
|
|
if i == filename:
|
|
return True
|
|
return False
|
|
except:
|
|
return False
|
|
|
|
def deleteFile(self,file):
|
|
try:
|
|
rc = self.ftpconn.delete(file)
|
|
if rc.startswith("250"):
|
|
return True
|
|
else:
|
|
return False
|
|
except:
|
|
return False
|
|
|
|
def mkdir(self,directory):
|
|
# FIXME: add rc
|
|
self.ftpconn.mkd(directory)
|
|
|
|
# this function also supports callback, because storbinary doesn't
|
|
def advancedStorBinary(self, cmd, fp, callback=None):
|
|
''' Store a file in binary mode. Our version supports a callback function'''
|
|
self.ftpconn.voidcmd('TYPE I')
|
|
conn = self.ftpconn.transfercmd(cmd)
|
|
while 1:
|
|
buf = fp.readline()
|
|
if not buf: break
|
|
conn.sendall(buf)
|
|
if callback: callback(buf)
|
|
conn.close()
|
|
|
|
# that's another workaround
|
|
#return "226"
|
|
try:
|
|
rc = self.ftpconn.voidresp()
|
|
except:
|
|
self.reconnectHost()
|
|
return "226"
|
|
return rc
|
|
|
|
def updateProgress(self, buf_len):
|
|
# get the buffer size
|
|
self.mykByteCount += float(buf_len)/1024
|
|
# create percentage
|
|
myUploadPercentage = round((round(self.mykByteCount,1)/self.myFileSize)*100,1)
|
|
currentprogress = myUploadPercentage
|
|
myUploadSize = round(self.mykByteCount,1)
|
|
if (currentprogress > self.oldprogress+0.5) and (myUploadPercentage < 100.1) and (myUploadSize <= self.myFileSize):
|
|
myUploadPercentage = str(myUploadPercentage)+"%"
|
|
|
|
# create text
|
|
currentText = brown(" <-> Upload status: ")+green(str(myUploadSize))+"/"+red(str(self.myFileSize))+" kB "+yellow("[")+str(myUploadPercentage)+yellow("]")
|
|
# print !
|
|
print_info(currentText, back = True)
|
|
# XXX too slow, reimplement self.updateProgress and do whatever you want
|
|
#self.Entropy.updateProgress(currentText, importance = 0, type = "info", back = True)
|
|
self.oldprogress = currentprogress
|
|
|
|
def uploadFile(self,file,ascii = False):
|
|
|
|
self.oldprogress = 0.0
|
|
|
|
def uploadFileAndUpdateProgress(buf):
|
|
self.updateProgress(len(buf))
|
|
|
|
for i in range(10): # ten tries
|
|
filename = file.split("/")[len(file.split("/"))-1]
|
|
try:
|
|
f = open(file,"r")
|
|
# get file size
|
|
self.myFileSize = round(float(os.stat(file)[6])/1024,1)
|
|
self.mykByteCount = 0
|
|
|
|
if self.isFileAvailable(filename+".tmp"):
|
|
self.deleteFile(filename+".tmp")
|
|
|
|
if (ascii):
|
|
rc = self.ftpconn.storlines("STOR "+filename+".tmp",f)
|
|
else:
|
|
rc = self.advancedStorBinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress )
|
|
|
|
# now we can rename the file with its original name
|
|
self.renameFile(filename+".tmp",filename)
|
|
f.close()
|
|
|
|
if rc.find("226") != -1: # upload complete
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
except Exception, e: # connection reset by peer
|
|
import traceback
|
|
traceback.print_exc()
|
|
self.Entropy.updateProgress("", importance = 0, type = "info")
|
|
self.Entropy.updateProgress(red("Upload issue: %s, retrying... #%s") % (str(e),str(i+1)),
|
|
importance = 1,
|
|
type = "warning",
|
|
header = " "
|
|
)
|
|
self.reconnectHost() # reconnect
|
|
if self.isFileAvailable(filename):
|
|
self.deleteFile(filename)
|
|
if self.isFileAvailable(filename+".tmp"):
|
|
self.deleteFile(filename+".tmp")
|
|
pass
|
|
|
|
def downloadFile(self,filepath,downloaddir,ascii = False):
|
|
|
|
self.oldprogress = 0.0
|
|
|
|
def downloadFileStoreAndUpdateProgress(buf):
|
|
# writing file buffer
|
|
f.write(buf)
|
|
# update progress
|
|
self.mykByteCount += float(len(buf))/1024
|
|
# create text
|
|
cnt = round(self.mykByteCount,1)
|
|
currentText = brown(" <-> Download status: ")+green(str(cnt))+"/"+red(str(self.myFileSize))+" kB"
|
|
# print !
|
|
self.Entropy.updateProgress(currentText, importance = 0, type = "info", back = True, count = (cnt, self.myFileSize), percent = True )
|
|
|
|
item = filepath.split("/")[len(filepath.split("/"))-1]
|
|
# look if the file exist
|
|
if self.isFileAvailable(item):
|
|
self.mykByteCount = 0
|
|
# get the file size
|
|
self.myFileSize = self.getFileSizeCompat(item)
|
|
if (self.myFileSize):
|
|
self.myFileSize = round(float(int(self.myFileSize))/1024,1)
|
|
if (self.myFileSize == 0):
|
|
self.myFileSize = 1
|
|
else:
|
|
self.myFileSize = 0
|
|
if (not ascii):
|
|
f = open(downloaddir+"/"+item,"wb")
|
|
rc = self.ftpconn.retrbinary('RETR '+item, downloadFileStoreAndUpdateProgress, 1024)
|
|
else:
|
|
f = open(downloaddir+"/"+item,"w")
|
|
rc = self.ftpconn.retrlines('RETR '+item, f.write)
|
|
f.flush()
|
|
f.close()
|
|
if rc.find("226") != -1: # upload complete
|
|
return True
|
|
else:
|
|
return False
|
|
else:
|
|
return None
|
|
|
|
# also used to move files
|
|
def renameFile(self,fromfile,tofile):
|
|
rc = self.ftpconn.rename(fromfile,tofile)
|
|
return rc
|
|
|
|
# not supported by dreamhost.com
|
|
def getFileSize(self,file):
|
|
return self.ftpconn.size(file)
|
|
|
|
def getFileSizeCompat(self,file):
|
|
data = self.getRoughList()
|
|
for item in data:
|
|
if item.find(file) != -1:
|
|
# extact the size
|
|
return item.split()[4]
|
|
return ""
|
|
|
|
def bufferizer(self,buf):
|
|
self.FTPbuffer.append(buf)
|
|
|
|
def getRoughList(self):
|
|
self.FTPbuffer = []
|
|
self.ftpconn.dir(self.bufferizer)
|
|
return self.FTPbuffer
|
|
|
|
def closeConnection(self):
|
|
self.ftpconn.quit()
|
|
|
|
|
|
'''
|
|
Entropy FTP/HTTP download interface
|
|
'''
|
|
class urlFetcher:
|
|
|
|
def __init__(self, url, pathToSave, checksum = True, showSpeed = True, resume = True):
|
|
|
|
self.url = url
|
|
self.resume = resume
|
|
self.url = self.encodeUrl(self.url)
|
|
self.pathToSave = pathToSave
|
|
self.checksum = checksum
|
|
self.showSpeed = showSpeed
|
|
self.initVars()
|
|
import entropyTools
|
|
self.entropyTools = entropyTools
|
|
|
|
# resume support
|
|
if os.path.isfile(self.pathToSave) and os.access(self.pathToSave,os.R_OK) and self.resume:
|
|
self.localfile = open(self.pathToSave,"awb")
|
|
self.localfile.seek(0,2)
|
|
self.startingposition = int(self.localfile.tell())
|
|
self.resumed = True
|
|
else:
|
|
self.localfile = open(self.pathToSave,"wb")
|
|
|
|
# setup proxy, doing here because config is dynamic
|
|
if etpConst['proxy']:
|
|
proxy_support = urllib2.ProxyHandler(etpConst['proxy'])
|
|
opener = urllib2.build_opener(proxy_support)
|
|
urllib2.install_opener(opener)
|
|
#FIXME else: unset opener??
|
|
|
|
def encodeUrl(self, url):
|
|
import urllib
|
|
url = os.path.join(os.path.dirname(url),urllib.quote(os.path.basename(url)))
|
|
return url
|
|
|
|
def initVars(self):
|
|
self.resumed = False
|
|
self.bufferSize = 8192
|
|
self.status = None
|
|
self.remotefile = None
|
|
self.downloadedsize = 0
|
|
self.average = 0
|
|
self.remotesize = 0
|
|
self.oldaverage = 0.0
|
|
# transfer status data
|
|
self.startingposition = 0
|
|
self.datatransfer = 0
|
|
self.time_remaining = "(infinite)"
|
|
self.elapsed = 0.0
|
|
self.updatestep = 0.2
|
|
self.speedlimit = etpConst['downloadspeedlimit'] # kbytes/sec
|
|
self.transferpollingtime = float(1)/4
|
|
|
|
def download(self):
|
|
self.initVars()
|
|
self.speedUpdater = self.entropyTools.TimeScheduled(
|
|
self.updateSpeedInfo,
|
|
self.transferpollingtime
|
|
)
|
|
self.speedUpdater.setName("download::"+self.url+str(random.random())) # set unique ID to thread, hopefully
|
|
self.speedUpdater.start()
|
|
|
|
# set timeout
|
|
socket.setdefaulttimeout(20)
|
|
|
|
# get file size if available
|
|
try:
|
|
self.remotefile = urllib2.urlopen(self.url)
|
|
except KeyboardInterrupt:
|
|
self.close()
|
|
raise
|
|
except:
|
|
self.close()
|
|
self.status = "-3"
|
|
return self.status
|
|
|
|
try:
|
|
self.remotesize = int(self.remotefile.headers.get("content-length"))
|
|
self.remotefile.close()
|
|
except KeyboardInterrupt:
|
|
self.close()
|
|
raise
|
|
except:
|
|
pass
|
|
|
|
# handle user stupidity
|
|
try:
|
|
request = self.url
|
|
if ((self.startingposition > 0) and (self.remotesize > 0)) and (self.startingposition < self.remotesize):
|
|
try:
|
|
request = urllib2.Request(self.url, headers = { "Range" : "bytes=" + str(self.startingposition) + "-" + str(self.remotesize) })
|
|
except KeyboardInterrupt:
|
|
self.close()
|
|
raise
|
|
except:
|
|
pass
|
|
elif (self.startingposition == self.remotesize):
|
|
return self.prepare_return()
|
|
else:
|
|
self.localfile = open(self.pathToSave,"wb")
|
|
self.remotefile = urllib2.urlopen(request)
|
|
except KeyboardInterrupt:
|
|
self.close()
|
|
raise
|
|
except:
|
|
self.close()
|
|
self.status = "-3"
|
|
return self.status
|
|
|
|
if self.remotesize > 0:
|
|
self.remotesize = float(int(self.remotesize))/1024
|
|
|
|
rsx = "x"
|
|
while rsx != '':
|
|
try:
|
|
rsx = self.remotefile.read(self.bufferSize)
|
|
except KeyboardInterrupt:
|
|
self.close()
|
|
raise
|
|
except:
|
|
# python 2.4 timeouts go here
|
|
self.close()
|
|
self.status = "-3"
|
|
return self.status
|
|
self.commitData(rsx)
|
|
if self.showSpeed:
|
|
self.updateProgress()
|
|
if self.speedlimit:
|
|
while self.datatransfer > self.speedlimit*1024:
|
|
time.sleep(0.1)
|
|
if self.showSpeed:
|
|
self.updateProgress()
|
|
|
|
# kill thread
|
|
self.close()
|
|
|
|
return self.prepare_return()
|
|
|
|
|
|
def prepare_return(self):
|
|
if self.checksum:
|
|
self.status = self.entropyTools.md5sum(self.pathToSave)
|
|
return self.status
|
|
else:
|
|
self.status = "-2"
|
|
return self.status
|
|
|
|
def commitData(self, mybuffer):
|
|
# writing file buffer
|
|
self.localfile.write(mybuffer)
|
|
# update progress info
|
|
self.downloadedsize = self.localfile.tell()
|
|
kbytecount = float(self.downloadedsize)/1024
|
|
self.average = int((kbytecount/self.remotesize)*100)
|
|
|
|
def updateProgress(self):
|
|
|
|
currentText = darkred(" <-> Downloading: ")+darkgreen(str(round(float(self.downloadedsize)/1024,1)))+"/"+red(str(round(self.remotesize,1))) + " kB"
|
|
# create progress bar
|
|
barsize = 10
|
|
bartext = "["
|
|
curbarsize = 1
|
|
if self.average > self.oldaverage+self.updatestep:
|
|
averagesize = (self.average*barsize)/100
|
|
for y in range(averagesize):
|
|
curbarsize += 1
|
|
bartext += "="
|
|
bartext += ">"
|
|
diffbarsize = barsize-curbarsize
|
|
for y in range(diffbarsize):
|
|
bartext += " "
|
|
if (self.showSpeed):
|
|
bartext += "] => "+str(self.entropyTools.bytesIntoHuman(self.datatransfer))+"/sec ~ ETA: "+str(self.time_remaining)
|
|
else:
|
|
bartext += "]"
|
|
average = str(self.average)
|
|
if len(average) < 2:
|
|
average = " "+average
|
|
currentText += " <-> "+average+"% "+bartext
|
|
# print !
|
|
print_info(currentText,back = True)
|
|
self.oldaverage = self.average
|
|
|
|
|
|
def close(self):
|
|
try:
|
|
self.localfile.flush()
|
|
self.localfile.close()
|
|
except:
|
|
pass
|
|
try:
|
|
self.remotefile.close()
|
|
except:
|
|
pass
|
|
self.speedUpdater.kill()
|
|
socket.setdefaulttimeout(2)
|
|
|
|
def updateSpeedInfo(self):
|
|
self.elapsed += self.transferpollingtime
|
|
# we have the diff size
|
|
self.datatransfer = (self.downloadedsize-self.startingposition) / self.elapsed
|
|
try:
|
|
self.time_remaining = int(round((int(round(self.remotesize*1024,0))-int(round(self.downloadedsize,0)))/self.datatransfer,0))
|
|
self.time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.time_remaining)
|
|
except:
|
|
self.time_remaining = "(infinite)"
|
|
|
|
|
|
class rssFeed:
|
|
|
|
def __init__(self, filename, maxentries = 100):
|
|
|
|
self.feed_title = etpConst['systemname']+" Online Repository Status"
|
|
self.feed_description = "Keep you updated on what's going on in the Official "+etpConst['systemname']+" Repository."
|
|
self.feed_language = "en-EN"
|
|
self.feed_editor = etpConst['rss-managing-editor']
|
|
self.feed_copyright = etpConst['systemname']+" (C) 2007-2009"
|
|
|
|
self.file = filename
|
|
self.items = {}
|
|
self.itemscounter = 0
|
|
self.maxentries = maxentries
|
|
from xml.dom import minidom
|
|
self.minidom = minidom
|
|
|
|
# sanity check
|
|
broken = False
|
|
if os.path.isfile(self.file):
|
|
try:
|
|
self.xmldoc = self.minidom.parse(self.file)
|
|
except:
|
|
#print "DEBUG: RSS broken, recreating in 5 seconds."
|
|
#time.sleep(5)
|
|
broken = True
|
|
|
|
if not os.path.isfile(self.file) or broken:
|
|
self.title = self.feed_title
|
|
self.description = self.feed_description
|
|
self.language = self.feed_language
|
|
self.cright = self.feed_copyright
|
|
self.editor = self.feed_editor
|
|
self.link = etpConst['rss-website-url']
|
|
f = open(self.file,"w")
|
|
f.write('')
|
|
f.close()
|
|
else:
|
|
# parse file
|
|
self.rssdoc = self.xmldoc.getElementsByTagName("rss")[0]
|
|
self.channel = self.rssdoc.getElementsByTagName("channel")[0]
|
|
self.title = self.channel.getElementsByTagName("title")[0].firstChild.data
|
|
self.link = self.channel.getElementsByTagName("link")[0].firstChild.data
|
|
self.description = self.channel.getElementsByTagName("description")[0].firstChild.data
|
|
self.language = self.channel.getElementsByTagName("language")[0].firstChild.data
|
|
self.cright = self.channel.getElementsByTagName("copyright")[0].firstChild.data
|
|
self.editor = self.channel.getElementsByTagName("managingEditor")[0].firstChild.data
|
|
entries = self.channel.getElementsByTagName("item")
|
|
self.itemscounter = len(entries)
|
|
if self.itemscounter > self.maxentries:
|
|
self.itemscounter = self.maxentries
|
|
mycounter = self.itemscounter
|
|
for item in entries:
|
|
if mycounter == 0: # max entries reached
|
|
break
|
|
mycounter -= 1
|
|
self.items[mycounter] = {}
|
|
self.items[mycounter]['title'] = item.getElementsByTagName("title")[0].firstChild.data
|
|
description = item.getElementsByTagName("description")[0].firstChild
|
|
if description:
|
|
self.items[mycounter]['description'] = description.data
|
|
else:
|
|
self.items[mycounter]['description'] = ""
|
|
link = item.getElementsByTagName("link")[0].firstChild
|
|
if link:
|
|
self.items[mycounter]['link'] = link.data
|
|
else:
|
|
self.items[mycounter]['link'] = ""
|
|
self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data
|
|
self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data
|
|
|
|
def addItem(self, title, link = '', description = ''):
|
|
self.itemscounter += 1
|
|
self.items[self.itemscounter] = {}
|
|
self.items[self.itemscounter]['title'] = title
|
|
self.items[self.itemscounter]['pubDate'] = time.strftime("%a, %d %b %Y %X +0000")
|
|
self.items[self.itemscounter]['description'] = description
|
|
self.items[self.itemscounter]['link'] = link
|
|
if link:
|
|
self.items[self.itemscounter]['guid'] = link
|
|
else:
|
|
self.items[self.itemscounter]['guid'] = "sabayonlinux.org~"+description+str(self.itemscounter)
|
|
return self.itemscounter
|
|
|
|
def removeEntry(self, id):
|
|
del self.items[id]
|
|
self.itemscounter -= 1
|
|
return len(self.itemscounter)
|
|
|
|
def getEntries(self):
|
|
return self.items, self.itemscounter
|
|
|
|
def writeChanges(self):
|
|
|
|
# filter entries to fit in maxentries
|
|
if self.itemscounter > self.maxentries:
|
|
tobefiltered = self.itemscounter - self.maxentries
|
|
for index in range(tobefiltered):
|
|
try:
|
|
del self.items[index]
|
|
except KeyError:
|
|
pass
|
|
|
|
doc = self.minidom.Document()
|
|
|
|
rss = doc.createElement("rss")
|
|
rss.setAttribute("version","2.0")
|
|
rss.setAttribute("xmlns:atom","http://www.w3.org/2005/Atom")
|
|
|
|
channel = doc.createElement("channel")
|
|
|
|
# title
|
|
title = doc.createElement("title")
|
|
title_text = doc.createTextNode(unicode(self.title))
|
|
title.appendChild(title_text)
|
|
channel.appendChild(title)
|
|
# link
|
|
link = doc.createElement("link")
|
|
link_text = doc.createTextNode(unicode(self.link))
|
|
link.appendChild(link_text)
|
|
channel.appendChild(link)
|
|
# description
|
|
description = doc.createElement("description")
|
|
desc_text = doc.createTextNode(unicode(self.description))
|
|
description.appendChild(desc_text)
|
|
channel.appendChild(description)
|
|
# language
|
|
language = doc.createElement("language")
|
|
lang_text = doc.createTextNode(unicode(self.language))
|
|
language.appendChild(lang_text)
|
|
channel.appendChild(language)
|
|
# copyright
|
|
cright = doc.createElement("copyright")
|
|
cr_text = doc.createTextNode(unicode(self.cright))
|
|
cright.appendChild(cr_text)
|
|
channel.appendChild(cright)
|
|
# managingEditor
|
|
managingEditor = doc.createElement("managingEditor")
|
|
ed_text = doc.createTextNode(unicode(self.editor))
|
|
managingEditor.appendChild(ed_text)
|
|
channel.appendChild(managingEditor)
|
|
|
|
keys = self.items.keys()
|
|
keys.reverse()
|
|
for key in keys:
|
|
|
|
# sanity check, you never know
|
|
try:
|
|
self.items[key]['title']
|
|
self.items[key]['link']
|
|
self.items[key]['guid']
|
|
self.items[key]['description']
|
|
self.items[key]['pubDate']
|
|
except KeyError:
|
|
self.removeEntry(key)
|
|
continue
|
|
|
|
# item
|
|
item = doc.createElement("item")
|
|
# title
|
|
item_title = doc.createElement("title")
|
|
item_title_text = doc.createTextNode(unicode(self.items[key]['title']))
|
|
item_title.appendChild(item_title_text)
|
|
item.appendChild(item_title)
|
|
# link
|
|
item_link = doc.createElement("link")
|
|
item_link_text = doc.createTextNode(unicode(self.items[key]['link']))
|
|
item_link.appendChild(item_link_text)
|
|
item.appendChild(item_link)
|
|
# guid
|
|
item_guid = doc.createElement("guid")
|
|
item_guid.setAttribute("isPermaLink","true")
|
|
item_guid_text = doc.createTextNode(unicode(self.items[key]['guid']))
|
|
item_guid.appendChild(item_guid_text)
|
|
item.appendChild(item_guid)
|
|
# description
|
|
item_desc = doc.createElement("description")
|
|
item_desc_text = doc.createTextNode(unicode(self.items[key]['description']))
|
|
item_desc.appendChild(item_desc_text)
|
|
item.appendChild(item_desc)
|
|
# pubdate
|
|
item_date = doc.createElement("pubDate")
|
|
item_date_text = doc.createTextNode(unicode(self.items[key]['pubDate']))
|
|
item_date.appendChild(item_date_text)
|
|
item.appendChild(item_date)
|
|
|
|
# add item to channel
|
|
channel.appendChild(item)
|
|
|
|
# add channel to rss
|
|
rss.appendChild(channel)
|
|
doc.appendChild(rss)
|
|
f = open(self.file,"w")
|
|
f.writelines(doc.toprettyxml(indent=" "))
|
|
f.flush()
|
|
f.close()
|
|
|
|
class TriggerInterface:
|
|
|
|
def __init__(self, EquoInstance, phase, pkgdata):
|
|
self.Entropy = EquoInstance
|
|
try:
|
|
self.Entropy.instanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
|
|
|
|
self.equoLog = self.Entropy.equoLog
|
|
self.validPhases = ("preinstall","postinstall","preremove","postremove")
|
|
self.pkgdata = pkgdata
|
|
self.prepared = False
|
|
self.triggers = set()
|
|
|
|
'''
|
|
@ description: Gentoo toolchain variables
|
|
'''
|
|
self.MODULEDB_DIR="/var/lib/module-rebuild/"
|
|
self.INITSERVICES_DIR="/var/lib/init.d/"
|
|
|
|
''' portage stuff '''
|
|
self.portageTools = None
|
|
if etpConst['gentoo-compat']:
|
|
import portageTools
|
|
self.portageTools = portageTools
|
|
|
|
self.phase = phase
|
|
# validate phase
|
|
self.phaseValidation()
|
|
|
|
def phaseValidation(self):
|
|
if self.phase not in self.validPhases:
|
|
raise exceptionTools.InvalidData("InvalidData: valid phases are %s" % (self.validPhases,))
|
|
|
|
def prepare(self):
|
|
self.triggers = eval("self."+self.phase)()
|
|
remove = set()
|
|
for trigger in self.triggers:
|
|
if trigger in etpUi[self.phase+'_triggers_disable']:
|
|
remove.add(trigger)
|
|
self.triggers.difference_update(remove)
|
|
del remove
|
|
self.prepared = True
|
|
|
|
def run(self):
|
|
for trigger in self.triggers:
|
|
eval("self.trigger_"+trigger)()
|
|
|
|
def kill(self):
|
|
self.prepared = False
|
|
self.triggers.clear()
|
|
|
|
def postinstall(self):
|
|
|
|
functions = set()
|
|
# Gentoo hook
|
|
if etpConst['gentoo-compat']:
|
|
functions.add('ebuild_postinstall')
|
|
|
|
if self.pkgdata['trigger']:
|
|
functions.add('call_ext_postinstall')
|
|
|
|
# triggers that are not needed when gentoo-compat is enabled
|
|
if not etpConst['gentoo-compat']:
|
|
|
|
if "gnome2" in self.pkgdata['eclasses']:
|
|
functions.add('iconscache')
|
|
functions.add('gconfinstallschemas')
|
|
functions.add('gconfreload')
|
|
|
|
if self.pkgdata['name'] == "pygobject":
|
|
functions.add('pygtksetup')
|
|
|
|
# fonts configuration
|
|
if self.pkgdata['category'] == "media-fonts":
|
|
functions.add("fontconfig")
|
|
|
|
# gcc configuration
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-devel/gcc":
|
|
functions.add("gccswitch")
|
|
|
|
# binutils configuration
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-devel/binutils":
|
|
functions.add("binutilsswitch")
|
|
|
|
# kde package ?
|
|
if "kde" in self.pkgdata['eclasses']:
|
|
functions.add("kbuildsycoca")
|
|
|
|
if "kde4-base" in self.pkgdata['eclasses'] or "kde4-meta" in self.pkgdata['eclasses']:
|
|
functions.add("kbuildsycoca4")
|
|
|
|
# update mime
|
|
if "fdo-mime" in self.pkgdata['eclasses']:
|
|
functions.add('mimeupdate')
|
|
functions.add('mimedesktopupdate')
|
|
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "dev-db/sqlite":
|
|
functions.add('sqliteinst')
|
|
|
|
# python configuration
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "dev-lang/python":
|
|
functions.add("pythoninst")
|
|
|
|
# opengl configuration
|
|
if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")):
|
|
try:
|
|
functions.remove("ebuild_postinstall") # disabling gentoo postinstall since we reimplemented it
|
|
except:
|
|
pass
|
|
functions.add("openglsetup")
|
|
|
|
# load linker paths
|
|
ldpaths = self.Entropy.entropyTools.collectLinkerPaths()
|
|
# prepare content
|
|
for x in self.pkgdata['content']:
|
|
if not etpConst['gentoo-compat']:
|
|
if x.startswith("/usr/share/icons") and x.endswith("index.theme"):
|
|
functions.add('iconscache')
|
|
if x.startswith("/usr/share/mime"):
|
|
functions.add('mimeupdate')
|
|
if x.startswith("/usr/share/applications"):
|
|
functions.add('mimedesktopupdate')
|
|
if x.startswith("/usr/share/omf"):
|
|
functions.add('scrollkeeper')
|
|
if x.startswith("/etc/gconf/schemas"):
|
|
functions.add('gconfreload')
|
|
if x == '/bin/su':
|
|
functions.add("susetuid")
|
|
if x.startswith('/usr/share/java-config-2/vm/'):
|
|
functions.add('add_java_config_2')
|
|
else:
|
|
if x.startswith('/lib/modules/'):
|
|
try:
|
|
functions.remove("ebuild_postinstall") # disabling gentoo postinstall since we reimplemented it
|
|
except:
|
|
pass
|
|
functions.add('kernelmod')
|
|
if x.startswith('/boot/kernel-'):
|
|
functions.add('addbootablekernel')
|
|
if x.startswith('/usr/src/'):
|
|
functions.add('createkernelsym')
|
|
if x.startswith('/etc/env.d/'):
|
|
functions.add('env_update')
|
|
if os.path.dirname(x) in ldpaths:
|
|
if x.find(".so") > -1:
|
|
functions.add('run_ldconfig')
|
|
del ldpaths
|
|
return functions
|
|
|
|
def preinstall(self):
|
|
|
|
functions = set()
|
|
if self.pkgdata['trigger']:
|
|
functions.add('call_ext_preinstall')
|
|
|
|
# Gentoo hook
|
|
if etpConst['gentoo-compat']:
|
|
functions.add('ebuild_preinstall')
|
|
|
|
for x in self.pkgdata['content']:
|
|
if x.startswith("/etc/init.d/"):
|
|
functions.add('initinform')
|
|
if x.startswith("/boot"):
|
|
functions.add('mountboot')
|
|
return functions
|
|
|
|
def postremove(self):
|
|
|
|
functions = set()
|
|
|
|
if self.pkgdata['trigger']:
|
|
functions.add('call_ext_postremove')
|
|
|
|
if not etpConst['gentoo-compat']:
|
|
|
|
# kde package ?
|
|
if "kde" in self.pkgdata['eclasses']:
|
|
functions.add("kbuildsycoca")
|
|
|
|
if "kde4-base" in self.pkgdata['eclasses'] or "kde4-meta" in self.pkgdata['eclasses']:
|
|
functions.add("kbuildsycoca4")
|
|
|
|
if self.pkgdata['name'] == "pygtk":
|
|
functions.add('pygtkremove')
|
|
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "dev-db/sqlite":
|
|
functions.add('sqliteinst')
|
|
|
|
# python configuration
|
|
if self.pkgdata['category']+"/"+self.pkgdata['name'] == "dev-lang/python":
|
|
functions.add("pythoninst")
|
|
|
|
# fonts configuration
|
|
if self.pkgdata['category'] == "media-fonts":
|
|
functions.add("fontconfig")
|
|
|
|
# load linker paths
|
|
ldpaths = self.Entropy.entropyTools.collectLinkerPaths()
|
|
|
|
for x in self.pkgdata['removecontent']:
|
|
if not etpConst['gentoo-compat']:
|
|
if x.startswith("/usr/share/icons") and x.endswith("index.theme"):
|
|
functions.add('iconscache')
|
|
if x.startswith("/usr/share/mime"):
|
|
functions.add('mimeupdate')
|
|
if x.startswith("/usr/share/applications"):
|
|
functions.add('mimedesktopupdate')
|
|
if x.startswith("/usr/share/omf"):
|
|
functions.add('scrollkeeper')
|
|
if x.startswith("/etc/gconf/schemas"):
|
|
functions.add('gconfreload')
|
|
else:
|
|
if x.startswith('/boot/kernel-'):
|
|
functions.add('removebootablekernel')
|
|
if x.startswith('/etc/init.d/'):
|
|
functions.add('removeinit')
|
|
if x.endswith('.py'):
|
|
functions.add('cleanpy')
|
|
if x.startswith('/etc/env.d/'):
|
|
functions.add('env_update')
|
|
if os.path.dirname(x) in ldpaths:
|
|
if x.find(".so") > -1:
|
|
functions.add('run_ldconfig')
|
|
del ldpaths
|
|
return functions
|
|
|
|
|
|
def preremove(self):
|
|
|
|
functions = set()
|
|
|
|
if self.pkgdata['trigger']:
|
|
functions.add('call_ext_preremove')
|
|
|
|
# Gentoo hook
|
|
if etpConst['gentoo-compat']:
|
|
functions.add('ebuild_preremove')
|
|
functions.add('ebuild_postremove') # doing here because we need /var/db/pkg stuff in place and also because doesn't make any difference
|
|
|
|
# opengl configuration
|
|
if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")):
|
|
try:
|
|
functions.remove("ebuild_preremove") # disabling gentoo postinstall since we reimplemented it
|
|
functions.remove("ebuild_postremove")
|
|
except:
|
|
pass
|
|
functions.add("openglsetup_xorg")
|
|
|
|
for x in self.pkgdata['removecontent']:
|
|
if x.startswith("/etc/init.d/"):
|
|
functions.add('initdisable')
|
|
if x.startswith("/boot"):
|
|
functions.add('mountboot')
|
|
|
|
return functions
|
|
|
|
|
|
'''
|
|
Real triggers
|
|
'''
|
|
def trigger_call_ext_preinstall(self):
|
|
rc = self.trigger_call_ext_generic()
|
|
return rc
|
|
|
|
def trigger_call_ext_postinstall(self):
|
|
rc = self.trigger_call_ext_generic()
|
|
return rc
|
|
|
|
def trigger_call_ext_preremove(self):
|
|
rc = self.trigger_call_ext_generic()
|
|
return rc
|
|
|
|
def trigger_call_ext_postremove(self):
|
|
rc = self.trigger_call_ext_generic()
|
|
return rc
|
|
|
|
def trigger_call_ext_generic(self):
|
|
|
|
# if mute, supress portage output
|
|
if etpUi['mute']:
|
|
oldsystderr = sys.stderr
|
|
oldsysstdout = sys.stdout
|
|
stdfile = open("/dev/null","w")
|
|
sys.stdout = stdfile
|
|
sys.stderr = stdfile
|
|
|
|
triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber())
|
|
while os.path.isfile(triggerfile):
|
|
triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber())
|
|
f = open(triggerfile,"w")
|
|
for x in self.pkgdata['trigger']:
|
|
f.write(x)
|
|
f.close()
|
|
|
|
# if mute, restore old stdout/stderr
|
|
if etpUi['mute']:
|
|
sys.stderr = oldsystderr
|
|
sys.stdout = oldsysstdout
|
|
stdfile.close()
|
|
|
|
stage = self.phase
|
|
pkgdata = self.pkgdata
|
|
my_ext_status = 0
|
|
execfile(triggerfile)
|
|
os.remove(triggerfile)
|
|
return my_ext_status
|
|
|
|
|
|
def trigger_fontconfig(self):
|
|
fontdirs = set()
|
|
for xdir in self.pkgdata['content']:
|
|
xdir = etpConst['systemroot']+xdir
|
|
if xdir.startswith(etpConst['systemroot']+"/usr/share/fonts"):
|
|
origdir = xdir[len(etpConst['systemroot'])+16:]
|
|
if origdir:
|
|
if origdir.startswith("/"):
|
|
origdir = origdir.split("/")[1]
|
|
if os.path.isdir(xdir[:16]+"/"+origdir):
|
|
fontdirs.add(xdir[:16]+"/"+origdir)
|
|
if (fontdirs):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring fonts directory...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring fonts directory..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
for fontdir in fontdirs:
|
|
self.trigger_setup_font_dir(fontdir)
|
|
self.trigger_setup_font_cache(fontdir)
|
|
del fontdirs
|
|
|
|
def trigger_gccswitch(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring GCC Profile...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring GCC Profile..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
# get gcc profile
|
|
pkgsplit = self.Entropy.entropyTools.catpkgsplit(self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'])
|
|
profile = self.pkgdata['chost']+"-"+pkgsplit[2]
|
|
self.trigger_set_gcc_profile(profile)
|
|
|
|
def trigger_iconscache(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Updating icons cache...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating icons cache..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
for item in self.pkgdata['content']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/usr/share/icons") and item.endswith("index.theme"):
|
|
cachedir = os.path.dirname(item)
|
|
self.trigger_generate_icons_cache(cachedir)
|
|
|
|
def trigger_mimeupdate(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Updating shared mime info database...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating shared mime info database..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_update_mime_db()
|
|
|
|
def trigger_mimedesktopupdate(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Updating desktop mime database...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating desktop mime database..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_update_mime_desktop_db()
|
|
|
|
def trigger_scrollkeeper(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Updating scrollkeeper database...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating scrollkeeper database..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_update_scrollkeeper_db()
|
|
|
|
def trigger_gconfreload(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Reloading GConf2 database...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Reloading GConf2 database..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_reload_gconf_db()
|
|
|
|
def trigger_binutilsswitch(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring Binutils Profile...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring Binutils Profile..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
# get binutils profile
|
|
pkgsplit = self.Entropy.entropyTools.catpkgsplit(self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version'])
|
|
profile = self.pkgdata['chost']+"-"+pkgsplit[2]
|
|
self.trigger_set_binutils_profile(profile)
|
|
|
|
def trigger_kernelmod(self):
|
|
if self.pkgdata['category'] != "sys-kernel":
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Updating moduledb...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating moduledb..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
item = 'a:1:'+self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
|
|
self.trigger_update_moduledb(item)
|
|
self.Entropy.updateProgress(
|
|
brown(" Running depmod..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
# get kernel modules dir name
|
|
name = ''
|
|
for item in self.pkgdata['content']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/lib/modules/"):
|
|
name = item[len(etpConst['systemroot']):]
|
|
name = name.split("/")[3]
|
|
break
|
|
if name:
|
|
self.trigger_run_depmod(name)
|
|
|
|
def trigger_pythoninst(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring Python...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring Python..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_python_update_symlink()
|
|
|
|
def trigger_sqliteinst(self):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring SQLite...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring SQLite..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_sqlite_update_symlink()
|
|
|
|
def trigger_initdisable(self):
|
|
for item in self.pkgdata['removecontent']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item):
|
|
# running?
|
|
running = os.path.isfile(etpConst['systemroot']+self.INITSERVICES_DIR+'/started/'+os.path.basename(item))
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
scheduled = not os.system('ROOT="'+myroot+'" rc-update show | grep '+os.path.basename(item)+'&> /dev/null')
|
|
self.trigger_initdeactivate(item, running, scheduled)
|
|
|
|
def trigger_initinform(self):
|
|
for item in self.pkgdata['content']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/etc/init.d/") and not os.path.isfile(etpConst['systemroot']+item):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] A new service will be installed: "+item)
|
|
self.Entropy.updateProgress(
|
|
brown(" A new service will be installed: ")+item,
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
|
|
def trigger_removeinit(self):
|
|
for item in self.pkgdata['removecontent']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Removing boot service: "+os.path.basename(item))
|
|
self.Entropy.updateProgress(
|
|
brown(" Removing boot service: ")+os.path.basename(item),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
try:
|
|
os.system('ROOT="'+myroot+'" rc-update del '+os.path.basename(item)+' &> /dev/null')
|
|
except:
|
|
pass
|
|
|
|
def trigger_openglsetup(self):
|
|
opengl = "xorg-x11"
|
|
if self.pkgdata['name'] == "nvidia-drivers":
|
|
opengl = "nvidia"
|
|
elif self.pkgdata['name'] == "ati-drivers":
|
|
opengl = "ati"
|
|
# is there eselect ?
|
|
eselect = os.system("eselect opengl &> /dev/null")
|
|
if eselect == 0:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Reconfiguring OpenGL to "+opengl+" ...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Reconfiguring OpenGL..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
quietstring = ''
|
|
if etpUi['quiet']: quietstring = " &>/dev/null"
|
|
if etpConst['systemroot']:
|
|
os.system('echo "eselect opengl set --use-old '+opengl+'" | chroot '+etpConst['systemroot']+quietstring)
|
|
else:
|
|
os.system('eselect opengl set --use-old '+opengl+quietstring)
|
|
else:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Eselect NOT found, cannot run OpenGL trigger")
|
|
self.Entropy.updateProgress(
|
|
brown(" Eselect NOT found, cannot run OpenGL trigger"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
|
|
def trigger_openglsetup_xorg(self):
|
|
eselect = os.system("eselect opengl &> /dev/null")
|
|
if eselect == 0:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Reconfiguring OpenGL to fallback xorg-x11 ...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Reconfiguring OpenGL..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
quietstring = ''
|
|
if etpUi['quiet']: quietstring = " &>/dev/null"
|
|
if etpConst['systemroot']:
|
|
os.system('echo "eselect opengl set xorg-x11" | chroot '+etpConst['systemroot']+quietstring)
|
|
else:
|
|
os.system('eselect opengl set xorg-x11'+quietstring)
|
|
else:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Eselect NOT found, cannot run OpenGL trigger")
|
|
self.Entropy.updateProgress(
|
|
brown(" Eselect NOT found, cannot run OpenGL trigger"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
|
|
# FIXME: this only supports grub (no lilo support)
|
|
def trigger_addbootablekernel(self):
|
|
kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")]
|
|
for kernel in kernels:
|
|
initramfs = "/boot/initramfs-"+kernel[13:]
|
|
if initramfs not in self.pkgdata['content']:
|
|
initramfs = ''
|
|
# configure GRUB
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring GRUB bootloader. Adding the new kernel...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring GRUB bootloader. Adding the new kernel..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_configure_boot_grub(kernel,initramfs)
|
|
|
|
|
|
# FIXME: this only supports grub (no lilo support)
|
|
def trigger_removebootablekernel(self):
|
|
kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")]
|
|
for kernel in kernels:
|
|
initramfs = "/boot/initramfs-"+kernel[13:]
|
|
if initramfs not in self.pkgdata['content']:
|
|
initramfs = ''
|
|
# configure GRUB
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring GRUB bootloader. Removing the selected kernel...")
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring GRUB bootloader. Removing the selected kernel..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
self.trigger_remove_boot_grub(kernel,initramfs)
|
|
|
|
def trigger_mountboot(self):
|
|
# is in fstab?
|
|
if etpConst['systemroot']:
|
|
return
|
|
if os.path.isfile("/etc/fstab"):
|
|
f = open("/etc/fstab","r")
|
|
fstab = f.readlines()
|
|
fstab = self.Entropy.entropyTools.listToUtf8(fstab)
|
|
f.close()
|
|
for line in fstab:
|
|
fsline = line.split()
|
|
if len(fsline) > 1:
|
|
if fsline[1] == "/boot":
|
|
if not os.path.ismount("/boot"):
|
|
# trigger mount /boot
|
|
rc = os.system("mount /boot &> /dev/null")
|
|
if rc == 0:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] Mounted /boot successfully")
|
|
self.Entropy.updateProgress(
|
|
brown(" Mounted /boot successfully"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
elif rc != 8192: # already mounted
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] Cannot mount /boot automatically !!")
|
|
self.Entropy.updateProgress(
|
|
brown(" Cannot mount /boot automatically !!"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
break
|
|
|
|
def trigger_kbuildsycoca(self):
|
|
if etpConst['systemroot']:
|
|
return
|
|
kdedirs = ''
|
|
try:
|
|
kdedirs = os.environ['KDEDIRS']
|
|
except:
|
|
pass
|
|
if kdedirs:
|
|
dirs = kdedirs.split(":")
|
|
for builddir in dirs:
|
|
if os.access(builddir+"/bin/kbuildsycoca",os.X_OK):
|
|
if not os.path.isdir("/usr/share/services"):
|
|
os.makedirs("/usr/share/services")
|
|
os.chown("/usr/share/services",0,0)
|
|
os.chmod("/usr/share/services",0755)
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Running kbuildsycoca to build global KDE database")
|
|
self.Entropy.updateProgress(
|
|
brown(" Running kbuildsycoca to build global KDE database"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
os.system(builddir+"/bin/kbuildsycoca --global --noincremental &> /dev/null")
|
|
|
|
def trigger_kbuildsycoca4(self):
|
|
if etpConst['systemroot']:
|
|
return
|
|
kdedirs = ''
|
|
try:
|
|
kdedirs = os.environ['KDEDIRS']
|
|
except:
|
|
pass
|
|
if kdedirs:
|
|
dirs = kdedirs.split(":")
|
|
for builddir in dirs:
|
|
if os.access(builddir+"/bin/kbuildsycoca4",os.X_OK):
|
|
if not os.path.isdir("/usr/share/services"):
|
|
os.makedirs("/usr/share/services")
|
|
os.chown("/usr/share/services",0,0)
|
|
os.chmod("/usr/share/services",0755)
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Running kbuildsycoca4 to build global KDE4 database")
|
|
self.Entropy.updateProgress(
|
|
brown(" Running kbuildsycoca to build global KDE database"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
# do it
|
|
kbuild4cmd = """
|
|
|
|
# Thanks to the hard work of kde4 gentoo overlay maintainers
|
|
|
|
for i in $(dbus-launch); do
|
|
export "$i"
|
|
done
|
|
|
|
# This is needed because we support multiple kde versions installed together.
|
|
XDG_DATA_DIRS="/usr/share:${KDEDIRS}/share:/usr/local/share"
|
|
"""+builddir+"""/bin/kbuildsycoca4 --global --noincremental &> /dev/null
|
|
kill ${DBUS_SESSION_BUS_PID}
|
|
|
|
"""
|
|
os.system(kbuild4cmd)
|
|
|
|
def trigger_gconfinstallschemas(self):
|
|
gtest = os.system("which gconftool-2 &> /dev/null")
|
|
if gtest == 0 or etpConst['systemroot']:
|
|
schemas = [x for x in self.pkgdata['content'] if x.startswith("/etc/gconf/schemas") and x.endswith(".schemas")]
|
|
self.Entropy.updateProgress(
|
|
brown(" Installing GConf2 schemas..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
for schema in schemas:
|
|
if not etpConst['systemroot']:
|
|
os.system("""
|
|
unset GCONF_DISABLE_MAKEFILE_SCHEMA_INSTALL
|
|
export GCONF_CONFIG_SOURCE=$(gconftool-2 --get-default-source)
|
|
gconftool-2 --makefile-install-rule """+schema+""" 1>/dev/null
|
|
""")
|
|
else:
|
|
os.system(""" echo "
|
|
unset GCONF_DISABLE_MAKEFILE_SCHEMA_INSTALL
|
|
export GCONF_CONFIG_SOURCE=$(gconftool-2 --get-default-source)
|
|
gconftool-2 --makefile-install-rule """+schema+""" " | chroot """+etpConst['systemroot']+""" &>/dev/null
|
|
""")
|
|
|
|
def trigger_pygtksetup(self):
|
|
python_sym_files = [x for x in self.pkgdata['content'] if x.endswith("pygtk.py-2.0") or x.endswith("pygtk.pth-2.0")]
|
|
for item in python_sym_files:
|
|
item = etpConst['systemroot']+item
|
|
filepath = item[:-4]
|
|
sympath = os.path.basename(item)
|
|
if os.path.isfile(item):
|
|
try:
|
|
if os.path.lexists(filepath):
|
|
os.remove(filepath)
|
|
os.symlink(sympath,filepath)
|
|
except OSError:
|
|
pass
|
|
|
|
def trigger_pygtkremove(self):
|
|
python_sym_files = [x for x in self.pkgdata['content'] if x.startswith("/usr/lib/python") and (x.endswith("pygtk.py-2.0") or x.endswith("pygtk.pth-2.0"))]
|
|
for item in python_sym_files:
|
|
item = etpConst['systemroot']+item
|
|
if os.path.isfile(item[:-4]):
|
|
os.remove(item[:-4])
|
|
|
|
def trigger_susetuid(self):
|
|
if os.path.isfile(etpConst['systemroot']+"/bin/su"):
|
|
self.Entropy.updateProgress(
|
|
brown(" Configuring '"+etpConst['systemroot']+"/bin/su' executable SETUID"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
os.chown(etpConst['systemroot']+"/bin/su",0,0)
|
|
os.system("chmod 4755 "+etpConst['systemroot']+"/bin/su")
|
|
#os.chmod("/bin/su",4755) #FIXME: probably there's something I don't know here since, masks?
|
|
|
|
def trigger_cleanpy(self):
|
|
pyfiles = [x for x in self.pkgdata['content'] if x.endswith(".py")]
|
|
for item in pyfiles:
|
|
item = etpConst['systemroot']+item
|
|
if os.path.isfile(item+"o"):
|
|
try: os.remove(item+"o")
|
|
except OSError: pass
|
|
if os.path.isfile(item+"c"):
|
|
try: os.remove(item+"c")
|
|
except OSError: pass
|
|
|
|
def trigger_createkernelsym(self):
|
|
for item in self.pkgdata['content']:
|
|
item = etpConst['systemroot']+item
|
|
if item.startswith(etpConst['systemroot']+"/usr/src/"):
|
|
# extract directory
|
|
try:
|
|
todir = item[len(etpConst['systemroot']):]
|
|
todir = todir.split("/")[3]
|
|
except:
|
|
continue
|
|
if os.path.isdir(etpConst['systemroot']+"/usr/src/"+todir):
|
|
# link to /usr/src/linux
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Creating kernel symlink "+etpConst['systemroot']+"/usr/src/linux for /usr/src/"+todir)
|
|
self.Entropy.updateProgress(
|
|
brown(" Creating kernel symlink "+etpConst['systemroot']+"/usr/src/linux for /usr/src/"+todir),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
if os.path.isfile(etpConst['systemroot']+"/usr/src/linux") or os.path.islink(etpConst['systemroot']+"/usr/src/linux"):
|
|
os.remove(etpConst['systemroot']+"/usr/src/linux")
|
|
if os.path.isdir(etpConst['systemroot']+"/usr/src/linux"):
|
|
mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber())
|
|
while os.path.isdir(mydir):
|
|
mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber())
|
|
shutil.move(etpConst['systemroot']+"/usr/src/linux",mydir)
|
|
try:
|
|
os.symlink(todir,etpConst['systemroot']+"/usr/src/linux")
|
|
except OSError: # not important in the end
|
|
pass
|
|
break
|
|
|
|
def trigger_run_ldconfig(self):
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Running ldconfig")
|
|
self.Entropy.updateProgress(
|
|
brown(" Regenerating /etc/ld.so.cache"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
os.system("ldconfig -r "+myroot+" &> /dev/null")
|
|
|
|
def trigger_env_update(self):
|
|
# clear linker paths cache
|
|
linkerPaths.clear()
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Running env-update")
|
|
if os.access(etpConst['systemroot']+"/usr/sbin/env-update",os.X_OK):
|
|
self.Entropy.updateProgress(
|
|
brown(" Updating environment using env-update"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
if etpConst['systemroot']:
|
|
os.system("echo 'env-update --no-ldconfig' | chroot "+etpConst['systemroot']+" &> /dev/null")
|
|
else:
|
|
os.system('env-update --no-ldconfig &> /dev/null')
|
|
|
|
def trigger_add_java_config_2(self):
|
|
vms = set()
|
|
for vm in self.pkgdata['content']:
|
|
vm = etpConst['systemroot']+vm
|
|
if vm.startswith(etpConst['systemroot']+"/usr/share/java-config-2/vm/") and os.path.isfile(vm):
|
|
vms.add(vm)
|
|
# sort and get the latter
|
|
if vms:
|
|
vms = list(vms)
|
|
vms.reverse()
|
|
myvm = vms[0].split("/")[-1]
|
|
if myvm:
|
|
if os.access(etpConst['systemroot']+"/usr/bin/java-config",os.X_OK):
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring JAVA using java-config with VM: "+myvm)
|
|
# set
|
|
self.Entropy.updateProgress(
|
|
brown(" Setting system VM to ")+bold(myvm)+brown("..."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
if not etpConst['systemroot']:
|
|
os.system("java-config -S "+myvm)
|
|
else:
|
|
os.system("echo 'java-config -S "+myvm+"' | chroot "+etpConst['systemroot']+" &> /dev/null")
|
|
else:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION /usr/bin/java-config does not exist. I was about to set JAVA VM: "+myvm)
|
|
self.Entropy.updateProgress(
|
|
bold(" Attention: ")+brown("/usr/bin/java-config does not exist. Cannot set JAVA VM."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
del vms
|
|
|
|
def trigger_ebuild_postinstall(self):
|
|
stdfile = open("/dev/null","w")
|
|
oldstderr = sys.stderr
|
|
sys.stderr = stdfile
|
|
|
|
myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(".ebuild")]
|
|
if myebuild:
|
|
myebuild = myebuild[0]
|
|
portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
|
|
self.Entropy.updateProgress(
|
|
brown(" Ebuild: pkg_postinst()"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
try:
|
|
if not os.path.isfile(self.pkgdata['unpackdir']+"/portage/"+portage_atom+"/temp/environment"): # if environment is not yet created, we need to run pkg_setup()
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'])
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup()) trigger for "+str(portage_atom)+". Something bad happened.")
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "postinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'])
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_postinst()) trigger for "+str(portage_atom)+". Something bad happened.")
|
|
except Exception, e:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] ATTENTION Cannot run Gentoo postinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e))
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("Cannot run Gentoo postint trigger for ")+bold(portage_atom)+brown(". Please report."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
sys.stderr = oldstderr
|
|
stdfile.close()
|
|
return 0
|
|
|
|
def trigger_ebuild_preinstall(self):
|
|
stdfile = open("/dev/null","w")
|
|
oldstderr = sys.stderr
|
|
sys.stderr = stdfile
|
|
|
|
myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(".ebuild")]
|
|
if myebuild:
|
|
myebuild = myebuild[0]
|
|
portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
|
|
self.Entropy.updateProgress(
|
|
brown(" Ebuild: pkg_preinst()"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
try:
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "setup", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir']) # create mysettings["T"]+"/environment"
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_setup()) trigger for "+str(portage_atom)+". Something bad happened.")
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "preinst", tree = "bintree", cpv = portage_atom, portage_tmpdir = self.pkgdata['unpackdir'])
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_preinst()) trigger for "+str(portage_atom)+". Something bad happened.")
|
|
except Exception, e:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot run Gentoo preinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e))
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("Cannot run Gentoo preinst trigger for ")+bold(portage_atom)+brown(". Please report."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
sys.stderr = oldstderr
|
|
stdfile.close()
|
|
return 0
|
|
|
|
def trigger_ebuild_preremove(self):
|
|
portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
|
|
myebuild = self.portageTools.getPortageAppDbPath()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild"
|
|
if os.path.isfile(myebuild):
|
|
self.Entropy.updateProgress(
|
|
brown(" Ebuild: pkg_prerm()"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
try:
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "prerm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom)
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo preremove trigger for "+str(portage_atom)+". Something bad happened.")
|
|
except Exception, e:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot run Gentoo preremove trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e))
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("Cannot run Gentoo preremove trigger for ")+bold(portage_atom)+brown(". Please report."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return 0
|
|
|
|
def trigger_ebuild_postremove(self):
|
|
portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
|
|
myebuild = self.portageTools.getPortageAppDbPath()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild"
|
|
if os.path.isfile(myebuild):
|
|
self.Entropy.updateProgress(
|
|
brown(" Ebuild: pkg_postrm()"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
try:
|
|
rc = self.portageTools.portage_doebuild(myebuild, mydo = "postrm", tree = "bintree", cpv = portage_atom, portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom)
|
|
if rc == 1:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot properly run Gentoo postremove trigger for "+str(portage_atom)+". Something bad happened.")
|
|
except Exception, e:
|
|
self.Entropy.equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[PRE] ATTENTION Cannot run Gentoo postremove trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e))
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("Cannot run Gentoo postremove trigger for ")+bold(portage_atom)+brown(". Please report."),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return 0
|
|
|
|
'''
|
|
Internal ones
|
|
'''
|
|
|
|
'''
|
|
@description: creates Xfont files
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_setup_font_dir(self, fontdir):
|
|
# mkfontscale
|
|
if os.access('/usr/bin/mkfontscale',os.X_OK):
|
|
os.system('/usr/bin/mkfontscale '+unicode(fontdir))
|
|
# mkfontdir
|
|
if os.access('/usr/bin/mkfontdir',os.X_OK):
|
|
os.system('/usr/bin/mkfontdir -e '+etpConst['systemroot']+'/usr/share/fonts/encodings -e '+etpConst['systemroot']+'/usr/share/fonts/encodings/large '+unicode(fontdir))
|
|
return 0
|
|
|
|
'''
|
|
@description: creates font cache
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_setup_font_cache(self, fontdir):
|
|
# fc-cache -f gooooo!
|
|
if os.access('/usr/bin/fc-cache',os.X_OK):
|
|
os.system('/usr/bin/fc-cache -f '+unicode(fontdir))
|
|
return 0
|
|
|
|
'''
|
|
@description: set chosen gcc profile
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_set_gcc_profile(self, profile):
|
|
if os.access(etpConst['systemroot']+'/usr/bin/gcc-config',os.X_OK):
|
|
redirect = ""
|
|
if etpUi['quiet']:
|
|
redirect = " &> /dev/null"
|
|
if etpConst['systemroot']:
|
|
os.system("echo '/usr/bin/gcc-config "+profile+"' | chroot "+etpConst['systemroot']+redirect)
|
|
else:
|
|
os.system('/usr/bin/gcc-config '+profile+redirect)
|
|
return 0
|
|
|
|
'''
|
|
@description: set chosen binutils profile
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_set_binutils_profile(self, profile):
|
|
if os.access(etpConst['systemroot']+'/usr/bin/binutils-config',os.X_OK):
|
|
redirect = ""
|
|
if etpUi['quiet']:
|
|
redirect = " &> /dev/null"
|
|
if etpConst['systemroot']:
|
|
os.system("echo '/usr/bin/binutils-config "+profile+"' | chroot "+etpConst['systemroot']+redirect)
|
|
else:
|
|
os.system('/usr/bin/binutils-config '+profile+redirect)
|
|
return 0
|
|
|
|
'''
|
|
@description: creates/updates icons cache
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_generate_icons_cache(self, cachedir):
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
if os.access('/usr/bin/gtk-update-icon-cache',os.X_OK):
|
|
os.system('ROOT="'+myroot+'" /usr/bin/gtk-update-icon-cache -qf '+cachedir)
|
|
return 0
|
|
|
|
'''
|
|
@description: updates /usr/share/mime database
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_update_mime_db(self):
|
|
if os.access(etpConst['systemroot']+'/usr/bin/update-mime-database',os.X_OK):
|
|
if not etpConst['systemroot']:
|
|
os.system('/usr/bin/update-mime-database /usr/share/mime')
|
|
else:
|
|
os.system("echo '/usr/bin/update-mime-database /usr/share/mime' | chroot "+etpConst['systemroot']+" &> /dev/null")
|
|
return 0
|
|
|
|
'''
|
|
@description: updates /usr/share/applications database
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_update_mime_desktop_db(self):
|
|
if os.access(etpConst['systemroot']+'/usr/bin/update-desktop-database',os.X_OK):
|
|
if not etpConst['systemroot']:
|
|
os.system('/usr/bin/update-desktop-database -q /usr/share/applications')
|
|
else:
|
|
os.system("echo '/usr/bin/update-desktop-database -q /usr/share/applications' | chroot "+etpConst['systemroot']+" &> /dev/null")
|
|
return 0
|
|
|
|
'''
|
|
@description: updates /var/lib/scrollkeeper database
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_update_scrollkeeper_db(self):
|
|
if os.access(etpConst['systemroot']+'/usr/bin/scrollkeeper-update',os.X_OK):
|
|
if not os.path.isdir(etpConst['systemroot']+'/var/lib/scrollkeeper'):
|
|
os.makedirs(etpConst['systemroot']+'/var/lib/scrollkeeper')
|
|
if not etpConst['systemroot']:
|
|
os.system('/usr/bin/scrollkeeper-update -q -p /var/lib/scrollkeeper')
|
|
else:
|
|
os.system("echo '/usr/bin/scrollkeeper-update -q -p /var/lib/scrollkeeper' | chroot "+etpConst['systemroot']+" &> /dev/null")
|
|
return 0
|
|
|
|
'''
|
|
@description: respawn gconfd-2 if found
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_reload_gconf_db(self):
|
|
if etpConst['systemroot']:
|
|
return 0
|
|
rc = os.system('pgrep -x gconfd-2')
|
|
if (rc == 0):
|
|
pids = commands.getoutput('pgrep -x gconfd-2').split("\n")
|
|
pidsstr = ''
|
|
for pid in pids:
|
|
if pid:
|
|
pidsstr += pid+' '
|
|
pidsstr = pidsstr.strip()
|
|
if pidsstr:
|
|
os.system('kill -HUP '+pidsstr)
|
|
return 0
|
|
|
|
'''
|
|
@description: updates moduledb
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_update_moduledb(self, item):
|
|
if os.access(etpConst['systemroot']+'/usr/sbin/module-rebuild',os.X_OK):
|
|
if os.path.isfile(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb'):
|
|
f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"r")
|
|
moduledb = f.readlines()
|
|
moduledb = self.Entropy.entropyTools.listToUtf8(moduledb)
|
|
f.close()
|
|
avail = [x for x in moduledb if x.strip() == item]
|
|
if (not avail):
|
|
f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"aw")
|
|
f.write(item+"\n")
|
|
f.flush()
|
|
f.close()
|
|
return 0
|
|
|
|
'''
|
|
@description: insert kernel object into kernel modules db
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_run_depmod(self, name):
|
|
if os.access('/sbin/depmod',os.X_OK):
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
os.system('/sbin/depmod -a -b '+myroot+' -r '+name+' &> /dev/null')
|
|
return 0
|
|
|
|
'''
|
|
@description: update /usr/bin/python and /usr/bin/python2 symlink
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_python_update_symlink(self):
|
|
bins = [x for x in os.listdir("/usr/bin") if x.startswith("python2.")]
|
|
if bins: # don't ask me why but it happened...
|
|
bins.sort()
|
|
latest = bins[-1]
|
|
|
|
latest = etpConst['systemroot']+"/usr/bin/"+latest
|
|
filepath = os.path.dirname(latest)+"/python"
|
|
sympath = os.path.basename(latest)
|
|
if os.path.isfile(latest):
|
|
try:
|
|
if os.path.lexists(filepath):
|
|
os.remove(filepath)
|
|
os.symlink(sympath,filepath)
|
|
except OSError:
|
|
pass
|
|
return 0
|
|
|
|
'''
|
|
@description: update /usr/bin/lemon symlink
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_sqlite_update_symlink(self):
|
|
bins = [x for x in os.listdir("/usr/bin") if x.startswith("lemon-")]
|
|
if bins:
|
|
bins.sort()
|
|
latest = bins[-1]
|
|
latest = etpConst['systemroot']+"/usr/bin/"+latest
|
|
|
|
filepath = os.path.dirname(latest)+"/lemon"
|
|
sympath = os.path.basename(latest)
|
|
if os.path.isfile(latest):
|
|
try:
|
|
if os.path.lexists(filepath):
|
|
os.remove(filepath)
|
|
os.symlink(sympath,filepath)
|
|
except OSError:
|
|
pass
|
|
return 0
|
|
|
|
'''
|
|
@description: shuts down selected init script, and remove from runlevel
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_initdeactivate(self, item, running, scheduled):
|
|
if not etpConst['systemroot']:
|
|
myroot = "/"
|
|
'''
|
|
causes WORLD to fall under
|
|
if (running):
|
|
os.system(item+' stop --quiet')
|
|
'''
|
|
else:
|
|
myroot = etpConst['systemroot']+"/"
|
|
if (scheduled):
|
|
os.system('ROOT="'+myroot+'" rc-update del '+os.path.basename(item))
|
|
return 0
|
|
|
|
'''
|
|
@description: append kernel entry to grub.conf
|
|
@output: returns int() as exit status
|
|
'''
|
|
def trigger_configure_boot_grub(self, kernel,initramfs):
|
|
|
|
if not os.path.isdir(etpConst['systemroot']+"/boot/grub"):
|
|
os.makedirs(etpConst['systemroot']+"/boot/grub")
|
|
if os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"):
|
|
# open in append
|
|
grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","aw")
|
|
# get boot dev
|
|
boot_dev = self.trigger_get_grub_boot_dev()
|
|
# test if entry has been already added
|
|
grubtest = open(etpConst['systemroot']+"/boot/grub/grub.conf","r")
|
|
content = grubtest.readlines()
|
|
content = self.Entropy.entropyTools.listToUtf8(content)
|
|
for line in content:
|
|
try: # handle stupidly encoded text
|
|
if line.find("title="+etpConst['systemname']+" ("+os.path.basename(kernel)+")\n") != -1:
|
|
grubtest.close()
|
|
return
|
|
# also check if we have the same kernel listed
|
|
if (line.find("kernel") != 1) and (line.find(os.path.basename(kernel)) != -1) and not line.strip().startswith("#"):
|
|
grubtest.close()
|
|
return
|
|
except UnicodeDecodeError:
|
|
continue
|
|
else:
|
|
# create
|
|
boot_dev = "(hd0,0)"
|
|
grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","w")
|
|
# write header - guess (hd0,0)... since it is weird having a running system without a bootloader, at least, grub.
|
|
grub_header = '''
|
|
default=0
|
|
timeout=10
|
|
'''
|
|
grub.write(grub_header)
|
|
cmdline = ' '
|
|
if os.path.isfile("/proc/cmdline"):
|
|
f = open("/proc/cmdline","r")
|
|
cmdline = " "+f.readline().strip()
|
|
params = cmdline.split()
|
|
if "dolvm" not in params: # support new kernels >= 2.6.23
|
|
cmdline += " dolvm "
|
|
f.close()
|
|
grub.write("title="+etpConst['systemname']+" ("+os.path.basename(kernel)+")\n")
|
|
grub.write("\troot "+boot_dev+"\n")
|
|
grub.write("\tkernel "+kernel+cmdline+"\n")
|
|
if initramfs:
|
|
grub.write("\tinitrd "+initramfs+"\n")
|
|
grub.write("\n")
|
|
grub.flush()
|
|
grub.close()
|
|
|
|
def trigger_remove_boot_grub(self, kernel,initramfs):
|
|
if os.path.isdir(etpConst['systemroot']+"/boot/grub") and os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"):
|
|
f = open(etpConst['systemroot']+"/boot/grub/grub.conf","r")
|
|
grub_conf = f.readlines()
|
|
f.close()
|
|
grub_conf = self.Entropy.entropyTools.listToUtf8(grub_conf)
|
|
# validate file encodings - damn what a crap
|
|
kernel, initramfs = self.Entropy.entropyTools.listToUtf8([kernel,initramfs])
|
|
kernelname = os.path.basename(kernel)
|
|
new_conf = []
|
|
found = False
|
|
for count in range(len(grub_conf)):
|
|
line = grub_conf[count].strip()
|
|
if (line.find(kernelname) != -1) or (line.find(kernelname) != -1):
|
|
found = True
|
|
# remove previous content up to title
|
|
rlines = 0
|
|
for x in range(len(new_conf))[::-1]:
|
|
rlines += 1
|
|
if new_conf[x].strip().startswith("title"):
|
|
break
|
|
new_conf = new_conf[::-1][rlines:][::-1]
|
|
if (found):
|
|
# check if the parameter belongs to title or it is something else
|
|
try:
|
|
line = grub_conf[count].strip().split()[0]
|
|
except IndexError: # in case of weird stuff (happened...)
|
|
new_conf.append(grub_conf[count])
|
|
continue
|
|
if line: # skip empty lines
|
|
if line in ["root","kernel","initrd","hide","unhide","chainloader","makeactive","rootnoverify"]:
|
|
# skip write
|
|
continue
|
|
else:
|
|
# skip completed
|
|
found = False
|
|
new_conf.append(grub_conf[count])
|
|
f = open(etpConst['systemroot']+"/boot/grub/grub.conf","w")
|
|
f.writelines(new_conf)
|
|
f.flush()
|
|
f.close()
|
|
|
|
def trigger_get_grub_boot_dev(self):
|
|
if etpConst['systemroot']:
|
|
return "(hd0,0)"
|
|
import re
|
|
df_avail = os.system("which df &> /dev/null")
|
|
if df_avail != 0:
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("cannot find df!! Cannot properly configure kernel! Defaulting to (hd0,0)"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return "(hd0,0)"
|
|
grub_avail = os.system("which grub &> /dev/null")
|
|
if grub_avail != 0:
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("cannot find grub!! Cannot properly configure kernel! Defaulting to (hd0,0)"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return "(hd0,0)"
|
|
|
|
gboot = commands.getoutput("df /boot").split("\n")[-1].split()[0]
|
|
if gboot.startswith("/dev/"):
|
|
# it's ok - handle /dev/md
|
|
if gboot.startswith("/dev/md"):
|
|
md = os.path.basename(gboot)
|
|
if not md.startswith("md"):
|
|
md = "md"+md
|
|
f = open("/proc/mdstat","r")
|
|
mdstat = f.readlines()
|
|
mdstat = [x for x in mdstat if x.startswith(md)]
|
|
f.close()
|
|
if mdstat:
|
|
mdstat = mdstat[0].strip().split()
|
|
mddevs = []
|
|
for x in mdstat:
|
|
if x.startswith("sd"):
|
|
mddevs.append(x[:-3])
|
|
mddevs.sort()
|
|
if mddevs:
|
|
gboot = "/dev/"+mddevs[0]
|
|
else:
|
|
gboot = "/dev/sda1"
|
|
else:
|
|
gboot = "/dev/sda1"
|
|
# get disk
|
|
match = re.subn("[0-9]","",gboot)
|
|
gdisk = match[0]
|
|
match = re.subn("[a-z/]","",gboot)
|
|
gpartnum = str(int(match[0])-1)
|
|
# now match with grub
|
|
device_map = etpConst['packagestmpdir']+"/grub.map"
|
|
if os.path.isfile(device_map):
|
|
os.remove(device_map)
|
|
# generate device.map
|
|
os.system('echo "quit" | grub --device-map='+device_map+' --no-floppy --batch &> /dev/null')
|
|
if os.path.isfile(device_map):
|
|
f = open(device_map,"r")
|
|
device_map_file = f.readlines()
|
|
f.close()
|
|
grub_dev = [x for x in device_map_file if (x.find(gdisk) != -1)]
|
|
if grub_dev:
|
|
grub_disk = grub_dev[0].strip().split()[0]
|
|
grub_dev = grub_disk[:-1]+","+gpartnum+")"
|
|
return grub_dev
|
|
else:
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("cannot match grub device with linux one!! Cannot properly configure kernel! Defaulting to (hd0,0)"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return "(hd0,0)"
|
|
else:
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("cannot find generated device.map!! Cannot properly configure kernel! Defaulting to (hd0,0)"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return "(hd0,0)"
|
|
else:
|
|
self.Entropy.updateProgress(
|
|
bold(" QA Warning: ")+brown("cannot run df /boot!! Cannot properly configure kernel! Defaulting to (hd0,0)"),
|
|
importance = 0,
|
|
header = red(" ##")
|
|
)
|
|
return "(hd0,0)"
|
|
|
|
|
|
class Callable:
|
|
def __init__(self, anycallable):
|
|
self.__call__ = anycallable
|
|
|
|
class MultipartPostHandler(urllib2.BaseHandler):
|
|
handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first
|
|
|
|
def http_request(self, request):
|
|
|
|
import urllib
|
|
doseq = 1
|
|
|
|
data = request.get_data()
|
|
if data is not None and type(data) != str:
|
|
v_files = []
|
|
v_vars = []
|
|
try:
|
|
for(key, value) in data.items():
|
|
if type(value) == file:
|
|
v_files.append((key, value))
|
|
else:
|
|
v_vars.append((key, value))
|
|
except TypeError:
|
|
systype, value, traceback = sys.exc_info()
|
|
raise TypeError, "not a valid non-string sequence or mapping object", traceback
|
|
|
|
if len(v_files) == 0:
|
|
data = urllib.urlencode(v_vars, doseq)
|
|
else:
|
|
boundary, data = self.multipart_encode(v_vars, v_files)
|
|
|
|
contenttype = 'multipart/form-data; boundary=%s' % boundary
|
|
if(request.has_header('Content-Type')
|
|
and request.get_header('Content-Type').find('multipart/form-data') != 0):
|
|
print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')
|
|
request.add_unredirected_header('Content-Type', contenttype)
|
|
request.add_data(data)
|
|
return request
|
|
|
|
def multipart_encode(vars, files, boundary = None, buf = None):
|
|
|
|
from cStringIO import StringIO
|
|
import mimetools, mimetypes
|
|
|
|
if boundary is None:
|
|
boundary = mimetools.choose_boundary()
|
|
if buf is None:
|
|
buf = StringIO()
|
|
for(key, value) in vars:
|
|
buf.write('--%s\r\n' % boundary)
|
|
buf.write('Content-Disposition: form-data; name="%s"' % key)
|
|
buf.write('\r\n\r\n' + value + '\r\n')
|
|
for(key, fd) in files:
|
|
file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
|
|
filename = fd.name.split('/')[-1]
|
|
contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
|
|
buf.write('--%s\r\n' % boundary)
|
|
buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
|
|
buf.write('Content-Type: %s\r\n' % contenttype)
|
|
# buffer += 'Content-Length: %s\r\n' % file_size
|
|
fd.seek(0)
|
|
buf.write('\r\n' + fd.read() + '\r\n')
|
|
buf.write('--' + boundary + '--\r\n\r\n')
|
|
buf = buf.getvalue()
|
|
return boundary, buf
|
|
multipart_encode = Callable(multipart_encode)
|
|
|
|
https_request = http_request
|
|
|
|
|
|
class ErrorReportInterface:
|
|
|
|
def __init__(self, post_url = etpHandlers['errorsend']):
|
|
self.url = post_url
|
|
self.opener = urllib2.build_opener(MultipartPostHandler)
|
|
self.generated = False
|
|
self.params = {}
|
|
|
|
if etpConst['proxy']:
|
|
proxy_support = urllib2.ProxyHandler(etpConst['proxy'])
|
|
opener = urllib2.build_opener(proxy_support)
|
|
urllib2.install_opener(opener)
|
|
|
|
def prepare(self, tb_text, name, email, report_data = ""):
|
|
self.params['arch'] = etpConst['currentarch']
|
|
self.params['stacktrace'] = tb_text
|
|
self.params['name'] = name
|
|
self.params['email'] = email
|
|
self.params['version'] = etpConst['entropyversion']
|
|
self.params['errordata'] = report_data
|
|
self.generated = True
|
|
|
|
# params is a dict, key(HTTP post item name): value
|
|
def submit(self):
|
|
if self.generated:
|
|
result = self.opener.open(self.url, self.params).read()
|
|
if result.strip() == "1":
|
|
return True
|
|
return False
|
|
else:
|
|
raise exceptionTools.PermissionDenied("PermissionDenied: not yet prepared")
|
|
|
|
|
|
'''
|
|
~~ GIVES YOU WINGS ~~
|
|
'''
|
|
class SecurityInterface:
|
|
|
|
# thanks to Gentoo "gentoolkit" package, License below:
|
|
|
|
# This program is licensed under the GPL, version 2
|
|
|
|
# WARNING: this code is only tested by a few people and should NOT be used
|
|
# on production systems at this stage. There are possible security holes and probably
|
|
# bugs in this code. If you test it please report ANY success or failure to
|
|
# me (genone@gentoo.org).
|
|
|
|
# The following planned features are currently on hold:
|
|
# - getting GLSAs from http/ftp servers (not really useful without the fixed ebuilds)
|
|
# - GPG signing/verification (until key policy is clear)
|
|
|
|
def __init__(self, EquoInstance):
|
|
self.Entropy = EquoInstance
|
|
try:
|
|
self.Entropy.instanceTest()
|
|
except:
|
|
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
|
|
self.lastfetch = None
|
|
self.previous_checksum = "0"
|
|
self.advisories_changed = None
|
|
self.adv_metadata = None
|
|
self.affected_atoms = None
|
|
|
|
from xml.dom import minidom
|
|
self.minidom = minidom
|
|
|
|
self.op_mappings = {
|
|
"le": "<=",
|
|
"lt": "<",
|
|
"eq": "=",
|
|
"gt": ">",
|
|
"ge": ">=",
|
|
"rge": ">=~",
|
|
"rle": "<=~",
|
|
"rgt": " >~",
|
|
"rlt": " <~"
|
|
}
|
|
|
|
self.unpackdir = os.path.join(etpConst['entropyunpackdir'],"security-"+str(self.Entropy.entropyTools.getRandomNumber()))
|
|
self.security_url = etpConst['securityurl']
|
|
self.unpacked_package = os.path.join(self.unpackdir,"glsa_package")
|
|
self.security_url_checksum = etpConst['securityurl']+etpConst['packageshashfileext']
|
|
self.download_package = os.path.join(self.unpackdir,os.path.basename(etpConst['securityurl']))
|
|
self.download_package_checksum = self.download_package+etpConst['packageshashfileext']
|
|
|
|
self.security_package = os.path.join(etpConst['securitydir'],os.path.basename(etpConst['securityurl']))
|
|
self.security_package_checksum = self.security_package+etpConst['packageshashfileext']
|
|
|
|
if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']):
|
|
os.remove(etpConst['securitydir'])
|
|
if not os.path.isdir(etpConst['securitydir']):
|
|
os.makedirs(etpConst['securitydir'])
|
|
elif os.path.isfile(self.security_package_checksum):
|
|
try:
|
|
f = open(self.security_package_checksum)
|
|
self.previous_checksum = f.readline().strip().split()[0]
|
|
f.close()
|
|
except: # FIXME really horrible exception trapping
|
|
try:
|
|
f.close()
|
|
except:
|
|
pass
|
|
|
|
def __prepare_unpack(self):
|
|
|
|
if os.path.isfile(self.unpackdir) or os.path.islink(self.unpackdir):
|
|
os.remove(self.unpackdir)
|
|
if os.path.isdir(self.unpackdir):
|
|
shutil.rmtree(self.unpackdir,True)
|
|
try:
|
|
os.rmdir(self.unpackdir)
|
|
except OSError:
|
|
pass
|
|
os.makedirs(self.unpackdir)
|
|
|
|
def __download_glsa_package(self):
|
|
return self.__generic_download(self.security_url, self.download_package)
|
|
|
|
def __download_glsa_package_checksum(self):
|
|
return self.__generic_download(self.security_url_checksum, self.download_package_checksum, showSpeed = False)
|
|
|
|
def __generic_download(self, url, save_to, showSpeed = True):
|
|
fetchConn = self.Entropy.urlFetcher(url, save_to, resume = False, showSpeed = showSpeed)
|
|
fetchConn.progress = self.Entropy.progress
|
|
rc = fetchConn.download()
|
|
del fetchConn
|
|
status = True
|
|
if rc in ("-1","-2","-3"):
|
|
status = False
|
|
return status
|
|
|
|
def __verify_checksum(self):
|
|
|
|
# read checksum
|
|
if not os.path.isfile(self.download_package_checksum) or not os.access(self.download_package_checksum,os.R_OK):
|
|
return 1
|
|
|
|
f = open(self.download_package_checksum)
|
|
try:
|
|
checksum = f.readline().strip().split()[0]
|
|
f.close()
|
|
except:
|
|
return 2
|
|
|
|
if checksum == self.previous_checksum:
|
|
self.advisories_changed = False
|
|
else:
|
|
self.advisories_changed = True
|
|
md5res = self.Entropy.entropyTools.compareMd5(self.download_package,checksum)
|
|
if not md5res:
|
|
return 3
|
|
return 0
|
|
|
|
def __unpack_advisories(self):
|
|
rc = self.Entropy.entropyTools.uncompressTarBz2(
|
|
self.download_package,
|
|
self.unpacked_package,
|
|
catchEmpty = True
|
|
)
|
|
return rc
|
|
|
|
def __clear_previous_advisories(self):
|
|
if os.listdir(etpConst['securitydir']):
|
|
shutil.rmtree(etpConst['securitydir'],True)
|
|
if not os.path.isdir(etpConst['securitydir']):
|
|
os.makedirs(etpConst['securitydir'])
|
|
|
|
def __put_advisories_in_place(self):
|
|
for advfile in os.listdir(self.unpacked_package):
|
|
from_file = os.path.join(self.unpacked_package,advfile)
|
|
to_file = os.path.join(etpConst['securitydir'],advfile)
|
|
shutil.move(from_file,to_file)
|
|
|
|
def __cleanup_garbage(self):
|
|
shutil.rmtree(self.unpackdir,True)
|
|
|
|
def clear(self, xcache = False):
|
|
self.adv_metadata = None
|
|
if xcache:
|
|
self.Entropy.dumpTools.dumpobj(etpCache['advisories'], {})
|
|
|
|
def __get_advisories_cache(self):
|
|
|
|
if self.adv_metadata != None:
|
|
return self.adv_metadata
|
|
|
|
if self.Entropy.xcache:
|
|
adv_cache = self.Entropy.dumpTools.loadobj(etpCache['advisories'])
|
|
dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir'])
|
|
try:
|
|
if adv_cache['systemroot'] == etpConst['systemroot'] and \
|
|
adv_cache['branch'] == etpConst['branch'] and \
|
|
adv_cache['data'] and \
|
|
adv_cache['checksum'] == dir_checksum:
|
|
self.adv_metadata = adv_cache['data'].copy()
|
|
return adv_cache['data']
|
|
except:
|
|
try:
|
|
self.Entropy.dumpTools.dumpobj(etpCache['advisories'], {})
|
|
except IOError:
|
|
pass
|
|
|
|
def __set_advisories_cache(self, adv_metadata):
|
|
if self.Entropy.xcache:
|
|
dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir'])
|
|
adv_cache = {}
|
|
adv_cache['branch'] = etpConst['branch']
|
|
adv_cache['data'] = adv_metadata
|
|
adv_cache['checksum'] = dir_checksum
|
|
adv_cache['systemroot'] = etpConst['systemroot']
|
|
try:
|
|
adv_cache = self.Entropy.dumpTools.dumpobj(etpCache['advisories'],adv_cache)
|
|
except IOError:
|
|
pass
|
|
|
|
def get_advisories_list(self):
|
|
if not self.check_advisories_availability():
|
|
return []
|
|
xmls = os.listdir(etpConst['securitydir'])
|
|
xmls = [x for x in xmls if x.endswith(".xml") and x.startswith("glsa-")]
|
|
xmls.sort()
|
|
return xmls
|
|
|
|
def get_advisories_metadata(self):
|
|
|
|
cached = self.__get_advisories_cache()
|
|
if cached != None:
|
|
return cached
|
|
|
|
adv_metadata = {}
|
|
xmls = self.get_advisories_list()
|
|
maxlen = len(xmls)
|
|
count = 0
|
|
for xml in xmls:
|
|
|
|
count += 1
|
|
if not etpUi['quiet']: self.Entropy.updateProgress(":: "+str(round((float(count)/maxlen)*100,1))+"% ::", importance = 0, type = "info", back = True)
|
|
|
|
xml_metadata = None
|
|
exc_string = ""
|
|
exc_err = ""
|
|
try:
|
|
xml_metadata = self.get_xml_metadata(xml)
|
|
except KeyboardInterrupt:
|
|
return {}
|
|
except Exception, e:
|
|
exc_string = str(Exception)
|
|
exc_err = str(e)
|
|
if xml_metadata == None:
|
|
more_info = ""
|
|
if exc_string:
|
|
more_info = " Error: %s: %s" % (exc_string,exc_err,)
|
|
self.Entropy.updateProgress(
|
|
blue("Warning: ")+bold(xml)+blue(" advisory is broken !") + more_info,
|
|
importance = 1,
|
|
type = "warning",
|
|
header = red(" !!! ")
|
|
)
|
|
continue
|
|
elif not xml_metadata:
|
|
continue
|
|
adv_metadata.update(xml_metadata)
|
|
|
|
adv_metadata = self.__filter_advisories(adv_metadata)
|
|
self.__set_advisories_cache(adv_metadata)
|
|
self.adv_metadata = adv_metadata.copy()
|
|
return adv_metadata
|
|
|
|
# this function filters advisories for packages that aren't
|
|
# in the repositories. Note: only keys will be matched
|
|
def __filter_advisories(self, adv_metadata):
|
|
keys = adv_metadata.keys()
|
|
for key in keys:
|
|
valid = True
|
|
if adv_metadata[key]['affected']:
|
|
affected = adv_metadata[key]['affected']
|
|
affected_keys = affected.keys()
|
|
valid = False
|
|
for a_key in affected_keys:
|
|
match = self.Entropy.atomMatch(a_key)
|
|
if match[0] != -1:
|
|
# it's in tree, it's valid
|
|
valid = True
|
|
break
|
|
if not valid:
|
|
del adv_metadata[key]
|
|
|
|
return adv_metadata
|
|
|
|
def is_affected(self, adv_key):
|
|
adv_data = self.get_advisories_metadata()
|
|
if adv_key not in adv_data:
|
|
return False
|
|
mydata = adv_data[adv_key].copy()
|
|
del adv_data
|
|
# get packages
|
|
if not mydata['affected']:
|
|
return False
|
|
for key in mydata['affected']:
|
|
vul_atoms = mydata['affected'][key][0]['vul_atoms']
|
|
unaff_atoms = mydata['affected'][key][0]['unaff_atoms']
|
|
unaffected_atoms = set()
|
|
for atom in unaff_atoms:
|
|
match = self.Entropy.clientDbconn.atomMatch(atom)
|
|
if match[0] != -1:
|
|
unaffected_atoms.add(match)
|
|
if not vul_atoms:
|
|
return False
|
|
for atom in vul_atoms:
|
|
match = self.Entropy.clientDbconn.atomMatch(atom)
|
|
if match[0] != -1 and match not in unaffected_atoms:
|
|
if self.affected_atoms == None:
|
|
self.affected_atoms = set()
|
|
self.affected_atoms.add(atom)
|
|
return True
|
|
return False
|
|
|
|
def get_affected_atoms(self):
|
|
adv_data = self.get_advisories_metadata()
|
|
adv_data_keys = adv_data.keys()
|
|
del adv_data
|
|
for key in adv_data_keys:
|
|
self.is_affected(key)
|
|
return self.affected_atoms
|
|
|
|
def get_xml_metadata(self, xmlfilename):
|
|
xml_data = {}
|
|
xmlfile = os.path.join(etpConst['securitydir'],xmlfilename)
|
|
try:
|
|
xmldoc = self.minidom.parse(xmlfile)
|
|
except:
|
|
return None
|
|
|
|
# get base data
|
|
glsa_tree = xmldoc.getElementsByTagName("glsa")[0]
|
|
glsa_product = glsa_tree.getElementsByTagName("product")[0]
|
|
if glsa_product.getAttribute("type") != "ebuild":
|
|
return {}
|
|
|
|
glsa_id = glsa_tree.getAttribute("id")
|
|
glsa_title = glsa_tree.getElementsByTagName("title")[0].firstChild.data
|
|
glsa_synopsis = glsa_tree.getElementsByTagName("synopsis")[0].firstChild.data
|
|
glsa_announced = glsa_tree.getElementsByTagName("announced")[0].firstChild.data
|
|
glsa_revised = glsa_tree.getElementsByTagName("revised")[0].firstChild.data
|
|
|
|
xml_data['filename'] = xmlfilename
|
|
xml_data['title'] = glsa_title.strip()
|
|
xml_data['synopsis'] = glsa_synopsis.strip()
|
|
xml_data['announced'] = glsa_announced.strip()
|
|
xml_data['revised'] = glsa_revised.strip()
|
|
xml_data['bugs'] = [x.firstChild.data.strip() for x in glsa_tree.getElementsByTagName("bug")]
|
|
xml_data['access'] = ""
|
|
try:
|
|
xml_data['access'] = glsa_tree.getElementsByTagName("access")[0].firstChild.data.strip()
|
|
except IndexError:
|
|
pass
|
|
|
|
# references
|
|
references = glsa_tree.getElementsByTagName("references")[0]
|
|
xml_data['references'] = [x.getAttribute("link").strip() for x in references.getElementsByTagName("uri")]
|
|
|
|
xml_data['description'] = glsa_tree.getElementsByTagName("description")[0].firstChild.data.strip()
|
|
xml_data['workaround'] = glsa_tree.getElementsByTagName("workaround")[0].firstChild.data.strip()
|
|
xml_data['resolution'] = glsa_tree.getElementsByTagName("resolution")[0].firstChild.data.strip()
|
|
xml_data['impact'] = glsa_tree.getElementsByTagName("impact")[0].firstChild.data.strip()
|
|
xml_data['impacttype'] = glsa_tree.getElementsByTagName("impact")[0].getAttribute("type").strip()
|
|
xml_data['background'] = ""
|
|
try:
|
|
xml_data['background'] = glsa_tree.getElementsByTagName("background")[0].firstChild.data.strip()
|
|
except IndexError:
|
|
pass
|
|
|
|
# affection information
|
|
affected = glsa_tree.getElementsByTagName("affected")[0]
|
|
affected_packages = {}
|
|
# we will then filter affected_packages using repositories information
|
|
# if not affected_packages: advisory will be dropped
|
|
for p in affected.getElementsByTagName("package"):
|
|
name = p.getAttribute("name")
|
|
if not affected_packages.has_key(name):
|
|
affected_packages[name] = []
|
|
|
|
pdata = {}
|
|
pdata["arch"] = p.getAttribute("arch").strip()
|
|
pdata["auto"] = (p.getAttribute("auto") == "yes")
|
|
pdata["vul_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("vulnerable")]
|
|
pdata["unaff_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("unaffected")]
|
|
pdata["vul_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("vulnerable")]
|
|
pdata["unaff_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("unaffected")]
|
|
affected_packages[name].append(pdata)
|
|
xml_data['affected'] = affected_packages.copy()
|
|
|
|
return {glsa_id: xml_data}
|
|
|
|
def __make_version(self, vnode):
|
|
"""
|
|
creates from the information in the I{versionNode} a
|
|
version string (format <op><version>).
|
|
|
|
@type vnode: xml.dom.Node
|
|
@param vnode: a <vulnerable> or <unaffected> Node that
|
|
contains the version information for this atom
|
|
@rtype: String
|
|
@return: the version string
|
|
"""
|
|
return self.op_mappings[vnode.getAttribute("range")] + vnode.firstChild.data.strip()
|
|
|
|
def __make_atom(self, pkgname, vnode):
|
|
"""
|
|
creates from the given package name and information in the
|
|
I{versionNode} a (syntactical) valid portage atom.
|
|
|
|
@type pkgname: String
|
|
@param pkgname: the name of the package for this atom
|
|
@type vnode: xml.dom.Node
|
|
@param vnode: a <vulnerable> or <unaffected> Node that
|
|
contains the version information for this atom
|
|
@rtype: String
|
|
@return: the portage atom
|
|
"""
|
|
return str(self.op_mappings[vnode.getAttribute("range")] + pkgname + "-" + vnode.firstChild.data.strip())
|
|
|
|
def check_advisories_availability(self):
|
|
if not os.path.lexists(etpConst['securitydir']):
|
|
return False
|
|
if not os.path.isdir(etpConst['securitydir']):
|
|
return False
|
|
else:
|
|
return True
|
|
return False
|
|
|
|
def fetch_advisories(self):
|
|
|
|
self.Entropy.updateProgress(
|
|
blue("Testing ")+bold("Security Advisories")+blue(" service connection"),
|
|
importance = 2,
|
|
type = "info",
|
|
header = red("@@ "),
|
|
footer = red(" ...")
|
|
)
|
|
|
|
# Test network connectivity
|
|
conntest = self.Entropy.entropyTools.get_remote_data(etpConst['conntestlink'])
|
|
if not conntest:
|
|
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: Cannot connect to %s" % (etpConst['conntestlink'],))
|
|
|
|
self.Entropy.updateProgress(
|
|
blue("Getting the latest ")+bold("Security Advisories")+darkgreen(" (GLSAs)"),
|
|
importance = 2,
|
|
type = "info",
|
|
header = red("@@ "),
|
|
footer = red(" ...")
|
|
)
|
|
|
|
# prepare directories
|
|
self.__prepare_unpack()
|
|
|
|
# download package
|
|
status = self.__download_glsa_package()
|
|
self.lastfetch = status
|
|
if not status:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: unable to download package, sorry."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 1
|
|
|
|
self.Entropy.updateProgress(
|
|
blue("Verifying checksum"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" # "),
|
|
footer = red(" ..."),
|
|
back = True
|
|
)
|
|
|
|
# download digest
|
|
status = self.__download_glsa_package_checksum()
|
|
if not status:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: cannot download checksum, sorry."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 2
|
|
|
|
# verify digest
|
|
status = self.__verify_checksum()
|
|
|
|
if status == 1:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: cannot open packages, sorry."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 3
|
|
elif status == 2:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: cannot read checksum, sorry."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 4
|
|
elif status == 3:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: digest verification failed, sorry."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 5
|
|
elif status == 0:
|
|
self.Entropy.updateProgress(
|
|
darkgreen("Verification Successful"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" # ")
|
|
)
|
|
else:
|
|
raise exceptionTools.InvalidData("InvalidData: return status not valid.")
|
|
|
|
# now unpack in place
|
|
status = self.__unpack_advisories()
|
|
if status != 0:
|
|
self.Entropy.updateProgress(
|
|
blue("Security Advisories: digest verification failed, try again later."),
|
|
importance = 2,
|
|
type = "error",
|
|
header = red(" ## ")
|
|
)
|
|
return 6
|
|
|
|
self.Entropy.updateProgress(
|
|
darkgreen("Installing Security Advisories"),
|
|
importance = 1,
|
|
type = "info",
|
|
header = red(" # "),
|
|
footer = red(" ...")
|
|
)
|
|
|
|
# clear previous
|
|
self.__clear_previous_advisories()
|
|
# copy over
|
|
self.__put_advisories_in_place()
|
|
# remove temp stuff
|
|
self.__cleanup_garbage()
|
|
|
|
if self.advisories_changed:
|
|
advtext = darkgreen("Security Advisories: updated successfully")
|
|
else:
|
|
advtext = darkred("Security Advisories: already up to date")
|
|
|
|
self.Entropy.updateProgress(
|
|
advtext,
|
|
importance = 2,
|
|
type = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
return 0
|