huge work on equo

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@384 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-08-04 18:12:49 +00:00
parent bc51e118e6
commit bf47ae1bdb
6 changed files with 270 additions and 94 deletions
+1
View File
@@ -48,6 +48,7 @@ def print_help():
print_info(" \t\t"+darkgreen("install")+red("\t\t\t to install a package"))
print_info(" \t\t\t"+red("--ask")+"\t\t ask before making any changes")
print_info(" \t\t\t"+red("--pretend")+"\t just show what would be done")
print_info(" \t\t\t"+red("--nodeps")+"\t do not manage any dependency")
print_info(" \t\t\t"+red("--verbose")+"\t show more details about what's going on")
print_info(" \t"+darkgreen(bold("database"))+brown("\t to handle installed packages database"))
+214 -46
View File
@@ -31,7 +31,7 @@ from entropyConstants import *
from clientConstants import *
from outputTools import *
from remoteTools import downloadData
from entropyTools import unpackGzip, compareMd5, bytesIntoHuman, convertUnixTimeToHumanTime, askquestion, getRandomNumber, dep_getcpv, isjustname, dep_getkey, compareVersions as entropyCompareVersions, catpkgsplit, filterDuplicatedEntries, extactDuplicatedEntries, isspecific, uncompressTarBz2
from entropyTools import unpackGzip, compareMd5, bytesIntoHuman, convertUnixTimeToHumanTime, askquestion, getRandomNumber, dep_getcpv, isjustname, dep_getkey, compareVersions as entropyCompareVersions, catpkgsplit, filterDuplicatedEntries, extactDuplicatedEntries, isspecific, uncompressTarBz2, extractXpak
from databaseTools import etpDatabase
import xpak
@@ -891,11 +891,11 @@ def generateDependencyTree(unsatisfiedDeps):
remainingDeps = unsatisfiedDeps[:]
dependenciesNotFound = []
treeview = []
tree = {}
treedepth = 0
tree[treedepth] = remainingDeps
depsOk = False
while (not depsOk):
#print "depth "+str(treedepth)
#print "neededdeps -> "+str(unsatisfiedDeps)
treedepth += 1
for undep in unsatisfiedDeps:
# obtain its dependencies
@@ -921,6 +921,8 @@ def generateDependencyTree(unsatisfiedDeps):
pass
# merge back remainingDeps into unsatisfiedDeps
remainingDeps = list(set(remainingDeps))
if (remainingDeps):
tree[treedepth] = remainingDeps[:]
unsatisfiedDeps = remainingDeps[:]
for x in unsatisfiedDeps:
treeview.append(x)
@@ -933,23 +935,38 @@ def generateDependencyTree(unsatisfiedDeps):
if (dependenciesNotFound):
# Houston, we've got a problem
print "error! DEPS NOT FOUND -> "+str(dependenciesNotFound)
treeview = dependenciesNotFound
treeview = {}
treeview[0] = {}
treeview[0][0] = dependenciesNotFound
return treeview,False
# filter duplicates
treedata = []
if (treeview):
treeview = list(set(treeview))
for atom in treeview:
data = atomMatch(atom)
treedata.append(data)
# filter duplicates
if (treedata):
treedata = list(set(treedata))
newtree = {} # tree list
if (tree):
for x in tree:
newtree[x] = []
for y in tree[x]:
newtree[x].append(atomMatch(y))
if (newtree[x]):
newtree[x] = list(set(newtree[x]))
# now filter newtree
treelength = len(newtree)
for count in range(treelength)[::-1]:
x = 0
while x < count:
# remove dups in this list
for z in newtree[count]:
try:
while 1:
newtree[x].remove(z)
#print "removing "+str(z)
except:
pass
x += 1
del tree
#print treeview
dbconn.closeDB()
return treedata,True
return newtree,True # treeview is used to show deps while tree is used to run the dependency code.
'''
@@ -960,24 +977,49 @@ def generateDependencyTree(unsatisfiedDeps):
@note: this is the function that should be used for 3rd party applications after using atomMatch()
'''
def getRequiredPackages(foundAtoms):
deplist = []
deptree = {}
depcount = -1
for atomInfo in foundAtoms:
depcount += 1
deps = getNeededDependencies(atomInfo)
treedata = ''
if (deps):
results = generateDependencyTree(deps)
if (results[1]):
results = results[0]
else:
return -1
for result in results:
deplist.append(result)
newtree, result = generateDependencyTree(deps)
if (not result):
return newtree, result
if (newtree):
deptree[depcount] = newtree.copy()
#print deptree[depcount]
# clean duplicates
if (deplist):
deplist = list(set(deplist))
newdeptree = deptree.copy() # tree list
if (deptree):
# now filter newtree
treelength = len(newdeptree)
for count in range(treelength)[::-1]:
pkglist = []
#print count
for x in newdeptree[count]:
for y in newdeptree[count][x]:
pkglist.append(y)
#print len(pkglist)
# remove dups in the other lists
for pkg in pkglist:
x = 0
while x < count:
#print x
for z in newdeptree[x]:
try:
while 1:
newdeptree[x][z].remove(pkg)
#print "removing "+str(pkg)
except:
pass
x += 1
del deptree
return deplist
return newdeptree,True
'''
@description: using given information (atom), retrieves idpackage of the installed atom
@@ -1085,7 +1127,14 @@ def fetchFile(url,digest = False):
return 0
return 0
def installFile(package):
'''
@description: unpack the given file on the system and also update gentoo db if requested
@input package: package file (without path)
@output: 0 = all fine, >0 = error!
'''
# FIXME: add gentoo db handling
def installFile(package, infoDict = None):
import shutil
pkgpath = etpConst['packagesbindir']+"/"+package
if not os.path.isfile(pkgpath):
@@ -1146,8 +1195,95 @@ def installFile(package):
return 4
os.chown(tofile,user,group)
shutil.copystat(fromfile,tofile)
os.system("rm -rf "+imageDir)
if infoDict is not None:
rc = installPackageIntoGentooDatabase(infoDict,unpackDir+etpConst['packagecontentdir']+"/"+package)
if (rc >= 0):
os.system("rm -rf "+unpackDir)
return rc
# remove unpack dir
os.system("rm -rf "+unpackDir)
return 0
'''
@description: inject the database information into the Gentoo database
@input package: dictionary containing information collected by installPackages (important are atom, slot, category, name, version)
@output: 0 = all fine, >0 = error!
'''
def installPackageIntoGentooDatabase(infoDict,packageFile):
# handle gentoo-compat
_portage_avail = False
try:
from portageTools import getInstalledAtoms as _portage_getInstalledAtoms, getPackageSlot as _portage_getPackageSlot, getPortageAppDbPath as _portage_getPortageAppDbPath
_portage_avail = True
except:
return -1 # no Portage support
if (_portage_avail):
portDbDir = _portage_getPortageAppDbPath()
# extract xpak from unpackDir+etpConst['packagecontentdir']+"/"+package
key = infoDict['category']+"/"+infoDict['name']
#print _portage_getInstalledAtom(key)
atomsfound = _portage_getInstalledAtoms(key)
### REMOVE
# parse slot and match and remove)
if atomsfound is not None:
pkgToRemove = ''
for atom in atomsfound:
atomslot = _portage_getPackageSlot(atom)
if atomslot == infoDict['slot']:
#print "match slot, remove -> "+str(atomslot)
pkgToRemove = atom
break
if (pkgToRemove):
removePath = portDbDir+pkgToRemove
os.system("rm -rf "+removePath)
#print "removing -> "+removePath
### INSTALL NEW
extractPath = os.path.dirname(packageFile)
extractPath += "/xpak"
extractXpak(packageFile,extractPath)
if not os.path.isdir(portDbDir+infoDict['category']):
os.makedirs(portDbDir+infoDict['category'])
os.rename(extractPath,portDbDir+infoDict['category']+"/"+infoDict['name']+"-"+infoDict['version'])
return 0
'''
@description: unpack the given file on the system and also update gentoo db if requested
@input package: package file (without path)
@output: 0 = all fine, >0 = error!
'''
def installPackageIntoDatabase(idpackage,repository):
# fetch info
dbconn = openRepositoryDatabase(repository)
data = dbconn.getPackageData(idpackage)
# get current revision
rev = dbconn.retrieveRevision(idpackage)
branch = dbconn.retrieveBranch(idpackage)
dbconn.closeDB()
# inject
clientDbconn = openClientDatabase()
idpk, rev, x, status = clientDbconn.addPackage(data, revision = rev, wantedBranch = branch, addBranch = False)
del x
if (not status):
clientDbconn.closeDB()
return 1 # it hasn't been insterted ? why??
# add idpk to the installedtable
clientDbconn.removePackageFromInstalledTable(idpk)
clientDbconn.addPackageToInstalledTable(idpk,repository)
clientDbconn.closeDB()
return 0
########################################################
####
## Database Tools
@@ -1164,6 +1300,7 @@ def package(options):
equoRequestPretend = False
equoRequestPackagesCheck = False
equoRequestVerbose = False
equoRequestDeps = True
rc = 0
_myopts = []
for opt in myopts:
@@ -1173,6 +1310,8 @@ def package(options):
equoRequestPretend = True
elif (opt == "--verbose"):
equoRequestVerbose = True
elif (opt == "--nodeps"):
equoRequestDeps = False
else:
_myopts.append(opt)
myopts = _myopts
@@ -1183,7 +1322,7 @@ def package(options):
if (options[0] == "install"):
if len(myopts) > 0:
rc,status = installPackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose)
rc,status = installPackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, deps = equoRequestDeps)
else:
print_error(red(" Nothing to do."))
rc = 127
@@ -1428,7 +1567,7 @@ def openClientDatabase():
## Actions Handling
#
def installPackages(packages, ask = False, pretend = False, verbose = False):
def installPackages(packages, ask = False, pretend = False, verbose = False, deps = True):
# check if I am root
if (not checkRoot()) and (not pretend):
@@ -1532,12 +1671,27 @@ def installPackages(packages, ask = False, pretend = False, verbose = False):
print_info(red(" @@ ")+blue("Calculating..."))
reqpackages = getRequiredPackages(foundAtoms)
# add dependencies
for dep in reqpackages:
runQueue.append(dep)
if (deps):
treepackages, result = getRequiredPackages(foundAtoms)
# add dependencies, explode them
if (not result):
print_error(red(" @@ ")+blue("Cannot find needed dependencies: ")+str(treepackages))
return 130
pkgs = []
for x in range(len(treepackages))[::-1]:
#print x
for z in treepackages[x]:
#print treepackages[x][z]
for a in treepackages[x][z]:
pkgs.append(a)
#print pkgs
for dep in pkgs:
runQueue.append(dep)
# remove duplicates
runQueue = [x for x in runQueue if x not in foundAtoms]
# add our requested packages at the end
for atomInfo in foundAtoms:
runQueue.append(atomInfo)
@@ -1566,12 +1720,18 @@ def installPackages(packages, ask = False, pretend = False, verbose = False):
pkgslot = dbconn.retrieveSlot(packageInfo[0])
pkgdigest = dbconn.retrieveDigest(packageInfo[0])
pkgfile = dbconn.retrieveDownloadURL(packageInfo[0])
pkgcat = dbconn.retrieveCategory(packageInfo[0])
pkgname = dbconn.retrieveName(packageInfo[0])
# fill action queue
actionQueue[pkgatom] = {}
actionQueue[pkgatom]['repository'] = packageInfo[1]
actionQueue[pkgatom]['idpackage'] = packageInfo[0]
actionQueue[pkgatom]['slot'] = pkgslot
actionQueue[pkgatom]['atom'] = pkgatom
actionQueue[pkgatom]['version'] = pkgver
actionQueue[pkgatom]['category'] = pkgcat
actionQueue[pkgatom]['name'] = pkgname
actionQueue[pkgatom]['remove'] = -1
actionQueue[pkgatom]['download'] = etpRepositories[packageInfo[1]]['packages']+"/"+os.path.basename(pkgfile)
actionQueue[pkgatom]['checksum'] = pkgdigest
@@ -1591,13 +1751,14 @@ def installPackages(packages, ask = False, pretend = False, verbose = False):
if (pkginstalled):
# we need to match slot
for idx in pkginstalled:
#print clientDbconn.retrieveAtom(idx)
islot = clientDbconn.retrieveSlot(idx)
if islot == pkgslot:
# found
installedVer = clientDbconn.retrieveVersion(idx)
installedTag = clientDbconn.retrieveVersionTag(idx)
if not installedTag:
installedTag = "NoTag"
installedTag = ''
installedRev = clientDbconn.retrieveRevision(idx)
actionQueue[pkgatom]['remove'] = idx
break
@@ -1637,6 +1798,11 @@ def installPackages(packages, ask = False, pretend = False, verbose = False):
print_info(red(" @@ ")+red("Packages needing downgrade:\t")+red(str(pkgsToDowngrade)))
print_info(red(" @@ ")+blue("Download size:\t\t\t")+bold(str(bytesIntoHuman(downloadSize))))
if (ask):
rc = askquestion(" Would you like to continue with the installation ?")
if rc == "No":
return 0,0
# running tasks
totalqueue = str(len(runQueue))
@@ -1662,8 +1828,6 @@ def installPackages(packages, ask = False, pretend = False, verbose = False):
# install
steps.append("install")
steps.append("database")
if (etpConst['gentoo-compat']):
steps.append("gentoo-sync")
steps.append("cleanup")
#print "steps for "+pkgatom+" -> "+str(steps)
@@ -1688,30 +1852,34 @@ def stepExecutor(step,infoDict):
output = fetchFile(infoDict['download'],infoDict['checksum'])
if output != 0:
if output == -1:
errormsg = red("Cannot find the package file online. Try to run: ")+bold("equo repo sync")+red("' and this command again.")
errormsg = red("Cannot find the package file online. Try to run: ")+bold("equo repo sync")+red("' and this command again. Error "+str(output))
else:
errormsg = red("Package checksum does not match. Try to run: '")+bold("equo repo sync")+red("' and this command again.")
errormsg = red("Package checksum does not match. Try to run: '")+bold("equo repo sync")+red("' and this command again. Error "+str(output))
print_error(errormsg)
return output
# otherwise fetch md5 too
print_info(red(" ## ")+blue("Fetching package checksum: ")+red(os.path.basename(infoDict['download']+etpConst['packageshashfileext'])))
output = fetchFile(infoDict['download']+etpConst['packageshashfileext'],False)
if output != 0:
errormsg = red("Cannot find the checksum file online. Try to run: ")+bold("equo repo sync")+red("' and this command again.")
errormsg = red("Cannot find the checksum file online. Try to run: ")+bold("equo repo sync")+red("' and this command again. Error "+str(output))
print_error(errormsg)
return output
elif step == "install":
print_info(red(" ## ")+blue("Installing package: ")+red(os.path.basename(infoDict['download'])))
output = installFile(os.path.basename(infoDict['download']))
if (etpConst['gentoo-compat']):
print_info(red(" ## ")+blue("Installing package: ")+red(os.path.basename(infoDict['download']))+" ## w/Gentoo compatibility")
output = installFile(os.path.basename(infoDict['download']),infoDict)
else:
print_info(red(" ## ")+blue("Installing package: ")+red(os.path.basename(infoDict['download'])))
output = installFile(os.path.basename(infoDict['download']))
if output != 0:
errormsg = red("An error occured while trying to install the package. Check if you have enough disk space on your hard disk.")
errormsg = red("An error occured while trying to install the package. Check if you have enough disk space on your hard disk. Error "+str(output))
print_error(errormsg)
return output
elif step == "databasex":
elif step == "database":
print_info(red(" ## ")+blue("Injecting into database: ")+red(os.path.basename(infoDict['download'])))
output = injectIntoDatabase(infoDict['idpackage'],infoDict['repository'])
output = installPackageIntoDatabase(infoDict['idpackage'],infoDict['repository'])
if output != 0:
errormsg = red("An error occured while trying to add the package to the database. What have you done?")
errormsg = red("An error occured while trying to add the package to the database. What have you done? Error "+str(output))
print_error(errormsg)
return output
+40 -1
View File
@@ -433,4 +433,43 @@ else:
else:
print "WARNING: invalid loglevel in: "+etpConst['databaseconf']
import time
time.sleep(5)
time.sleep(5)
# Portage /var/db/<pkgcat>/<pkgname-pkgver>/*
# you never know if gentoo devs change these things
dbDESCRIPTION = "DESCRIPTION"
dbHOMEPAGE = "HOMEPAGE"
dbCHOST = "CHOST"
dbCATEGORY = "CATEGORY"
dbCFLAGS = "CFLAGS"
dbCXXFLAGS = "CXXFLAGS"
dbLICENSE = "LICENSE"
dbSRC_URI = "SRC_URI"
dbUSE = "USE"
dbIUSE = "IUSE"
dbSLOT = "SLOT"
dbPROVIDE = "PROVIDE"
dbDEPEND = "DEPEND"
dbRDEPEND = "RDEPEND"
dbPDEPEND = "PDEPEND"
dbNEEDED = "NEEDED"
dbOR = "|or|"
dbKEYWORDS = "KEYWORDS"
dbCONTENTS = "CONTENTS"
dbPORTAGE_ELOG_OPTS = 'PORTAGE_ELOG_CLASSES="warn info log" PORTAGE_ELOG_SYSTEM="save" PORT_LOGDIR="'+etpConst['logdir']+'"'
# Portage variables reference
# vdbVARIABLE --> $VARIABLE
vdbPORTDIR = "PORTDIR"
vdbPORTDIR_OVERLAY = "PORTDIR_OVERLAY"
# Portage & misc commands
cdbEMERGE = "emerge"
cdbRunEmerge = vdbPORTDIR+"='"+etpConst['portagetreedir']+"' "+vdbPORTDIR_OVERLAY+"='"+etpConst['overlays']+"' "+cdbEMERGE
cdbStartDistcc = "/etc/init.d/distccd start --nodeps"
cdbStopDistcc = "/etc/init.d/distccd stop --nodeps"
cdbStatusDistcc = "/etc/init.d/distccd status"
# Portage options
odbBuild = " -b "
odbNodeps = " --nodeps "
+4
View File
@@ -362,6 +362,10 @@ def removePackageOperators(atom):
# Version compare function taken from portage_versions.py
# portage_versions.py -- core Portage functionality
# Copyright 1998-2006 Gentoo Foundation
ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\\d*)*)(-r(\\d+))?$")
suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
def compareVersions(ver1, ver2, silent=1):
if ver1 == ver2:
+10 -7
View File
@@ -45,8 +45,7 @@ os.environ['DISTDIR'] = etpConst['distfilesdir']
import portage
import portage_const
from portage_dep import isvalidatom, isspecific, isjustname, dep_getkey, dep_getcpv #FIXME: Use the ones from entropyTools
from entropyConstants import *
from serverConstants import *
#from serverConstants import *
initializePortageTree()
# colours support
@@ -58,12 +57,14 @@ import sys
import os
import commands
import entropyTools
import enzymeTools
# Logging initialization
import logTools
portageLog = logTools.LogFile(level=etpConst['spmbackendloglevel'],filename = etpConst['spmbackendlogfile'], header = "[Portage]")
# portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFunction: example. ")
try:
import logTools
portageLog = logTools.LogFile(level=etpConst['spmbackendloglevel'],filename = etpConst['spmbackendlogfile'], header = "[Portage]")
# portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFunction: example. ")
except:
pass
############
# Functions and Classes
@@ -152,7 +153,7 @@ def calculateFullAtomsDependencies(atoms, deep = False, extraopts = ""):
def calculateAtomUSEFlags(atom):
portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"calculateAtomUSEFlags: called -> "+str(atom))
try:
@@ -325,6 +326,7 @@ def unmerge(atom):
# TO THIS FUNCTION:
# must be provided a valid and complete atom
def extractPkgNameVer(atom):
import enzymeTools
portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"extractPkgNameVer: called -> "+atom)
package = dep_getcpv(atom)
package = atom.split("/")[len(atom.split("/"))-1]
@@ -345,6 +347,7 @@ def extractPkgNameVer(atom):
return pkgname,pkgver
def emerge(atom, options, outfile = None, redirect = "&>", simulate = False):
import enzymeTools
portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"emerge: called -> "+atom+" | options: "+str(options)+" | redirect: "+str(redirect)+" | outfile: "+str(outfile)+" | simulate: "+str(simulate))
if (simulate):
return 0,"" # simulation enabled
+1 -40
View File
@@ -266,43 +266,4 @@ else:
url = line.split("|")[2].strip()
if not url.endswith("/"):
url = url+"/"
etpRemoteSupport[servername] = url
# Portage /var/db/<pkgcat>/<pkgname-pkgver>/*
# you never know if gentoo devs change these things
dbDESCRIPTION = "DESCRIPTION"
dbHOMEPAGE = "HOMEPAGE"
dbCHOST = "CHOST"
dbCATEGORY = "CATEGORY"
dbCFLAGS = "CFLAGS"
dbCXXFLAGS = "CXXFLAGS"
dbLICENSE = "LICENSE"
dbSRC_URI = "SRC_URI"
dbUSE = "USE"
dbIUSE = "IUSE"
dbSLOT = "SLOT"
dbPROVIDE = "PROVIDE"
dbDEPEND = "DEPEND"
dbRDEPEND = "RDEPEND"
dbPDEPEND = "PDEPEND"
dbNEEDED = "NEEDED"
dbOR = "|or|"
dbKEYWORDS = "KEYWORDS"
dbCONTENTS = "CONTENTS"
dbPORTAGE_ELOG_OPTS = 'PORTAGE_ELOG_CLASSES="warn info log" PORTAGE_ELOG_SYSTEM="save" PORT_LOGDIR="'+etpConst['logdir']+'"'
# Portage variables reference
# vdbVARIABLE --> $VARIABLE
vdbPORTDIR = "PORTDIR"
vdbPORTDIR_OVERLAY = "PORTDIR_OVERLAY"
# Portage & misc commands
cdbEMERGE = "emerge"
cdbRunEmerge = vdbPORTDIR+"='"+etpConst['portagetreedir']+"' "+vdbPORTDIR_OVERLAY+"='"+etpConst['overlays']+"' "+cdbEMERGE
cdbStartDistcc = "/etc/init.d/distccd start --nodeps"
cdbStopDistcc = "/etc/init.d/distccd stop --nodeps"
cdbStatusDistcc = "/etc/init.d/distccd status"
# Portage options
odbBuild = " -b "
odbNodeps = " --nodeps "
etpRemoteSupport[servername] = url