first work on the package reorganization

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@495 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2007-09-13 22:52:57 +00:00
parent 88a112f60e
commit 0f762d2dd5
6 changed files with 93 additions and 97 deletions
+1
View File
@@ -12,6 +12,7 @@ TODO list:
- strengthen build dependencies filter
- write a dependency reordering function
- rewrite branches management
- equo: during removal/add handle /etc/init.d/* files ?
Project Status:
- entropy: will handle all the three tools below
+2 -2
View File
@@ -1329,7 +1329,7 @@ def installPackageIntoDatabase(idpackage, repository, clientDbconn = None):
if clientDbconn == None:
closedb = True
clientDbconn = openClientDatabase()
idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = rev, forcedBranch = True, addBranch = False)
idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = rev, forcedBranch = True)
del x
if (not status):
clientDbconn.closeDB()
@@ -1606,7 +1606,7 @@ def database(options):
# filling
print_info(" "+bold("(")+darkgreen(str(count))+"/"+blue(total)+bold(")")+red(" Injecting ")+bold(atomName), back = True)
# fill client database
idpk, rev, xx, status = clientDbconn.addPackage(atomInfo, wantedBranch = atomBranch, addBranch = False)
idpk, rev, xx, status = clientDbconn.addPackage(atomInfo, wantedBranch = atomBranch)
# now add the package to the installed table
clientDbconn.addPackageToInstalledTable(idpk,x[1])
+31 -31
View File
@@ -75,29 +75,35 @@ def database(options):
activatorTools.packages(["sync","--ask"])
# now fill the database
pkglist = os.listdir(etpConst['packagesbindir'])
# filter .md5
_pkglist = []
for i in pkglist:
if not i.endswith(etpConst['packageshashfileext']):
_pkglist.append(i)
pkglist = _pkglist
pkgbranches = os.listdir(etpConst['packagesbindir'])
pkgbranches = [x for x if os.path.isdir(x)]
for mybranch in pkgbranches:
pkglist = os.listdir(etpConst['packagesbindir']+"/"+mybranch)
# filter .md5
_pkglist = []
for i in pkglist:
if not i.endswith(etpConst['packageshashfileext']):
_pkglist.append(i)
pkglist = _pkglist
print_info(green(" * ")+red("Reinitializing Entropy database using Packages in the repository ..."))
currCounter = 0
atomsnumber = len(pkglist)
import reagentTools
for pkg in pkglist:
print_info(green(" * ")+red("Analyzing: ")+bold(pkg), back = True)
currCounter += 1
print_info(green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(pkg)+red(" ..."))
etpData = reagentTools.extractPkgData(package = etpConst['packagesbindir']+"/"+pkg, structuredLayout = True)
# remove shait
entropyTools.spawnCommand("rm -rf "+etpConst['packagestmpdir']+"/"+pkg+"*")
# fill the db entry
idpk, revision, etpDataUpdated, accepted = dbconn.addPackage(etpData)
print_info(green(" * ")+red("Reinitializing Entropy database using Packages in the repository ..."))
currCounter = 0
atomsnumber = len(pkglist)
import reagentTools
for pkg in pkglist:
print_info(darkgreen(" [")+red(mybranch)+darkgreen("] ")+red("Analyzing: ")+bold(pkg), back = True)
currCounter += 1
print_info(darkgreen(" [")+red(mybranch)+darkgreen("] ")+green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(pkg)+red(" ..."))
etpData = reagentTools.extractPkgData(package = etpConst['packagesbindir']+"/"+mybranch+"/"+pkg, mybranch)
# remove shait
entropyTools.spawnCommand("rm -rf "+etpConst['packagestmpdir']+"/"+pkg+"*")
# fill the db entry
idpk, revision, etpDataUpdated, accepted = dbconn.addPackage(etpData, wantedBranch = mybranch)
dbconn.commitChanges()
dbconn.commitChanges()
# regen dependstable
reagentTools.dependsTableInitialize(dbconn, False)
@@ -866,7 +872,7 @@ class etpDatabase:
# this function manages the submitted package
# if it does not exist, it fires up addPackage
# otherwise it fires up updatePackage
def handlePackage(self, etpData, addBranch = True, forcedRevision = -1, forcedBranch = False):
def handlePackage(self, etpData, forcedRevision = -1, forcedBranch = False):
if (self.readOnly):
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlePackage: Cannot handle this in read only.")
@@ -882,16 +888,16 @@ class etpDatabase:
if (forcedRevision < 0):
forcedRevision = 0
if (forcedBranch):
idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, addBranch = addBranch, revision = forcedRevision, wantedBranch = etpData['branch'])
idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, revision = forcedRevision, wantedBranch = etpData['branch'])
else:
idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, addBranch = addBranch, revision = forcedRevision)
idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, revision = forcedRevision)
else:
idpk, revision, etpDataUpdated, accepted = self.updatePackage(etpData, forcedRevision) # branch and revision info will be overwritten
return idpk, revision, etpDataUpdated, accepted
# FIXME: default add an unstable package ~~ use indexes
def addPackage(self, etpData, revision = 0, wantedBranch = "unstable", addBranch = True):
def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"):
if (self.readOnly):
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: Cannot handle this in read only.")
@@ -899,12 +905,6 @@ class etpDatabase:
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: called.")
if (addBranch):
# Handle package name
etpData['download'] = etpData['download'].split(".tbz2")[0]
# add branch name
etpData['download'] += "-"+wantedBranch+".tbz2"
# if a similar package, in the same branch exists, mark for removal
searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch)
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar))
+10 -3
View File
@@ -379,9 +379,7 @@ etpConst = {
'/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage'
],
'officialrepositoryname': "sabayonlinux.org", # our official repository name
'packagedbdir': "/db", # directory of the database file in the .tbz2 package
'packagedbfile': "/data.db", # database file in the directory above
'packagecontentdir': "/package", # directory of the package file in the .tbz2 package
'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database
'dependenciesfilter': ['sys-devel/automake','sys-devel/autoconf','sys-devel/libtool','dev-util/pkgconfig','sys-devel/make'],
'developmentcategories': ['sys-devel','dev-'],
'pidfile': "/var/run/equo.pid",
@@ -463,6 +461,15 @@ if not os.path.isdir(etpConst['entropyworkdir']):
print "you need to run this as root at least once."
sys.exit(100)
# check for packages and upload directories
if os.getuid() == 0:
for x in etpConst['branches']:
if not os.path.isdir(etpConst['packagesbindir']+"/"+x):
os.makedirs(etpConst['packagesbindir']+"/"+x)
if not os.path.isdir(etpConst['packagessuploaddir']+"/"+x):
os.makedirs(etpConst['packagessuploaddir']+"/"+x)
# entropy section
if (not os.path.isfile(etpConst['entropyconf'])):
print "ERROR: "+etpConst['entropyconf']+" does not exist"
+11
View File
@@ -126,6 +126,17 @@ def extractXpak(tbz2File,tmpdir = None):
tbz2.decompose(tmpdir)
return tmpdir
def aggregateEntropyDb(tbz2file,dbfile):
f = open(tbz2file,"abw")
g = open(dbfile,"rb")
dbx = g.readlines()
# append tag
f.write(etpConst['databasestarttag'])
for x in dbx:
f.write(x)
f.flush()
f.close()
# This function creates the .hash file related to the given package file
# @returns the complete hash file path
# FIXME: add more hashes, SHA1 for example
+38 -61
View File
@@ -56,7 +56,6 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran
print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait..."))
etpData = extractPkgData(package, enzymeRequestBranch)
if dbconnection is None:
dbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True)
else:
@@ -81,7 +80,7 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision)))
return True, newFileName, idpk
elif (accepted) and (revision == 0):
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created.")
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created or version bumped.")
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created."))
return True, newFileName, idpk
else:
@@ -101,7 +100,7 @@ def update(options):
print_info(yellow(" * ")+red("Scanning the database for differences..."))
dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True)
from portageTools import getInstalledPackagesCounters, quickpkg
from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot
installedPackages = getInstalledPackagesCounters()
installedCounters = {}
databasePackages = dbconn.listAllPackages()
@@ -123,7 +122,22 @@ def update(options):
match = installedCounters.get(x[0], None)
#print match
if (not match):
toBeRemoved.append(x[1])
# check if the package is in toBeAdded
if (toBeAdded):
atomkey = dep_getkey(dbconn.retrieveAtom(x[1]))
atomslot = dbconn.retrieveSlot(x[1])
add = True
for pkgdata in toBeAdded:
addslot = getPackageSlot(pkgdata[0])
addkey = dep_getkey(pkgdata[0])
if (atomkey == addkey) and (atomslot == addslot):
# do not add to toBeRemoved
add = False
break
if add:
toBeRemoved.append(x[1])
else:
toBeRemoved.append(x[1])
if (not toBeRemoved) and (not toBeAdded):
print_info(yellow(" * ")+red("Nothing to do, check later."))
@@ -143,10 +157,10 @@ def update(options):
print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True)
rwdbconn.removePackage(x)
rwdbconn.closeDB()
print_info(yellow(" @@ ")+blue("Database removal complete.")+red(atom))
print_info(yellow(" @@ ")+blue("Database removal complete."))
if (toBeAdded):
print_info(yellow(" @@ ")+blue("These are the packages that would be added to the add list:"))
print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:"))
for x in toBeAdded:
print_info(yellow(" # ")+red(x[0]))
rc = askquestion(">> Would you like to packetize them now ?")
@@ -200,35 +214,26 @@ def update(options):
if (rc):
etpCreated += 1
# move the file with its new name
spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f")
spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName+" -f")
print_info(yellow(" * ")+red("Injecting database information into ")+bold(newFileName)+red(", please wait..."), back = True)
# uncompressing
tdir = etpConst['packagestmpdir']+"/injection"
if os.path.isdir(tdir):
os.system("rm -rf "+tdir)
os.makedirs(tdir)
os.mkdir(tdir+etpConst['packagecontentdir']) # content directory
os.mkdir(tdir+etpConst['packagedbdir']) # content directory
dbpath = tdir+etpConst['packagedbdir']+etpConst['packagedbfile']
# fill /package
spawnCommand("mv "+etpConst['packagessuploaddir']+"/"+newFileName+" "+tdir+etpConst['packagecontentdir']+"/")
dbpath = etpConst['packagestmpdir']+"/"+etpConst['packagedbfile']
# create db
pkgDbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True, dbFile = dbpath, clientDatabase = True)
pkgDbconn.initializeDatabase()
data = dbconn.getPackageData(idpk)
rev = dbconn.retrieveRevision(idpk)
# inject
pkgDbconn.addPackage(data, revision = rev, wantedBranch = data['branch'], addBranch = False)
pkgDbconn.addPackage(data, revision = rev, wantedBranch = data['branch'])
pkgDbconn.closeDB()
# recompose the new file
compressTarBz2(etpConst['packagessuploaddir']+"/"+newFileName,tdir+"/")
# update the checksum in the database
digest = md5sum(etpConst['packagessuploaddir']+"/"+newFileName)
# append the database to the new file
aggregateEntropyDb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName, dbfile = dbpath)
digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName)
dbconn.setDigest(idpk,digest)
hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+newFileName)
# remove tdir
spawnCommand("rm -rf "+tdir)
hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName)
# remove garbage
spawnCommand("rm -rf "+dbpath)
print_info(yellow(" * ")+red("Database injection complete for ")+newFileName)
else:
@@ -246,7 +251,7 @@ def update(options):
print_info(green(" * ")+red("Statistics: ")+blue("Entries created/updated: ")+bold(str(etpCreated))+yellow(" - ")+darkblue("Entries discarded: ")+bold(str(etpNotCreated)))
# This function extracts all the info from a .tbz2 file and returns them
def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = False):
def extractPkgData(package, etpBranch = etpConst['branch']):
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"extractPkgData: called -> package: "+str(package))
@@ -257,11 +262,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F
print_info(yellow(" * ")+red("Getting package name/version..."),back = True)
tbz2File = package
package = package.split(".tbz2")[0]
if package.split("-")[len(package.split("-"))-1].startswith("unstable"):
package = string.join(package.split("-unstable")[:len(package.split("-unstable"))-1],"-unstable")
if package.split("-")[len(package.split("-"))-1].startswith("stable"):
etpBranch = "stable"
package = string.join(package.split("-stable")[:len(package.split("-stable"))-1],"-stable")
if package.split("-")[len(package.split("-"))-1].startswith("t"):
package = string.join(package.split("-t")[:len(package.split("-t"))-1],"-t")
@@ -289,16 +289,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F
# .tbz2 md5
etpData['digest'] = md5sum(tbz2File)
if (structuredLayout):
# extract tbz2
structuredPackageDir = etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']+"-structured"
if os.path.isdir(structuredPackageDir):
spawnCommand("rm -rf "+structuredPackageDir)
os.makedirs(structuredPackageDir)
uncompressTarBz2(tbz2File,structuredPackageDir)
tbz2filename = os.path.basename(tbz2File)
tbz2File = structuredPackageDir+etpConst['packagecontentdir']+"/"+tbz2filename
print_info(yellow(" * ")+red("Getting package mtime..."),back = True)
# .tbz2 md5
etpData['datecreation'] = str(getFileUnixMtime(tbz2File))
@@ -434,9 +424,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F
# add strict kernel dependency
# done below
# modify etpData['download']
# done below
print_info(yellow(" * ")+red("Getting package download URL..."),back = True)
# Fill download relative URI
if (kernelDependentModule):
@@ -444,7 +431,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F
versiontag = "-"+etpData['versiontag']
else:
versiontag = ""
etpData['download'] = etpConst['binaryurirelativepath']+etpData['name']+"-"+etpData['version']+versiontag+".tbz2"
etpData['download'] = etpConst['binaryurirelativepath']+"/"+etpData['branch']+"/"+etpData['name']+"-"+etpData['version']+versiontag+".tbz2"
print_info(yellow(" * ")+red("Getting package counter..."),back = True)
# Fill category
@@ -700,9 +687,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F
# removing temporary directory
os.system("rm -rf "+tbz2TmpDir)
if (structuredLayout):
if os.path.isdir(structuredPackageDir):
spawnCommand("rm -rf "+structuredPackageDir)
print_info(yellow(" * ")+red("Done"),back = True)
return etpData
@@ -885,6 +869,7 @@ def smartgenerator(atomInfo):
# check if the application package is available, otherwise, download
pkgfilepath = dbconn.retrieveDownloadURL(idpackage)
pkgcontent = dbconn.retrieveContent(idpackage)
pkgbranch = dbconn.retrieveBranch(idpackage)
pkgfilename = os.path.basename(pkgfilepath)
pkgname = pkgfilename.split(".tbz2")[0]
@@ -915,18 +900,14 @@ def smartgenerator(atomInfo):
#print "DEBUG: "+pkgtmpdir
if os.path.isdir(pkgtmpdir):
spawnCommand("rm -rf "+pkgtmpdir)
os.makedirs(pkgtmpdir+"/content")
mainBinaryPath = etpConst['packagesbindir']+"/"+pkgfilename
mainBinaryPath = etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfilename
print_info(green(" * ")+red("Unpacking main package ")+bold(str(pkgfilename)))
uncompressTarBz2(mainBinaryPath,pkgtmpdir) # first unpack
uncompressTarBz2(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename,pkgtmpdir+"/content") # second unpack
if os.path.isfile(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename):
os.remove(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename)
binaryExecs = []
for file in pkgcontent:
# remove /
filepath = pkgtmpdir+"/content"+file
filepath = pkgtmpdir+file
import commands
if os.access(filepath,os.X_OK):
# test if it's an exec
@@ -937,16 +918,12 @@ def smartgenerator(atomInfo):
# now uncompress all the rest
contentdir = pkgtmpdir+"/content"
for dep in pkgs:
download = os.path.basename(dbconn.retrieveDownloadURL(dep[0]))
depbranch = dbconn.retrieveBranch(dep[0])
print_info(green(" * ")+red("Unpacking dependency package ")+bold(str(download)))
deppath = etpConst['packagesbindir']+"/"+download
deppath = etpConst['packagesbindir']+"/"+depbranch+"/"+download
uncompressTarBz2(deppath,pkgtmpdir) # first unpack
uncompressTarBz2(pkgtmpdir+etpConst['packagecontentdir']+"/"+download,contentdir) # second unpack
if os.path.isfile(pkgtmpdir+etpConst['packagecontentdir']+"/"+download):
os.remove(pkgtmpdir+etpConst['packagecontentdir']+"/"+download)
# remove unwanted files (header files)