sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some radical updates to the binary packages management before my vacation, when I will come back I will have to debug all this new code. In this way, it is possible to have the same package name/version in the database because we split it into two branches

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@283 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-04-13 14:59:48 +00:00
parent 1c2aebb47e
commit de4dbf8936
5 changed files with 129 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO
+19 -1
View File
@@ -6,9 +6,27 @@ TODO list (for developers only):
- reagent: complete smartapps section
- mirrorTools: add PASV option (supported/not supported)
- enzyme, reagent: test kernel dependent packages
- enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails
- enzyme, add a module that integrates revdep-rebuild
- add FXP support to activator ?
- add FXP support to activator ? (yes but dreamhost does not support it :|)
- entropy, needed part, add multithreading support
- entropy, databaseTools, finalize support for branches. There can be two packages with the same cat/name-version in the database but with different branch.
CHANGES NEEDED:
- when using retrievePackageVar must be specified the Branch
AT THE END:
- recreate the database, reupload the new packages. resync mirrors.
- If a package is already pulled in, do not add twice:
dhcppc3 handlers # python enzyme overlay sync sabayon; python enzyme build networkmanager nm-applet knetworkmanager kpowersave --force-rebuild
>> synced overlay: sabayon
>> Sanity check on packages...
>> * These are the actions that will be taken, in order:
>> * [BUILD] [U ] net-misc/networkmanager-0.6.5-r1 [overlay: 1 ]
>> * [BUILD] [U ] net-misc/networkmanager-0.6.5-r1 [overlay: 1 ]
>> * [BUILD] [N ] net-misc/nm-applet-0.6.5-r1 [overlay: 1 ]
>> * [BUILD] [U ] net-misc/knetworkmanager-0.1-r3 [overlay: 1 ]
>> * [BUILD] [RS] sys-power/kpowersave-0.7.2-r3 [overlay: 1 ]
!! Remember that kernel dependent packages have: sys-kernel/linux-core-2.6.20-sabayon-r3 (for example)
libraries/activatorTools.py
+6 -4
View File
@@ -320,8 +320,9 @@ def packages(options):
if remotePackage.split()[8] == item:
fileSize = remotePackage.split()[4]
break
totalDownloadSize += int(fileSize)
print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload
totalDownloadSize += int(fileSize)
print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedDownloadQueue.append([item,fileSize])
else:
if (not item.endswith(etpConst['packageshashfileext'])):
@@ -336,8 +337,9 @@ def packages(options):
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
else: # otherwise it is in the packages dir
fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6]
totalUploadSize += int(fileSize)
print_info(bold("\t[") + red("REMOTE UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload
totalUploadSize += int(fileSize)
print_info(bold("\t[") + red("REMOTE UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedUploadQueue.append([item,fileSize])
print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue))))
libraries/databaseTools.py
+95 -53
View File
@@ -642,7 +642,7 @@ def database(options):
brokenPkgsList = []
for pkg in availList:
entropyTools.print_info(entropyTools.red(" Checking Checksum of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True)
entropyTools.print_info(entropyTools.red(" Checking hash of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True)
storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkg,"digest")
result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkg,storedmd5)
if (result):
@@ -863,36 +863,32 @@ class etpDatabase:
# otherwise it fires up updatePackage
def handlePackage(self, etpData, forceBump = False):
if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])):
update, revision = self.addPackage(etpData)
update, revision, etpDataUpdated = self.addPackage(etpData)
else:
update, revision = self.updatePackage(etpData,forceBump)
return update, revision
update, revision, etpDataUpdated = self.updatePackage(etpData,forceBump)
return update, revision, etpDataUpdated
# default add an unstable package
def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"):
# check if the package is slotted
log.log(2,"[DB] Adding package: "+etpData['category']+"/"+etpData['name']+"-"+etpData['version'])
log.log(2," which slot is: "+etpData['slot'])
# Handle package name
etpData['download'] = etpData['download'].split(".tbz2")[0]
# add branch name
etpData['download'] += "-"+wantedBranch+".tbz2"
# if a similar package exist, enter here
searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'])
if (searchsimilar != []):
log.log(2," which searchsimilar is not empty")
# there are other packages with the same category/name
# do we have to remove anything?
removelist = []
for oldpkg in searchsimilar:
# if it's the same, skip
# get the package slot
slot = self.retrievePackageVar(oldpkg,"slot")
branch = self.retrievePackageVar(oldpkg,"branch")
log.log(2," there is: "+oldpkg+" which slot is: "+slot+" and branch: "+branch)
if (etpData['slot'] == slot) and (wantedBranch == branch):
# remove!
log.log(2," unfortunately,"+etpData['category']+"/"+etpData['name']+"-"+etpData['version']+" is similar to "+oldpkg+"because their slot is: "+etpData['slot']+" and branch: "+wantedBranch+". So REMOVING.")
removelist.append(oldpkg)
for pkg in removelist:
self.removePackage(pkg)
searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch)
removelist = []
for oldpkg in searchsimilar:
# get the package slot
slot = self.retrievePackageVar(oldpkg, "slot", branch = wantedBranch)
if (etpData['slot'] == slot):
# remove!
removelist.append(oldpkg)
for pkg in removelist:
self.removePackage(pkg)
# wantedBranch = etpData['branch']
self.cursor.execute(
@@ -929,42 +925,75 @@ class etpDatabase:
)
)
self.commitChanges()
return True,revision
return True, revision, etpData
# Update already available atom in db
# returns True,revision if the package has been updated
# returns False,revision if not
# FIXME: this must be fixed to work with branches, supporting multiple packages with the same key but different branch
def updatePackage(self, etpData, forceBump = False):
# check if the data correspond
# if not, update, else drop
curRevision = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"revision")
curBranch = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"branch")
oldPkgInfo = etpData['category']+"/"+etpData['name']+"-"+etpData['version']
rc = self.comparePackagesData(etpData,oldPkgInfo)
if (not rc) or (forceBump):
# update !
curRevision += 1
# remove the table
self.removePackage(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])
# readd table
self.addPackage(etpData,curRevision,curBranch)
self.commitChanges()
return True, curRevision
# are there any stable packages?
searchsimilarStable = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = "stable")
# filter the one with the same version
stableFound = False
for pkg in searchsimilarStable:
# get version
dbStoredVer = self.retrievePackageVar(pkg, "version", branch = "stable")
if etpData['version'] == dbStoredVer:
# found it !
stablePackage = pkg
stableFound = True
break
if (stableFound):
# in this case, we should compare etpData['neededlibs'] with the db entry to see if there has been a API breakage
dbStoredNeededLibs = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "neededlibs", "stable")
if (etpData['neededlibs'] == dbStoredNeededLibs):
# it is safe to keep it as stable because of:
# - name/version match
# - same libraries requirements
# setup etpData['branch'] accordingly
etpData['branch'] = "stable"
# get selected package revision
if (self.isSpecificPackageAvailable(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'] , etpData['branch'])):
curRevision = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "revision", etpData['branch'])
else:
self.commitChanges()
return False,curRevision
curRevision = 0
# do I really have to update the database entry? If the information are the same, drop all
oldPkgInfo = etpData['category']+"/"+etpData['name']+"-"+etpData['version']
rc = self.comparePackagesData(etpData, oldPkgInfo, dbPkgBranch = etpData['branch'])
if (rc) and (not forceBump):
return False, curRevision, etpData # in this case etpData content does not matter
# OTHERWISE:
# remove the current selected package, if exists
if (self.isSpecificPackageAvailable(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'] , etpData['branch'])):
self.removePackage(etpData['category']+"/"+etpData['name']+"-"+etpData['version'], branch = etpData['branch'])
# bump revision nevertheless
curRevision += 1
# add the new one
self.addPackage(etpData,curRevision,etpData['branch'])
# You must provide the full atom to this function
def removePackage(self,key):
# FIXME: this must be fixed to work with branches
def removePackage(self,key, branch = "unstable"):
key = entropyTools.removePackageOperators(key)
self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'"')
self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'" AND branch = "'+branch+'"')
self.commitChanges()
# WARNING: this function must be kept in sync with Entropy database schema
# returns True if equal
# returns False if not
def comparePackagesData(self,etpData,dbPkgInfo):
# FIXME: this must be fixed to work with branches
def comparePackagesData(self,etpData,dbPkgInfo, dbPkgBranch = "unstable"):
myEtpData = etpData.copy()
@@ -974,7 +1003,7 @@ class etpDatabase:
# fill content
for i in myEtpData:
myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i)
myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i,dbPkgBranch)
for i in etpData:
if etpData[i] != myEtpData[i]:
@@ -983,19 +1012,19 @@ class etpDatabase:
return True
# You must provide the full atom to this function
def retrievePackageInfo(self,pkgkey):
def retrievePackageInfo(self,pkgkey, branch = "unstable"):
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "'+pkgkey+'"')
self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
result.append(row)
return result
# You must provide the full atom to this function
def retrievePackageVar(self,pkgkey,pkgvar):
def retrievePackageVar(self,pkgkey,pkgvar, branch = "unstable"):
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'"')
self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
result.append(row[0])
if len(result) > 0:
@@ -1017,6 +1046,8 @@ class etpDatabase:
return result
# You must provide the full atom to this function
# FIXME: add pkgcat/name split?
# FIXME: do SELECT atom instead of SELECT * ?
def isPackageAvailable(self,pkgkey):
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
@@ -1027,6 +1058,17 @@ class etpDatabase:
return False
return True
# This version is more specific and supports branches
def isSpecificPackageAvailable(self,pkgkey, branch):
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
result.append(row)
if result == []:
return False
return True
def searchPackages(self,keyword):
result = []
self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "%'+keyword+'%"')
@@ -1037,11 +1079,11 @@ class etpDatabase:
# this function search packages with the same pkgcat/pkgname
# you must provide something like: media-sound/amarok
# optionally, you can add version too.
def searchSimilarPackages(self,atom):
def searchSimilarPackages(self,atom, branch = "unstable"):
category = atom.split("/")[0]
name = atom.split("/")[1]
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'"')
self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "'+branch+'"')
for row in self.cursor:
result.append(row[0])
return result
libraries/portageTools.py
+3 -3
View File
@@ -423,9 +423,9 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles):
pkgBinaryFiles = _pkgBinaryFiles
newPkgBinaryFiles = _newPkgBinaryFiles
print "DEBUG:"
print pkgBinaryFiles
print newPkgBinaryFiles
#print "DEBUG:"
#print pkgBinaryFiles
#print newPkgBinaryFiles
# check for version bumps
for pkg in pkgBinaryFiles:
libraries/reagentTools.py
+6 -9
View File
@@ -42,21 +42,20 @@ def generator(package, enzymeRequestBump = False, dbconnection = None):
if (not validFile):
print_warning(package+" does not exist !")
packagename = package.split("/")[len(package.split("/"))-1]
packagename = os.path.basename(package)
print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait..."))
etpData = extractPkgData(package)
# return back also the new possible package filename, so that we can make decisions on that
newFileName = os.path.basename(etpData['download'])
if dbconnection is None:
dbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True)
else:
dbconn = dbconnection
updated, revision = dbconn.handlePackage(etpData,enzymeRequestBump)
updated, revision, etpDataUpdated = dbconn.handlePackage(etpData,enzymeRequestBump)
# return back also the new possible package filename, so that we can make decisions on that
newFileName = os.path.basename(etpDataUpdated['download'])
if dbconnection is None:
dbconn.commitChanges()
@@ -108,8 +107,8 @@ def enzyme(options):
if (rc):
etpCreated += 1
# create .hash file
hashFilePath = createHashFile(tbz2path)
os.system("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f")
hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+newFileName)
os.system("mv "+hashFilePath+" "+etpConst['packagessuploaddir']+"/ -f")
else:
etpNotCreated += 1
@@ -240,8 +239,6 @@ def extractPkgData(package):
# add strict kernel dependency
# done below
# change file name
# modify etpData['download']
# done below