From de4dbf893617f2d6925fb7441aa590e6bc0fcc6e Mon Sep 17 00:00:00 2001 From: lxnay Date: Fri, 13 Apr 2007 14:59:48 +0000 Subject: [PATCH] some radical updates to the binary packages management before my vacation, when I will come back I will have to debug all this new code. In this way, it is possible to have the same package name/version in the database because we split it into two branches git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@283 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 20 ++++- libraries/activatorTools.py | 10 ++- libraries/databaseTools.py | 148 +++++++++++++++++++++++------------- libraries/portageTools.py | 6 +- libraries/reagentTools.py | 15 ++-- 5 files changed, 129 insertions(+), 70 deletions(-) diff --git a/TODO b/TODO index 0276ed33a..97b9c6c29 100644 --- a/TODO +++ b/TODO @@ -6,9 +6,27 @@ TODO list (for developers only): - reagent: complete smartapps section - mirrorTools: add PASV option (supported/not supported) - enzyme, reagent: test kernel dependent packages +- enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails - enzyme, add a module that integrates revdep-rebuild -- add FXP support to activator ? +- add FXP support to activator ? (yes but dreamhost does not support it :|) - entropy, needed part, add multithreading support +- entropy, databaseTools, finalize support for branches. There can be two packages with the same cat/name-version in the database but with different branch. + CHANGES NEEDED: + - when using retrievePackageVar must be specified the Branch + AT THE END: + - recreate the database, reupload the new packages. resync mirrors. + +- If a package is already pulled in, do not add twice: +dhcppc3 handlers # python enzyme overlay sync sabayon; python enzyme build networkmanager nm-applet knetworkmanager kpowersave --force-rebuild +>> synced overlay: sabayon +>> Sanity check on packages... +>> * These are the actions that will be taken, in order: +>> * [BUILD] [U ] net-misc/networkmanager-0.6.5-r1 [overlay: 1 ] +>> * [BUILD] [U ] net-misc/networkmanager-0.6.5-r1 [overlay: 1 ] +>> * [BUILD] [N ] net-misc/nm-applet-0.6.5-r1 [overlay: 1 ] +>> * [BUILD] [U ] net-misc/knetworkmanager-0.1-r3 [overlay: 1 ] +>> * [BUILD] [RS] sys-power/kpowersave-0.7.2-r3 [overlay: 1 ] + !! Remember that kernel dependent packages have: sys-kernel/linux-core-2.6.20-sabayon-r3 (for example) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 4d9f919a1..ce11c3c6d 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -320,8 +320,9 @@ def packages(options): if remotePackage.split()[8] == item: fileSize = remotePackage.split()[4] break - totalDownloadSize += int(fileSize) - print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload + totalDownloadSize += int(fileSize) + print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) detailedDownloadQueue.append([item,fileSize]) else: if (not item.endswith(etpConst['packageshashfileext'])): @@ -336,8 +337,9 @@ def packages(options): fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] else: # otherwise it is in the packages dir fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6] - totalUploadSize += int(fileSize) - print_info(bold("\t[") + red("REMOTE UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload + totalUploadSize += int(fileSize) + print_info(bold("\t[") + red("REMOTE UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) detailedUploadQueue.append([item,fileSize]) print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue)))) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 333425530..ee35b1cd2 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -642,7 +642,7 @@ def database(options): brokenPkgsList = [] for pkg in availList: - entropyTools.print_info(entropyTools.red(" Checking Checksum of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True) + entropyTools.print_info(entropyTools.red(" Checking hash of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True) storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkg,"digest") result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkg,storedmd5) if (result): @@ -863,36 +863,32 @@ class etpDatabase: # otherwise it fires up updatePackage def handlePackage(self, etpData, forceBump = False): if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])): - update, revision = self.addPackage(etpData) + update, revision, etpDataUpdated = self.addPackage(etpData) else: - update, revision = self.updatePackage(etpData,forceBump) - return update, revision + update, revision, etpDataUpdated = self.updatePackage(etpData,forceBump) + return update, revision, etpDataUpdated # default add an unstable package def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"): # check if the package is slotted - - log.log(2,"[DB] Adding package: "+etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - log.log(2," which slot is: "+etpData['slot']) + + # Handle package name + etpData['download'] = etpData['download'].split(".tbz2")[0] + # add branch name + etpData['download'] += "-"+wantedBranch+".tbz2" + # if a similar package exist, enter here - searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name']) - if (searchsimilar != []): - log.log(2," which searchsimilar is not empty") - # there are other packages with the same category/name - # do we have to remove anything? - removelist = [] - for oldpkg in searchsimilar: - # if it's the same, skip - # get the package slot - slot = self.retrievePackageVar(oldpkg,"slot") - branch = self.retrievePackageVar(oldpkg,"branch") - log.log(2," there is: "+oldpkg+" which slot is: "+slot+" and branch: "+branch) - if (etpData['slot'] == slot) and (wantedBranch == branch): - # remove! - log.log(2," unfortunately,"+etpData['category']+"/"+etpData['name']+"-"+etpData['version']+" is similar to "+oldpkg+"because their slot is: "+etpData['slot']+" and branch: "+wantedBranch+". So REMOVING.") - removelist.append(oldpkg) - for pkg in removelist: - self.removePackage(pkg) + searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch) + removelist = [] + for oldpkg in searchsimilar: + # get the package slot + slot = self.retrievePackageVar(oldpkg, "slot", branch = wantedBranch) + if (etpData['slot'] == slot): + # remove! + removelist.append(oldpkg) + + for pkg in removelist: + self.removePackage(pkg) # wantedBranch = etpData['branch'] self.cursor.execute( @@ -929,42 +925,75 @@ class etpDatabase: ) ) self.commitChanges() - return True,revision + return True, revision, etpData # Update already available atom in db # returns True,revision if the package has been updated # returns False,revision if not + # FIXME: this must be fixed to work with branches, supporting multiple packages with the same key but different branch def updatePackage(self, etpData, forceBump = False): - # check if the data correspond - # if not, update, else drop - curRevision = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"revision") - curBranch = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"branch") - oldPkgInfo = etpData['category']+"/"+etpData['name']+"-"+etpData['version'] - rc = self.comparePackagesData(etpData,oldPkgInfo) - if (not rc) or (forceBump): - # update ! - curRevision += 1 - # remove the table - self.removePackage(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - # readd table - self.addPackage(etpData,curRevision,curBranch) - self.commitChanges() - return True, curRevision + + # are there any stable packages? + searchsimilarStable = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = "stable") + # filter the one with the same version + stableFound = False + for pkg in searchsimilarStable: + # get version + dbStoredVer = self.retrievePackageVar(pkg, "version", branch = "stable") + if etpData['version'] == dbStoredVer: + # found it ! + stablePackage = pkg + stableFound = True + break + + if (stableFound): + + # in this case, we should compare etpData['neededlibs'] with the db entry to see if there has been a API breakage + dbStoredNeededLibs = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "neededlibs", "stable") + if (etpData['neededlibs'] == dbStoredNeededLibs): + # it is safe to keep it as stable because of: + # - name/version match + # - same libraries requirements + # setup etpData['branch'] accordingly + etpData['branch'] = "stable" + + + # get selected package revision + if (self.isSpecificPackageAvailable(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'] , etpData['branch'])): + curRevision = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "revision", etpData['branch']) else: - self.commitChanges() - return False,curRevision + curRevision = 0 + + # do I really have to update the database entry? If the information are the same, drop all + oldPkgInfo = etpData['category']+"/"+etpData['name']+"-"+etpData['version'] + rc = self.comparePackagesData(etpData, oldPkgInfo, dbPkgBranch = etpData['branch']) + if (rc) and (not forceBump): + return False, curRevision, etpData # in this case etpData content does not matter + + # OTHERWISE: + # remove the current selected package, if exists + if (self.isSpecificPackageAvailable(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'] , etpData['branch'])): + self.removePackage(etpData['category']+"/"+etpData['name']+"-"+etpData['version'], branch = etpData['branch']) + + # bump revision nevertheless + curRevision += 1 + + # add the new one + self.addPackage(etpData,curRevision,etpData['branch']) # You must provide the full atom to this function - def removePackage(self,key): + # FIXME: this must be fixed to work with branches + def removePackage(self,key, branch = "unstable"): key = entropyTools.removePackageOperators(key) - self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'"') + self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'" AND branch = "'+branch+'"') self.commitChanges() # WARNING: this function must be kept in sync with Entropy database schema # returns True if equal # returns False if not - def comparePackagesData(self,etpData,dbPkgInfo): + # FIXME: this must be fixed to work with branches + def comparePackagesData(self,etpData,dbPkgInfo, dbPkgBranch = "unstable"): myEtpData = etpData.copy() @@ -974,7 +1003,7 @@ class etpDatabase: # fill content for i in myEtpData: - myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i) + myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i,dbPkgBranch) for i in etpData: if etpData[i] != myEtpData[i]: @@ -983,19 +1012,19 @@ class etpDatabase: return True # You must provide the full atom to this function - def retrievePackageInfo(self,pkgkey): + def retrievePackageInfo(self,pkgkey, branch = "unstable"): pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] - self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "'+pkgkey+'"') + self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"') for row in self.cursor: result.append(row) return result # You must provide the full atom to this function - def retrievePackageVar(self,pkgkey,pkgvar): + def retrievePackageVar(self,pkgkey,pkgvar, branch = "unstable"): pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] - self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'"') + self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"') for row in self.cursor: result.append(row[0]) if len(result) > 0: @@ -1017,6 +1046,8 @@ class etpDatabase: return result # You must provide the full atom to this function + # FIXME: add pkgcat/name split? + # FIXME: do SELECT atom instead of SELECT * ? def isPackageAvailable(self,pkgkey): pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] @@ -1027,6 +1058,17 @@ class etpDatabase: return False return True + # This version is more specific and supports branches + def isSpecificPackageAvailable(self,pkgkey, branch): + pkgkey = entropyTools.removePackageOperators(pkgkey) + result = [] + self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "'+pkgkey+'" AND branch = "'+branch+'"') + for row in self.cursor: + result.append(row) + if result == []: + return False + return True + def searchPackages(self,keyword): result = [] self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "%'+keyword+'%"') @@ -1037,11 +1079,11 @@ class etpDatabase: # this function search packages with the same pkgcat/pkgname # you must provide something like: media-sound/amarok # optionally, you can add version too. - def searchSimilarPackages(self,atom): + def searchSimilarPackages(self,atom, branch = "unstable"): category = atom.split("/")[0] name = atom.split("/")[1] result = [] - self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'"') + self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "'+branch+'"') for row in self.cursor: result.append(row[0]) return result diff --git a/libraries/portageTools.py b/libraries/portageTools.py index 4b4a4c76f..eb3ee610a 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -423,9 +423,9 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles): pkgBinaryFiles = _pkgBinaryFiles newPkgBinaryFiles = _newPkgBinaryFiles - print "DEBUG:" - print pkgBinaryFiles - print newPkgBinaryFiles + #print "DEBUG:" + #print pkgBinaryFiles + #print newPkgBinaryFiles # check for version bumps for pkg in pkgBinaryFiles: diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 4889d591e..1b132025c 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -42,21 +42,20 @@ def generator(package, enzymeRequestBump = False, dbconnection = None): if (not validFile): print_warning(package+" does not exist !") - packagename = package.split("/")[len(package.split("/"))-1] + packagename = os.path.basename(package) print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait...")) etpData = extractPkgData(package) - # return back also the new possible package filename, so that we can make decisions on that - newFileName = os.path.basename(etpData['download']) - - if dbconnection is None: dbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) else: dbconn = dbconnection - updated, revision = dbconn.handlePackage(etpData,enzymeRequestBump) + updated, revision, etpDataUpdated = dbconn.handlePackage(etpData,enzymeRequestBump) + + # return back also the new possible package filename, so that we can make decisions on that + newFileName = os.path.basename(etpDataUpdated['download']) if dbconnection is None: dbconn.commitChanges() @@ -108,8 +107,8 @@ def enzyme(options): if (rc): etpCreated += 1 # create .hash file - hashFilePath = createHashFile(tbz2path) os.system("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f") + hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+newFileName) os.system("mv "+hashFilePath+" "+etpConst['packagessuploaddir']+"/ -f") else: etpNotCreated += 1 @@ -240,8 +239,6 @@ def extractPkgData(package): # add strict kernel dependency # done below - # change file name - # modify etpData['download'] # done below