From 08dfec939616e852fe1fa147fb25f1d7a1ee92b5 Mon Sep 17 00:00:00 2001 From: lxnay Date: Thu, 5 Apr 2007 22:11:51 +0000 Subject: [PATCH] some nice daily fixes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@270 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + libraries/databaseTools.py | 47 ++++++++++++++++---- libraries/entropyConstants.py | 1 + libraries/entropyTools.py | 5 +++ libraries/logTools.py | 84 +++++++++++++++++++++++++++++++++++ libraries/reagentTools.py | 49 +++++++++++++------- 6 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 libraries/logTools.py diff --git a/TODO b/TODO index e5d8405a2..4ffc57002 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,7 @@ TODO list (for developers only): - build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form) - entropy: add support for stable/unstable branch (test it) - reagent: get orphans working and not taking forever +- improve logTools and put it in an usable status - activator tasks: - test SYNC (that includes the tidy tool) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 44790f8b0..7464796fe 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -33,6 +33,10 @@ import os import sys import string +# load the log file +import logTools +log = logTools.LogFile(level=2,filename = etpConst['databaselogfile']) + # TIP OF THE DAY: # never nest closeDB() and re-init inside a loop !!!!!!!!!!!! NEVER ! @@ -45,6 +49,7 @@ def database(options): # do some check, print some warnings entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Initializing Entropy database..."), back = True) + log.log(0,"[DB OP] Called database --initialize") # database file: etpConst['etpdatabasefilepath'] if os.path.isfile(etpConst['etpdatabasefilepath']): entropyTools.print_info(entropyTools.red(" * ")+entropyTools.bold("WARNING")+entropyTools.red(": database file already exists. Overwriting.")) @@ -52,12 +57,15 @@ def database(options): if rc == "No": sys.exit(0) os.system("rm -f "+etpConst['etpdatabasefilepath']) + log.log(0,"[DB OP] Removed old database file") # initialize the database + log.log(0,"[DB OP] Connecting to the database") dbconn = etpDatabase(readOnly = False, noUpload = True) dbconn.initializeDatabase() # sync packages directory + log.log(0,"Syncing binary packages") import activatorTools activatorTools.packages(["sync","--ask"]) @@ -65,21 +73,28 @@ def database(options): pkglist = os.listdir(etpConst['packagesbindir']) entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Reinitializing Entropy database using Packages in the repository ...")) + log.log(0,"[DB OP] Preparing to start reinitialization") currCounter = 0 atomsnumber = len(pkglist) import reagentTools for pkg in pkglist: - + log.log(0,"[DB OP] Analyzing "+str(pkg)) entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Analyzing: ")+entropyTools.bold(pkg), back = True) currCounter += 1 entropyTools.print_info(entropyTools.green(" (")+ entropyTools.blue(str(currCounter))+"/"+entropyTools.red(str(atomsnumber))+entropyTools.green(") ")+entropyTools.red("Analyzing ")+entropyTools.bold(pkg)+entropyTools.red(" ...")) etpData = reagentTools.extractPkgData(etpConst['packagesbindir']+"/"+pkg) + log.log(3,"[DB OP] etpData status (should be properly filled now):") + for i in etpData: + log.log(3,i+": "+etpData[i]) + # remove shait os.system("rm -rf "+etpConst['packagestmpdir']+"/"+pkg) # fill the db entry + log.log(0,"[DB OP] Launching etpDatabase.addPackage()") dbconn.addPackage(etpData) dbconn.commitChanges() + log.close() dbconn.closeDB() entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Entropy database has been reinitialized using binary packages available")) @@ -145,7 +160,7 @@ def database(options): for conflict in conflicts: entropyTools.print_info(entropyTools.darkred("\t # Conflict: ")+conflict) entropyTools.print_info(entropyTools.red("\t Entry API: ")+entropyTools.green(result[24])) - entropyTools.print_info(entropyTools.red("\t Entry creation date: ")+str(result[25])) + entropyTools.print_info(entropyTools.red("\t Entry creation date: ")+str(entropyTools.convertUnixTimeToHumanTime(int(result[25])))) if (result[26]): entropyTools.print_info(entropyTools.red("\t Built with needed libraries")) libs = result[26].split() @@ -707,7 +722,7 @@ class etpDatabase: # this function manages the submitted package # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage - def handlePackage(self,etpData,forceBump = False): + def handlePackage(self, etpData, forceBump = False): if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])): update, revision = self.addPackage(etpData) else: @@ -715,23 +730,27 @@ class etpDatabase: return update, revision # default add an unstable package - def addPackage(self,etpData, revision = 0, wantedBranch = "unstable"): + def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"): # check if the package is slotted + log.log(2,"[DB] Adding package: "+etpData['category']+"/"+etpData['name']+"-"+etpData['version']) + log.log(2," which slot is: "+etpData['slot']) # if a similar package exist, enter here - # FIXME: for future reference, add an option that forces a package to stay? - # NOTE: this never removes stable packages. will be done by the stabilize function! - searchsimilar = self.searchPackages(etpData['category']+"/"+etpData['name']) + searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name']) if (searchsimilar != []): + log.log(2," which searchsimilar is not empty") # there are other packages with the same category/name # do we have to remove anything? removelist = [] for oldpkg in searchsimilar: + # if it's the same, skip # get the package slot slot = self.retrievePackageVar(oldpkg[0],"slot") branch = self.retrievePackageVar(oldpkg[0],"branch") + log.log(2," there is: "+oldpkg[0]+" which slot is: "+slot+" and branch: "+branch) if (etpData['slot'] == slot) and (wantedBranch == branch): # remove! + log.log(2," unfortunately,"+etpData['category']+"/"+etpData['name']+"-"+etpData['version']+" is similar to "+oldpkg[0]+"because their slot is: "+etpData['slot']+" and branch: "+wantedBranch+". So REMOVING.") removelist.append(oldpkg[0]) for pkg in removelist: self.removePackage(pkg) @@ -776,7 +795,7 @@ class etpDatabase: # Update already available atom in db # returns True,revision if the package has been updated # returns False,revision if not - def updatePackage(self,etpData,forceBump = False): + def updatePackage(self, etpData, forceBump = False): # check if the data correspond # if not, update, else drop curRevision = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"revision") @@ -860,6 +879,18 @@ class etpDatabase: result.append(row) return result + # this function search packages with the same pkgcat/pkgname + # you must provide something like: media-sound/amarok + # optionally, you can add version too. + def searchSimilarPackages(self,atom): + category = atom.split("/")[0] + name = atom.split("/")[1] + result = [] + self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'"') + for row in self.cursor: + result.append(row[0]) + return result + def listAllPackages(self): result = [] self.cursor.execute('SELECT * FROM etpData') diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 81faa90dd..f7ecaa777 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -159,6 +159,7 @@ etpConst = { 'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db" 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit + 'databaselogfile': ETP_LOG_DIR+"/database.log", # database operations log file 'distcc-status': False, # used by Enzyme, if True distcc is enabled 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file 'etpdatabasedir': ETP_DIR+ETP_DBDIR, # diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 0485a6c7d..6135132e4 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -867,6 +867,11 @@ def convertUnixTimeToMtime(unixtime): outputtime += chr return outputtime +def convertUnixTimeToHumanTime(unixtime): + from datetime import datetime + humantime = str(datetime.fromtimestamp(unixtime)) + return humantime + # get a list, returns a sorted list def alphaSorter(seq): def stripter(s, goodchrs): diff --git a/libraries/logTools.py b/libraries/logTools.py new file mode 100644 index 000000000..003d69996 --- /dev/null +++ b/libraries/logTools.py @@ -0,0 +1,84 @@ +#!/usr/bin/python +''' + # DESCRIPTION: + # Entropy log facility + + Copyright (C) 2007 Fabio Erculiani + # Most of the code taken from Anaconda, copyrighted by: + # Alexander Larsson + # Matt Wilson + # + # Copyright 2002 Red Hat, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' + +import sys + +class LogFile: + def __init__ (self, level = 0, filename = None): + self.handler = self.default_handler + self.level = level + self.logFile = None + self.open(filename) + + def close (self): + try: + self.logFile.close () + except: + pass + + def open (self, file = None): + if type(file) == type("hello"): + try: + self.logFile = open(file, "w") + except: + self.logFile = sys.stderr + elif file: + self.logFile = file + else: + self.logFile = sys.stderr + + def getFile (self): + return self.logFile.fileno () + + def __call__(self, format, *args): + self.handler (format % args) + + def default_handler (self, string): + self.logFile.write ("* %s\n" % (string)) + self.logFile.flush () + + def set_loglevel(self, level): + self.level = level + + def log(self, level, message): + if self.level >= level: + self.handler(message) + + def ladd(self, level, file, message): + if self.level >= level: + self.handler("++ %s \t%s" % (file, message)) + + def ldel(self, level, file, message): + if self.level >= level: + self.handler("-- %s \t%s" % (file, message)) + + def lch(self, level, file, message): + if self.level >= level: + self.handler("-+ %s \t%s" % (file, message)) + + +log = LogFile() diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 2703f328d..3807d826d 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -39,8 +39,7 @@ def generator(package, enzymeRequestBump = False, dbconnection = None): if os.path.isfile(package) and package.endswith(".tbz2"): validFile = True if (not validFile): - print_error("no valid .tbz2 file specified") - sys.exit(501) + print_warning(package+" does not exist !") packagename = package.split("/")[len(package.split("/"))-1] @@ -447,10 +446,9 @@ def smartapps(options): print_info(green(" * ")+red("Smartapps creation done, remember to test them before publishing.")) - - # tool that generates .tar.bz2 packages with all the binary dependencies included # @returns the package file path +# NOTE: this section is highly portage dependent def smartgenerator(atom): dbconn = databaseTools.etpDatabase(readOnly = True) @@ -481,9 +479,14 @@ def smartgenerator(atom): for dep in extraDeps: depnames = dbconn.searchPackages(dep) for depname in depnames: - _extraDeps.append(depname[0]) # get atom - - extraDeps = _extraDeps + _extraDeps.append(depname[0]) + if depname[0].find("dev-libs/glib") != -1: + # add pango + pangopkgs = dbconn.searchSimilarPackages("x11-libs/pango") + for pangopkg in pangopkgs: + extraDeps.append(pangopkg) + + extraDeps = list(set(_extraDeps)) extraPackages = [] # get their files @@ -563,7 +566,7 @@ def smartgenerator(atom): if len(glibcPkg) > 0: glibcContent = dbconn.retrievePackageVar(glibcPkg[0][0],"content") for file in glibcContent.split(): - if (file.startswith("/lib/")) and (file.startswith("/lib64/")) and (file.find(".so") != -1): + if ((file.startswith("/lib/")) or (file.startswith("/lib64/"))) and (file.find(".so") != -1): librariesBlacklist.append(file) # add here more blacklisted files @@ -586,23 +589,30 @@ def smartgenerator(atom): if not os.path.isdir(pkgtmpdir+libdir): os.makedirs(pkgtmpdir+libdir) os.system("cp -p "+lib+" "+pkgtmpdir+libdir) - # ^^ libraries copied in place! now link! (with magic) pkgneededlibs = _pkgneededlibs # collect libraries in the directories + # catch /usr/lib/gcc/ + gcclibpath = "" + for i in pkgneededlibs: + if i.startswith("/usr/lib/gcc"): + gcclibpath += ":"+i + break + # now create the bash script for each binaryExecs os.makedirs(pkgtmpdir+"/wrp") bashScript = [] bashScript.append( '#!/bin/sh\n' 'cd $1\n' - 'export PATH=$PWD:$PWD/sbin:$PWD/bin:$PWD/usr/bin:$PWD/usr/sbin:$PWD/usr/X11R6/bin:$PWD/libexec:$PWD/usr/local/bin:$PWD/usr/local/sbin:$PATH\n' - 'export LD_LIBRARY_PATH=$PWD/lib:$PWD/lib64:$PWD/usr/lib:$PWD/usr/lib64:$PWD/usr/qt/3/lib:$PWD/usr/qt/3/lib64:$PWD/usr/kde/3.5/lib:$PWD/usr/kde/3.5/lib64:$LD_LIBRARY_PATH\n' - 'export KDEDIRS=$PWD/usr/kde/3.5:$PWD/usr:$KDEDIRS\n' - + 'MYPYP=$(find $PWD/lib/python2.4/site-packages/ -type d -printf %p: 2> /dev/null)\n' 'MYPYP2=$(find $PWD/lib/python2.5/site-packages/ -type d -printf %p: 2> /dev/null)\n' 'export PYTHONPATH=$MYPYP:MYPYP2:$PYTHONPATH\n' + + 'export PATH=$PWD:$PWD/sbin:$PWD/bin:$PWD/usr/bin:$PWD/usr/sbin:$PWD/usr/X11R6/bin:$PWD/libexec:$PWD/usr/local/bin:$PWD/usr/local/sbin:$PATH\n' + 'export LD_LIBRARY_PATH=$PWD/lib:$PWD/lib64'+gcclibpath+':$PWD/usr/lib:$PWD/usr/lib64:$PWD/usr/qt/3/lib:$PWD/usr/qt/3/lib64:$PWD/usr/kde/3.5/lib:$PWD/usr/kde/3.5/lib64:$LD_LIBRARY_PATH\n' + 'export KDEDIRS=$PWD/usr/kde/3.5:$PWD/usr:$KDEDIRS\n' 'export PERL5LIB=$PWD/usr/lib/perl5:$PWD/share/perl5:$PWD/usr/lib/perl5/5.8.1' ':$PWD/usr/lib/perl5/5.8.2:' @@ -618,6 +628,16 @@ def smartgenerator(atom): 'export MANPATH=$PWD/share/man:$MANPATH\n' 'export GUILE_LOAD_PATH=$PWD/share/:$GUILE_LOAD_PATH\n' 'export SCHEME_LIBRARY_PATH=$PWD/share/slib:$SCHEME_LIBRARY_PATH\n' + + '# Setup pango\n' + 'MYPANGODIR=$(find $PWD/usr/lib/pango -name modules)\n' + 'if [ -n "$MYPANGODIR" ]; then\n' + ' export PANGO_RC_FILE=$PWD/etc/pango/pangorc\n' + ' echo "[Pango]" > $PANGO_RC_FILE\n' + ' echo "ModulesPath=${MYPANGODIR}" >> $PANGO_RC_FILE\n' + ' echo "ModuleFiles=${PWD}/etc/pango/pango.modules" >> $PANGO_RC_FILE\n' + ' pango-querymodules > ${PWD}/etc/pango/pango.modules\n' + 'fi\n' '$2\n' ) f = open(pkgtmpdir+"/wrp/wrapper","w") @@ -641,7 +661,6 @@ def smartgenerator(atom): ' int rc = system(\n' ' "pid=$(pidof '+file+'.exe);"\n' ' "listpid=$(ps x | grep $pid);"\n' - ' "listpid=$(ps x | grep $pid);"\n' ' "filename=$(echo $listpid | cut -d\' \' -f 5);"' ' "currdir=$(dirname $filename);"\n' ' "/bin/sh $currdir/wrp/wrapper $currdir '+file+'" );\n' @@ -654,7 +673,7 @@ def smartgenerator(atom): f.close() # now compile os.system("cd "+pkgtmpdir+"/ ; g++ -Wall "+file+".cc -o "+file+".exe") - os.remove(pkgtmpdir+"/"+file+".cc") + #os.remove(pkgtmpdir+"/"+file+".cc") # now compress in .tar.bz2 and place in etpConst['smartappsdir'] #print etpConst['smartappsdir']+"/"+pkgname+"-"+etpConst['currentarch']+".tar.bz2"