From 0f762d2dd5842f280cb982fc817f8cf176260ccd Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Thu, 13 Sep 2007 22:52:57 +0000 Subject: [PATCH] first work on the package reorganization git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@495 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + client/equoTools.py | 4 +- libraries/databaseTools.py | 62 +++++++++++----------- libraries/entropyConstants.py | 13 +++-- libraries/entropyTools.py | 11 ++++ libraries/reagentTools.py | 99 ++++++++++++++--------------------- 6 files changed, 93 insertions(+), 97 deletions(-) diff --git a/TODO b/TODO index c69e719a9..d9cb8af0f 100644 --- a/TODO +++ b/TODO @@ -12,6 +12,7 @@ TODO list: - strengthen build dependencies filter - write a dependency reordering function - rewrite branches management + - equo: during removal/add handle /etc/init.d/* files ? Project Status: - entropy: will handle all the three tools below diff --git a/client/equoTools.py b/client/equoTools.py index 16dbbb051..faf10c6da 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -1329,7 +1329,7 @@ def installPackageIntoDatabase(idpackage, repository, clientDbconn = None): if clientDbconn == None: closedb = True clientDbconn = openClientDatabase() - idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = rev, forcedBranch = True, addBranch = False) + idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = rev, forcedBranch = True) del x if (not status): clientDbconn.closeDB() @@ -1606,7 +1606,7 @@ def database(options): # filling print_info(" "+bold("(")+darkgreen(str(count))+"/"+blue(total)+bold(")")+red(" Injecting ")+bold(atomName), back = True) # fill client database - idpk, rev, xx, status = clientDbconn.addPackage(atomInfo, wantedBranch = atomBranch, addBranch = False) + idpk, rev, xx, status = clientDbconn.addPackage(atomInfo, wantedBranch = atomBranch) # now add the package to the installed table clientDbconn.addPackageToInstalledTable(idpk,x[1]) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 5ee7c71d7..1fc13a22e 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -75,29 +75,35 @@ def database(options): activatorTools.packages(["sync","--ask"]) # now fill the database - pkglist = os.listdir(etpConst['packagesbindir']) - # filter .md5 - _pkglist = [] - for i in pkglist: - if not i.endswith(etpConst['packageshashfileext']): - _pkglist.append(i) - pkglist = _pkglist + pkgbranches = os.listdir(etpConst['packagesbindir']) + pkgbranches = [x for x if os.path.isdir(x)] + + for mybranch in pkgbranches: + + pkglist = os.listdir(etpConst['packagesbindir']+"/"+mybranch) + + # filter .md5 + _pkglist = [] + for i in pkglist: + if not i.endswith(etpConst['packageshashfileext']): + _pkglist.append(i) + pkglist = _pkglist - print_info(green(" * ")+red("Reinitializing Entropy database using Packages in the repository ...")) - currCounter = 0 - atomsnumber = len(pkglist) - import reagentTools - for pkg in pkglist: - print_info(green(" * ")+red("Analyzing: ")+bold(pkg), back = True) - currCounter += 1 - print_info(green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(pkg)+red(" ...")) - etpData = reagentTools.extractPkgData(package = etpConst['packagesbindir']+"/"+pkg, structuredLayout = True) - # remove shait - entropyTools.spawnCommand("rm -rf "+etpConst['packagestmpdir']+"/"+pkg+"*") - # fill the db entry - idpk, revision, etpDataUpdated, accepted = dbconn.addPackage(etpData) + print_info(green(" * ")+red("Reinitializing Entropy database using Packages in the repository ...")) + currCounter = 0 + atomsnumber = len(pkglist) + import reagentTools + for pkg in pkglist: + print_info(darkgreen(" [")+red(mybranch)+darkgreen("] ")+red("Analyzing: ")+bold(pkg), back = True) + currCounter += 1 + print_info(darkgreen(" [")+red(mybranch)+darkgreen("] ")+green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(pkg)+red(" ...")) + etpData = reagentTools.extractPkgData(package = etpConst['packagesbindir']+"/"+mybranch+"/"+pkg, mybranch) + # remove shait + entropyTools.spawnCommand("rm -rf "+etpConst['packagestmpdir']+"/"+pkg+"*") + # fill the db entry + idpk, revision, etpDataUpdated, accepted = dbconn.addPackage(etpData, wantedBranch = mybranch) - dbconn.commitChanges() + dbconn.commitChanges() # regen dependstable reagentTools.dependsTableInitialize(dbconn, False) @@ -866,7 +872,7 @@ class etpDatabase: # this function manages the submitted package # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage - def handlePackage(self, etpData, addBranch = True, forcedRevision = -1, forcedBranch = False): + def handlePackage(self, etpData, forcedRevision = -1, forcedBranch = False): if (self.readOnly): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlePackage: Cannot handle this in read only.") @@ -882,16 +888,16 @@ class etpDatabase: if (forcedRevision < 0): forcedRevision = 0 if (forcedBranch): - idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, addBranch = addBranch, revision = forcedRevision, wantedBranch = etpData['branch']) + idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, revision = forcedRevision, wantedBranch = etpData['branch']) else: - idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, addBranch = addBranch, revision = forcedRevision) + idpk, revision, etpDataUpdated, accepted = self.addPackage(etpData, revision = forcedRevision) else: idpk, revision, etpDataUpdated, accepted = self.updatePackage(etpData, forcedRevision) # branch and revision info will be overwritten return idpk, revision, etpDataUpdated, accepted # FIXME: default add an unstable package ~~ use indexes - def addPackage(self, etpData, revision = 0, wantedBranch = "unstable", addBranch = True): + def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"): if (self.readOnly): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: Cannot handle this in read only.") @@ -899,12 +905,6 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: called.") - if (addBranch): - # Handle package name - etpData['download'] = etpData['download'].split(".tbz2")[0] - # add branch name - etpData['download'] += "-"+wantedBranch+".tbz2" - # if a similar package, in the same branch exists, mark for removal searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch) dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar)) diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index c73842348..6fd3577f7 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -379,9 +379,7 @@ etpConst = { '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' ], 'officialrepositoryname': "sabayonlinux.org", # our official repository name - 'packagedbdir': "/db", # directory of the database file in the .tbz2 package - 'packagedbfile': "/data.db", # database file in the directory above - 'packagecontentdir': "/package", # directory of the package file in the .tbz2 package + 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database 'dependenciesfilter': ['sys-devel/automake','sys-devel/autoconf','sys-devel/libtool','dev-util/pkgconfig','sys-devel/make'], 'developmentcategories': ['sys-devel','dev-'], 'pidfile': "/var/run/equo.pid", @@ -463,6 +461,15 @@ if not os.path.isdir(etpConst['entropyworkdir']): print "you need to run this as root at least once." sys.exit(100) + +# check for packages and upload directories +if os.getuid() == 0: + for x in etpConst['branches']: + if not os.path.isdir(etpConst['packagesbindir']+"/"+x): + os.makedirs(etpConst['packagesbindir']+"/"+x) + if not os.path.isdir(etpConst['packagessuploaddir']+"/"+x): + os.makedirs(etpConst['packagessuploaddir']+"/"+x) + # entropy section if (not os.path.isfile(etpConst['entropyconf'])): print "ERROR: "+etpConst['entropyconf']+" does not exist" diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 7528c9b60..b7843671c 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -126,6 +126,17 @@ def extractXpak(tbz2File,tmpdir = None): tbz2.decompose(tmpdir) return tmpdir +def aggregateEntropyDb(tbz2file,dbfile): + f = open(tbz2file,"abw") + g = open(dbfile,"rb") + dbx = g.readlines() + # append tag + f.write(etpConst['databasestarttag']) + for x in dbx: + f.write(x) + f.flush() + f.close() + # This function creates the .hash file related to the given package file # @returns the complete hash file path # FIXME: add more hashes, SHA1 for example diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index dd400e4e7..52ba1d4af 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -56,7 +56,6 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait...")) etpData = extractPkgData(package, enzymeRequestBranch) - if dbconnection is None: dbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) else: @@ -81,7 +80,7 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision))) return True, newFileName, idpk elif (accepted) and (revision == 0): - reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created.") + reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created or version bumped.") print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created.")) return True, newFileName, idpk else: @@ -101,7 +100,7 @@ def update(options): print_info(yellow(" * ")+red("Scanning the database for differences...")) dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True) - from portageTools import getInstalledPackagesCounters, quickpkg + from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot installedPackages = getInstalledPackagesCounters() installedCounters = {} databasePackages = dbconn.listAllPackages() @@ -123,7 +122,22 @@ def update(options): match = installedCounters.get(x[0], None) #print match if (not match): - toBeRemoved.append(x[1]) + # check if the package is in toBeAdded + if (toBeAdded): + atomkey = dep_getkey(dbconn.retrieveAtom(x[1])) + atomslot = dbconn.retrieveSlot(x[1]) + add = True + for pkgdata in toBeAdded: + addslot = getPackageSlot(pkgdata[0]) + addkey = dep_getkey(pkgdata[0]) + if (atomkey == addkey) and (atomslot == addslot): + # do not add to toBeRemoved + add = False + break + if add: + toBeRemoved.append(x[1]) + else: + toBeRemoved.append(x[1]) if (not toBeRemoved) and (not toBeAdded): print_info(yellow(" * ")+red("Nothing to do, check later.")) @@ -143,10 +157,10 @@ def update(options): print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True) rwdbconn.removePackage(x) rwdbconn.closeDB() - print_info(yellow(" @@ ")+blue("Database removal complete.")+red(atom)) + print_info(yellow(" @@ ")+blue("Database removal complete.")) if (toBeAdded): - print_info(yellow(" @@ ")+blue("These are the packages that would be added to the add list:")) + print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:")) for x in toBeAdded: print_info(yellow(" # ")+red(x[0])) rc = askquestion(">> Would you like to packetize them now ?") @@ -200,35 +214,26 @@ def update(options): if (rc): etpCreated += 1 # move the file with its new name - spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f") - + spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName+" -f") print_info(yellow(" * ")+red("Injecting database information into ")+bold(newFileName)+red(", please wait..."), back = True) - # uncompressing - tdir = etpConst['packagestmpdir']+"/injection" - if os.path.isdir(tdir): - os.system("rm -rf "+tdir) - os.makedirs(tdir) - os.mkdir(tdir+etpConst['packagecontentdir']) # content directory - os.mkdir(tdir+etpConst['packagedbdir']) # content directory - dbpath = tdir+etpConst['packagedbdir']+etpConst['packagedbfile'] - # fill /package - spawnCommand("mv "+etpConst['packagessuploaddir']+"/"+newFileName+" "+tdir+etpConst['packagecontentdir']+"/") + + dbpath = etpConst['packagestmpdir']+"/"+etpConst['packagedbfile'] # create db pkgDbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True, dbFile = dbpath, clientDatabase = True) pkgDbconn.initializeDatabase() data = dbconn.getPackageData(idpk) rev = dbconn.retrieveRevision(idpk) # inject - pkgDbconn.addPackage(data, revision = rev, wantedBranch = data['branch'], addBranch = False) + pkgDbconn.addPackage(data, revision = rev, wantedBranch = data['branch']) pkgDbconn.closeDB() - # recompose the new file - compressTarBz2(etpConst['packagessuploaddir']+"/"+newFileName,tdir+"/") - # update the checksum in the database - digest = md5sum(etpConst['packagessuploaddir']+"/"+newFileName) + # append the database to the new file + aggregateEntropyDb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName, dbfile = dbpath) + + digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName) dbconn.setDigest(idpk,digest) - hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+newFileName) - # remove tdir - spawnCommand("rm -rf "+tdir) + hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName) + # remove garbage + spawnCommand("rm -rf "+dbpath) print_info(yellow(" * ")+red("Database injection complete for ")+newFileName) else: @@ -246,7 +251,7 @@ def update(options): print_info(green(" * ")+red("Statistics: ")+blue("Entries created/updated: ")+bold(str(etpCreated))+yellow(" - ")+darkblue("Entries discarded: ")+bold(str(etpNotCreated))) # This function extracts all the info from a .tbz2 file and returns them -def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = False): +def extractPkgData(package, etpBranch = etpConst['branch']): reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"extractPkgData: called -> package: "+str(package)) @@ -257,11 +262,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F print_info(yellow(" * ")+red("Getting package name/version..."),back = True) tbz2File = package package = package.split(".tbz2")[0] - if package.split("-")[len(package.split("-"))-1].startswith("unstable"): - package = string.join(package.split("-unstable")[:len(package.split("-unstable"))-1],"-unstable") - if package.split("-")[len(package.split("-"))-1].startswith("stable"): - etpBranch = "stable" - package = string.join(package.split("-stable")[:len(package.split("-stable"))-1],"-stable") if package.split("-")[len(package.split("-"))-1].startswith("t"): package = string.join(package.split("-t")[:len(package.split("-t"))-1],"-t") @@ -289,16 +289,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F # .tbz2 md5 etpData['digest'] = md5sum(tbz2File) - if (structuredLayout): - # extract tbz2 - structuredPackageDir = etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']+"-structured" - if os.path.isdir(structuredPackageDir): - spawnCommand("rm -rf "+structuredPackageDir) - os.makedirs(structuredPackageDir) - uncompressTarBz2(tbz2File,structuredPackageDir) - tbz2filename = os.path.basename(tbz2File) - tbz2File = structuredPackageDir+etpConst['packagecontentdir']+"/"+tbz2filename - print_info(yellow(" * ")+red("Getting package mtime..."),back = True) # .tbz2 md5 etpData['datecreation'] = str(getFileUnixMtime(tbz2File)) @@ -434,9 +424,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F # add strict kernel dependency # done below - # modify etpData['download'] - # done below - print_info(yellow(" * ")+red("Getting package download URL..."),back = True) # Fill download relative URI if (kernelDependentModule): @@ -444,7 +431,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F versiontag = "-"+etpData['versiontag'] else: versiontag = "" - etpData['download'] = etpConst['binaryurirelativepath']+etpData['name']+"-"+etpData['version']+versiontag+".tbz2" + etpData['download'] = etpConst['binaryurirelativepath']+"/"+etpData['branch']+"/"+etpData['name']+"-"+etpData['version']+versiontag+".tbz2" print_info(yellow(" * ")+red("Getting package counter..."),back = True) # Fill category @@ -700,9 +687,6 @@ def extractPkgData(package, etpBranch = etpConst['branch'], structuredLayout = F # removing temporary directory os.system("rm -rf "+tbz2TmpDir) - if (structuredLayout): - if os.path.isdir(structuredPackageDir): - spawnCommand("rm -rf "+structuredPackageDir) print_info(yellow(" * ")+red("Done"),back = True) return etpData @@ -885,6 +869,7 @@ def smartgenerator(atomInfo): # check if the application package is available, otherwise, download pkgfilepath = dbconn.retrieveDownloadURL(idpackage) pkgcontent = dbconn.retrieveContent(idpackage) + pkgbranch = dbconn.retrieveBranch(idpackage) pkgfilename = os.path.basename(pkgfilepath) pkgname = pkgfilename.split(".tbz2")[0] @@ -915,18 +900,14 @@ def smartgenerator(atomInfo): #print "DEBUG: "+pkgtmpdir if os.path.isdir(pkgtmpdir): spawnCommand("rm -rf "+pkgtmpdir) - os.makedirs(pkgtmpdir+"/content") - mainBinaryPath = etpConst['packagesbindir']+"/"+pkgfilename + mainBinaryPath = etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfilename print_info(green(" * ")+red("Unpacking main package ")+bold(str(pkgfilename))) uncompressTarBz2(mainBinaryPath,pkgtmpdir) # first unpack - uncompressTarBz2(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename,pkgtmpdir+"/content") # second unpack - if os.path.isfile(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename): - os.remove(pkgtmpdir+etpConst['packagecontentdir']+"/"+pkgfilename) binaryExecs = [] for file in pkgcontent: # remove / - filepath = pkgtmpdir+"/content"+file + filepath = pkgtmpdir+file import commands if os.access(filepath,os.X_OK): # test if it's an exec @@ -937,16 +918,12 @@ def smartgenerator(atomInfo): # now uncompress all the rest - contentdir = pkgtmpdir+"/content" for dep in pkgs: download = os.path.basename(dbconn.retrieveDownloadURL(dep[0])) + depbranch = dbconn.retrieveBranch(dep[0]) print_info(green(" * ")+red("Unpacking dependency package ")+bold(str(download))) - deppath = etpConst['packagesbindir']+"/"+download + deppath = etpConst['packagesbindir']+"/"+depbranch+"/"+download uncompressTarBz2(deppath,pkgtmpdir) # first unpack - uncompressTarBz2(pkgtmpdir+etpConst['packagecontentdir']+"/"+download,contentdir) # second unpack - if os.path.isfile(pkgtmpdir+etpConst['packagecontentdir']+"/"+download): - os.remove(pkgtmpdir+etpConst['packagecontentdir']+"/"+download) - # remove unwanted files (header files)