From aaf3d1fc13afbe7958d634e792db16dcf1997393 Mon Sep 17 00:00:00 2001 From: lxnay Date: Thu, 29 Mar 2007 16:54:38 +0000 Subject: [PATCH] more work on the SQLite integration git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@212 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- handlers/activator | 6 +- handlers/reagent | 6 +- libraries/activatorTools.py | 2 +- libraries/databaseTools.py | 138 +++++++++++++++++++++++++---- libraries/entropyConstants.py | 73 +++++++++++----- libraries/entropyTools.py | 6 +- libraries/reagentTools.py | 159 ++++++---------------------------- 7 files changed, 209 insertions(+), 181 deletions(-) diff --git a/handlers/activator b/handlers/activator index f2d41f7ab..e6fe37bc3 100644 --- a/handlers/activator +++ b/handlers/activator @@ -86,16 +86,16 @@ if (not entropyTools.isRoot()): # sync mirrors tool # will be removed -if (options[0] == "sync"): +elif (options[0] == "sync"): activatorTools.sync(options[1:]) sys.exit(0) # database tool -if (options[0] == "database"): +elif (options[0] == "database"): activatorTools.database(options[1:]) sys.exit(0) # database tool -if (options[0] == "packages"): +elif (options[0] == "packages"): activatorTools.packages(options[1:]) sys.exit(0) \ No newline at end of file diff --git a/handlers/reagent b/handlers/reagent index 0fe6e964f..975f6bc23 100644 --- a/handlers/reagent +++ b/handlers/reagent @@ -45,12 +45,12 @@ def print_help(): entropyTools.print_info(entropyTools.blue("Tools available: ")) entropyTools.print_info(" \t"+entropyTools.green("generator")+entropyTools.yellow("\t to add the package to the database, revision bump if needed")) entropyTools.print_info(" \t\t"+entropyTools.red("--force-bump")+"\t\t to force the revision bumping of the package entry") - entropyTools.print_info(" \t"+entropyTools.green("digest")+entropyTools.yellow("\t\t to digest a specified directory")) entropyTools.print_info(" \t"+entropyTools.green("enzyme")+entropyTools.yellow("\t\t to analyze enzyme store directory")) entropyTools.print_info(" \t\t"+entropyTools.red("--force-bump")+"\t\t to force the revision bumping of the package entries") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t Entropy database tool manager.")) entropyTools.print_info(" \t\t"+entropyTools.red("--initialize")+"\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]") entropyTools.print_info(" \t\t"+entropyTools.green("search")+"\t\t\t Search a package inside the Entropy packages database") + entropyTools.print_info(" \t\t"+entropyTools.green("dump-package-info")+"\t Dump the stored information of package atom(s) to a file") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("cleanup"))+entropyTools.yellow("\t\t to clean temporary files")) options = sys.argv[1:] @@ -79,10 +79,6 @@ if len(options) < 1 or string.join(options).find("--help") != -1 or string.join( if (not entropyTools.isRoot()): print_error("you must be root in order to run "+APPNAME) -# digest tool (creates a digest on the specified directory) -if (options[0] == "digest"): - reagentTools.createDigest(options[1:]) - sys.exit(0) elif (options[0] == "generator"): reagentTools.generator(options[1:]) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 3f0296411..33300b0a9 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -63,7 +63,7 @@ def sync(options): print_info(green(" * ")+red("Collecting local Entropy repository entries..."),back = True) localtetpcounter = 0 localEtpFiles = [] - for (dir, sub, files) in os.walk(etpConst['packagesdatabasedir']): + for (dir, sub, files) in os.walk(etpConst['etpdatabasedir']): localEtpFiles.append(dir) for file in files: localEtpFiles.append(dir+"/"+file) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index a6e053e44..127ce1f04 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -51,14 +51,35 @@ def database(options): # fill the database dbconn = etpDatabase() dbconn.initializeDatabase() + + print_info(green(" * ")+red("Reinitializing Entropy database using Portage database...")) + # now run quickpkg for all the packages and then extract data + installedAtoms, atomsnumber = getInstalledPackages() + currCounter = 0 + import reagentTools + for atom in installedAtoms: + currCounter += 1 + print_info(green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(atom)+red(" ...")) + quickpkg(atom,etpConst['packagestmpdir']) + # file is etpConst['packagestmpdir']+"/atomscan/"+pkgnamever.tbz2 + etpData = reagentTools.extractPkgData(etpConst['packagestmpdir']+"/"+atom.split("/")[1]+".tbz2") + # fill the db entry + dbconn.addPackage(etpData) + os.system("rm -rf "+etpConst['packagestmpdir']+"/"+atom.split("/")[1]+"*") + dbconn.commitChanges() + dbconn.commitChanges() dbconn.closeDB() - print_info(green(" * ")+red("Entropy database initialized.")) + print_info(green(" * ")+red("Entropy database has been reinitialized using Portage database entries")) + elif (options[0] == "search"): mykeywords = options[1:] if (len(mykeywords) == 0): print_error(yellow(" * ")+red("Not enough parameters")) sys.exit(302) + if (not os.path.isfile(etpConst['etpdatabasefile'])): + print_error(yellow(" * ")+red("Entropy Datbase does not exist")) + sys.exit(303) # search tool print_info(green(" * ")+red("Searching inside the Entropy database...")) dbconn = etpDatabase() @@ -91,33 +112,56 @@ def database(options): if (result[17]): print_info(red("\t Slot: ")+yellow(result[17])) #print_info(red("\t Blah: ")+result[18]) # I don't need to print mirrorlinks - if (result[19]): - deps = result[19].split() + if (result[20]): + deps = result[20].split() print_info(red("\t Dependencies")) for dep in deps: print_info(darkred("\t # Depends on: ")+dep) #print_info(red("\t Blah: ")+result[20]) --> it's a dup of [21] - if (result[21]): - rundeps = result[21].split() + if (result[22]): + rundeps = result[22].split() print_info(red("\t Built with runtime dependencies")) for rundep in rundeps: print_info(darkred("\t # Dependency: ")+rundep) - if (result[22]): + if (result[23]): print_info(red("\t Conflicts with")) - conflicts = result[22].split() + conflicts = result[23].split() for conflict in conflicts: print_info(darkred("\t # Conflict: ")+conflict) - print_info(red("\t Entry API: ")+green(result[23])) - print_info(red("\t Entry revision: ")+str(result[24])) + print_info(red("\t Entry API: ")+green(result[24])) + print_info(red("\t Entry revision: ")+str(result[25])) #print result print dbconn.closeDB() + elif (options[0] == "dump-package-info"): + mypackages = options[1:] + if (len(mypackages) == 0): + print_error(yellow(" * ")+red("Not enough parameters")) + sys.exit(302) + for package in mypackages: + print_info(green(" * ")+red("Searching package ")+bold(package)+red(" ...")) + if isjustname(package) or (package.find("/") == -1): + print_warning(yellow(" * ")+red("Package ")+bold(package)+red(" is not a complete atom.")) + continue + # open db connection + dbconn = etpDatabase() + if (not dbconn.isPackageAvailable(package)): + # package does not exist in the Entropy database + print_warning(yellow(" * ")+red("Package ")+bold(package)+red(" does not exist in Entropy database.")) + dbconn.closeDB() + continue + etpData = dbconn.retrievePackageInfo(package) + print etpData + dbconn.closeDB() + ############ # Functions ##################################################################################### + + class etpDatabase: def __init__(self): @@ -130,8 +174,20 @@ class etpDatabase: self.connection.close() def commitChanges(self): + self.taintDatabase() self.connection.commit() + def taintDatabase(self): + # taint the database status + f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'],"w") + f.write(etpConst['currentarch']+" database tainted\n") + f.flush() + f.close() + + def untaintDatabase(self): + # untaint the database status + os.system("rm -f "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) + def discardChanges(self): self.connection.rollback() @@ -143,16 +199,16 @@ class etpDatabase: # this function manages the submitted package # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage - def handlePackage(self,etpData): + def handlePackage(self,etpData,forceBump = False): if (not isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])): self.addPackage(etpData) else: - self.updatePackage(etpData) + self.updatePackage(etpData,forceBump) def addPackage(self,etpData, revision = 0): self.cursor.execute( 'INSERT into etpData VALUES ' - '(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' + '(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' , ( etpData['category']+"/"+etpData['name']+"-"+etpData['version'], etpData['name'], etpData['version'], @@ -171,6 +227,7 @@ class etpDatabase: etpData['digest'], etpData['sources'], etpData['slot'], + etpData['content'], etpData['mirrorlinks'], etpData['dependencies'], etpData['rundependencies'], @@ -181,16 +238,19 @@ class etpDatabase: ) ) self.commitChanges() + return True,revision # Update already available atom in db # returns True,revision if the package has been updated # returns False,revision if not - def updatePackage(self,etpData): + def updatePackage(self,etpData,forceBump = False): # check if the data correspond # if not, update, else drop - curRevision = dbconn.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"revision") - oldPkgInfo = retrievePackageInfo(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - if etpData != oldPkgInfo: + curRevision = self.retrievePackageVar(etpData['category']+"/"+etpData['name']+"-"+etpData['version'],"revision") + # FIXME: I don't know if this works + oldPkgInfo = self.retrievePackageInfo(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) + rc = self.comparePackagesData(etpData,oldPkgInfo) + if (not rc) or (forceBump): # update ! curRevision += 1 # remove the table @@ -209,6 +269,52 @@ class etpDatabase: self.commitChanges() return result + # WARNING: this function must be kept in sync with Entropy database schema + # returns True if equal + # returns False if not + def comparePackagesData(self,etpData,dbPkgInfo): + + # reset before using the tmpEtpData dictionary + for i in tmpEtpData: + tmpEtpData[i] = "" + + # fill content + for i in tmpEtpData: + tmpEtpData[i] = self.retrievePackageVar(dbPkgInfo,i) + + """ + oldEtpData['name'] = self.retrievePackageVar(dbPkgInfo,"name") + oldEtpData['version'] = self.retrievePackageVar(dbPkgInfo,"version") + oldEtpData['description'] = self.retrievePackageVar(dbPkgInfo,"description") + oldEtpData['category'] = self.retrievePackageVar(dbPkgInfo,"category") + oldEtpData['chost'] = self.retrievePackageVar(dbPkgInfo,"chost") + oldEtpData['cflags'] = self.retrievePackageVar(dbPkgInfo,"cflags") + oldEtpData['cxxflags'] = self.retrievePackageVar(dbPkgInfo,"cxxflags") + oldEtpData['homepage'] = self.retrievePackageVar(dbPkgInfo,"homepage") + oldEtpData['useflags'] = self.retrievePackageVar(dbPkgInfo,"useflags") + oldEtpData['license'] = self.retrievePackageVar(dbPkgInfo,"license") + oldEtpData['keywords'] = self.retrievePackageVar(dbPkgInfo,"keywords") + oldEtpData['binkeywords'] = self.retrievePackageVar(dbPkgInfo,"binkeywords") + oldEtpData['packagepath'] = self.retrievePackageVar(dbPkgInfo,"packagepath") + oldEtpData['download'] = self.retrievePackageVar(dbPkgInfo,"download") + oldEtpData['digest'] = self.retrievePackageVar(dbPkgInfo,"digest") + oldEtpData['sources'] = self.retrievePackageVar(dbPkgInfo,"sources") + oldEtpData['slot'] = self.retrievePackageVar(dbPkgInfo,"slot") + oldEtpData['content'] = self.retrievePackageVar(dbPkgInfo,"content") + oldEtpData['mirrorlinks'] = self.retrievePackageVar(dbPkgInfo,"mirrorlinks") + oldEtpData['dependencies'] = self.retrievePackageVar(dbPkgInfo,"dependencies") + oldEtpData['rundependencies'] = self.retrievePackageVar(dbPkgInfo,"rundependencies") + oldEtpData['rundependenciesXT'] = self.retrievePackageVar(dbPkgInfo,"rundependenciesXT") + oldEtpData['conflicts'] = self.retrievePackageVar(dbPkgInfo,"conflicts") + oldEtpData['etpapi'] = self.retrievePackageVar(dbPkgInfo,"etpapi") + """ + + for i in etpData: + if etpData[i] != tmpEtpData[i]: + return False + + return True + # You must provide the full atom to this function def retrievePackageInfo(self,pkgkey): pkgkey = removePackageOperators(pkgkey) diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 2aa50a071..974283858 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -26,7 +26,7 @@ import string import random import sys -# Specifications of the content of .etp file +# Specifications of the content of etpData # THIS IS THE KEY PART OF ENTROPY BINARY PACKAGES MANAGEMENT # DO NOT EDIT THIS UNLESS YOU KNOW WHAT YOU'RE DOING !! etpData = { @@ -47,6 +47,7 @@ etpData = { 'digest': "", # md5 hash of the .tbz2 package 'sources': "", # link to the sources 'slot': "", # this is filled if the package is slotted + 'content': "", # content of the package (files) 'mirrorlinks': "", # =mirror://openoffice|link1|link2|link3 'dependencies': "", # dependencies 'rundependencies': "", # runtime dependencies @@ -55,19 +56,48 @@ etpData = { 'etpapi': "", # blockers } -# Entropy database SQL initialization Schema +# Used for convenience... it's temporary, do not count on it and reset before using +tmpEtpData = { + 'name': "", # the Package Name + 'version': "", # the Package version plus our -etpXX revision + 'description': "", # the Package description + 'category': "", # the gentoo category + 'chost': "", # the CHOST used to compile it + 'cflags': "", # CFLAGS used + 'cxxflags': "", # CXXFLAGS used + 'homepage': "", # home page of the package + 'useflags': "", # USE flags used + 'license': "", # License adpoted + 'keywords': "", # supported ARCHs (by the SRC) + 'binkeywords': "", # supported ARCHs (by the BIN) + 'packagepath': "", # path where the .tbz2 file is stored + 'download': "", # link to download the binary package + 'digest': "", # md5 hash of the .tbz2 package + 'sources': "", # link to the sources + 'slot': "", # this is filled if the package is slotted + 'content': "", # content of the package (files) + 'mirrorlinks': "", # =mirror://openoffice|link1|link2|link3 + 'dependencies': "", # dependencies + 'rundependencies': "", # runtime dependencies + 'rundependenciesXT': "", # runtime dependencies + version + 'conflicts': "", # blockers + 'etpapi': "", # blockers +} + +# Entropy database SQL initialization Schema and data structure +# MUST BE KEPT IN SYNC with etpData above etpSQLInit = """ CREATE TABLE etpData ( atom VARCHAR(75) PRIMARY KEY, - name VARCHAR(255), + name VARCHAR(50), version VARCHAR(25), - description VARCHAR(255), + description VARCHAR(100), category VARCHAR(25), chost VARCHAR(100), - cflags VARCHAR(255), - cxxflags VARCHAR(255), - homepage VARCHAR(255), - useflags VARCHAR(350), + cflags VARCHAR(100), + cxxflags VARCHAR(100), + homepage VARCHAR(100), + useflags VARCHAR(150), license VARCHAR(25), keywords VARCHAR(50), binkeywords VARCHAR(50), @@ -76,11 +106,12 @@ CREATE TABLE etpData ( digest VARCHAR(32), sources VARCHAR(500), slot VARCHAR(10), - mirrorlinks VARCHAR(500), - dependencies VARCHAR(300), - rundependencies VARCHAR(450), - rundependenciesXT VARCHAR(450), - conflicts VARCHAR(300), + content VARCHAR(100), + mirrorlinks VARCHAR(200), + dependencies VARCHAR(100), + rundependencies VARCHAR(250), + rundependenciesXT VARCHAR(250), + conflicts VARCHAR(100), etpapi VARCHAR(3), revision INTEGER(3) ) @@ -101,7 +132,7 @@ ETP_TMPFILE = "/.random-"+ETP_RANDOM+".tmp" ETP_REPODIR = "/packages"+"/"+ETP_ARCH_CONST ETP_PORTDIR = "/portage" ETP_DISTFILESDIR = "/distfiles" -ETP_DBDIR = "/database"+"/"+ETP_ARCH_CONST +ETP_DBDIR = "/database/"+ETP_ARCH_CONST ETP_UPLOADDIR = "/upload"+"/"+ETP_ARCH_CONST ETP_STOREDIR = "/store"+"/"+ETP_ARCH_CONST ETP_CONF_DIR = "/etc/entropy" @@ -117,7 +148,6 @@ etpConst = { 'packagesbindir': ETP_DIR+ETP_REPODIR, # etpConst['packagesbindir'] --> repository where the packages will be stored # by the clients: to query if a package has been already downloaded # by the servers or rsync mirrors: to store already uploaded packages to the main rsync server - 'packagesdatabasedir': ETP_DIR+ETP_DBDIR, # FIXME: REMOVE THIS ! 'packagesstoredir': ETP_DIR+ETP_STOREDIR, # etpConst['packagesstoredir'] --> directory where .tbz2 files are stored waiting for being processed by reagent 'packagessuploaddir': ETP_DIR+ETP_UPLOADDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror 'portagetreedir': ETP_DIR+ETP_PORTDIR, # directory where is stored our local portage tree @@ -131,19 +161,20 @@ etpConst = { 'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file 'activatoruploaduris': [],# list of URIs that activator can use to upload files (parsed from activator.conf) 'activatordownloaduris': [],# list of URIs that activator can use to fetch data - 'digestfile': "Manifest", # file that contains md5 hashes - 'extension': ".etp", # entropy files extension 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. 'etpurirelativepath': "database/"+ETP_ARCH_CONST+"/", # Relative remote path for the .etp repository. - 'etpdatabasefile': "__database.tar.bz2", # compressed file that contains the whole Entropy database directory tree - # YOU MUST APPEND %ARCH% at the beginning - 'etpdatabaselockfile': "database.lock", # the remote database lock file - 'etpdatabasedownloadlockfile': "download.lock", # the remote database download lock file + # TO BE REMOVED? CHECK + 'etpdatabaselockfile': "packages.db.lock", # the remote database lock file + 'etpdatabasedownloadlockfile': "packages.db.download.lock", # the remote database download lock file + 'etpdatabasetaintfile': "packages.db.tainted", # when this file exists, the database is not synced anymore with the online one 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit 'distcc-status': False, # used by Enzyme, if True distcc is enabled 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file + 'etpdatabasedir': ETP_DIR+ETP_DBDIR, # FIXME: REMOVE THIS ! 'etpdatabasefile': ETP_DIR+ETP_DBDIR+"/packages.db", # Entropy sqlite database file 'etpapi': ETP_API, # Entropy database API revision + 'headertext': ETP_HEADER_TEXT, # header text that can be outputted to a file + 'currentarch': ETP_ARCH_CONST, # contains the current running architecture } # Create paths diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 4bfa9074a..ba8f3fb65 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -993,7 +993,7 @@ def syncRemoteDatabases(): # check if the local DB exists etpDbLocalPath = etpConst['etpurirelativepath'] - etpDbLocalFile = etpConst['packagesdatabasedir'] + etpDbLocalFile = etpConst['etpdatabasedir'] if etpDbLocalFile.endswith("/"): etpDbLocalFile = etpDbLocalFile[:len(etpDbLocalFile)-1] etpDbLocalFile += etpConst['etpdatabasefile'] @@ -1114,7 +1114,7 @@ def syncRemoteDatabases(): if (generateAndUpload): print_info(green(" * ")+red("Compressing ETP Repository to ")+bold(etpDbLocalFile),back = True) - rc = compressTarBz2(etpDbLocalFile,etpConst['packagesdatabasedir']) + rc = compressTarBz2(etpDbLocalFile,etpConst['etpdatabasedir']) if (rc): print_error(red(" * Cannot compress "+etpDbLocalFile)) print_error(red(" *** Cannot continue")) @@ -1199,7 +1199,7 @@ def downloadDatabase(uri,dbfile): print_warning(yellow(" * ")+red("Cannot properly download to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) # removing old tree print_info(green(" * ")+red("Uncompressing downloaded database ..."),back = True) - os.system("rm -rf "+etpConst['packagesdatabasedir']+"/*") + os.system("rm -rf "+etpConst['etpdatabasedir']+"/*") rc = uncompressTarBz2(dbfile,"/") if (rc): print_error(red(" * Cannot uncompress "+dbfile)) diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 7b75c55e0..7e2f85b68 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -29,39 +29,6 @@ import commands import re import sys -# Create the manifest file inside the directory provided by -# 'path' and hash all the *.etpConst['extension'] files -def createDigest(path): - if path.endswith(etpConst['extension']): - # remove file name and keep the rest of the path - _path = path.split("/")[:len(path.split("/"))-1] - path = "" - for i in _path: - if (i): - path += "/"+i - if (not os.path.isdir(path)): - print_error(path+" does not exist") - sys.exit(102) - digestContent = os.listdir(path) - # only .etp files - _digestContent = digestContent - digestContent = [] - for i in _digestContent: - if i.endswith(etpConst['extension']): - digestContent.append(i) - if (not digestContent[0].endswith(etpConst['extension'])): - print_error(path+" does not contain "+etpConst['extension']+" files") - sys.exit(103) - print_info(green(" * ")+red("Digesting files in ")+path) - digestOut = [] - for i in digestContent: - digestOut.append("MD5 "+md5sum(path+"/"+i)+" "+i+"\n") - f = open(path+"/"+etpConst['digestfile'],"w") - f.writelines(digestOut) - f.flush() - f.close() - - def generator(packages, enzymeRequestBump = False): _packages = [] @@ -87,37 +54,20 @@ def generator(packages, enzymeRequestBump = False): print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait...")) etpData = extractPkgData(package) - # now try to import etpData inside the database + # now import etpData inside the database dbconn = databaseTools.etpDatabase() - #dbconn.searchPackages(etpData['name']) - #dbconn.retrievePackageInfo(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - #dbconn.removePackage(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - dbconn.searchPackages(etpData['name']) - - #dbconn.retrievePackageInfo(etpData['category']+"/"+etpData['name']+"-"+etpData['version']) - #dbconn.addPackage(etpData) + updated, revision = dbconn.handlePackage(etpData,enzymeRequestBump) dbconn.closeDB() + + if (updated) and (revision != 0): + print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision))) + elif (updated) and (revision == 0): + print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created.")) + else: + print_info(green(" * ")+red("Package ")+bold(packagename)+red(" does not need to be updated. Current revision: ")+bold(str(revision))) - # look where I can store the file and return its path - etpOutput, etpOutfilePath = allocateFile(etpData,enzymeRequestBump) - rc = False - - if etpOutfilePath is not None: - print_info(green(" * ")+red("Writing Entropy Specifications file: ")+etpOutfilePath) - f = open(etpOutfilePath,"w") - f.writelines(etpOutput) - f.flush() - f.close() - # digesting directory - createDigest(etpOutfilePath) - rc = True - else: - print_info(green(" * ")+red("Not generating a new Entropy Specifications file, not needed for ")+bold(packagename)) - # clean garbage - os.system("rm -rf "+etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']) - -# Enzyme tool called, we need to parse the Store directory and call generator() +# This tool is used by Entropy after enzyme, it simply parses the content of etpConst['packagesstoredir'] def enzyme(options): enzymeRequestBump = False @@ -231,6 +181,23 @@ def extractPkgData(package): except IOError: etpData['slot'] = "" + print_info(yellow(" * ")+red("Getting package content..."),back = True) + # dbCONTENTS + try: + f = open(tbz2TmpDir+dbCONTENTS,"r") + content = f.readlines() + f.close() + outcontent = [] + for line in content: + line = line.strip().split() + if line[0] == "obj": + outcontent.append(line[1]) + import string + etpData['content'] = string.join(outcontent," ") + + except IOError: + etpData['content'] = "" + print_info(yellow(" * ")+red("Getting package download URL..."),back = True) # Fill download relative URI etpData['download'] = etpConst['binaryurirelativepath']+etpData['name']+"-"+etpData['version']+".tbz2" @@ -398,75 +365,3 @@ def extractPkgData(package): print_info(yellow(" * ")+red("Done"),back = True) return etpData - -# This function generates the right path for putting the .etp file -# and take count of already available ones bumping version only if needed -def allocateFile(etpData, enzymeRequestBump = False): - - # this will be the first thing to return - etpOutput = [] - # append header - etpOutput.append(ETP_HEADER_TEXT) - - # order keys - keys = [] - for i in etpData: - keys.append(i) - - sortedKeys = alphaSorter(keys) - - for i in sortedKeys: - if (etpData[i]): - etpOutput.append(i+": "+etpData[i]+"\n") - - # locate directory structure - etpOutfileDir = etpConst['packagesdatabasedir']+"/"+etpData['category']+"/"+etpData['name'] - #etpOutfileDir = translateArch(etpOutfileDir,etpData['chost']) - etpOutfileName = etpData['name']+"-"+etpData['version']+"-etp"+ETP_REVISION_CONST+etpConst['extension'] - etpOutfilePath = etpOutfileDir+"/"+etpOutfileName - - # we've the directory, then create it - if (not os.path.isdir(etpOutfileDir)): - try: - os.makedirs(etpOutfileDir) - except OSError: - pass - # it's a brand new dir - etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] - else: # directory already exists, check for already available files - alreadyAvailableFiles = [] - for i in range(MAX_ETP_REVISION_COUNT+1): - testfile = re.subn(ETP_REVISION_CONST,str(i), etpOutfilePath)[0] - if (os.path.isfile(testfile)): - alreadyAvailableFiles.append(testfile) - if (alreadyAvailableFiles == []): - etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] - else: - # grab the last one - possibleOldFile = alreadyAvailableFiles[len(alreadyAvailableFiles)-1] - # now compares both to see if they're equal or not - try: - import md5 - import string - a = open(possibleOldFile,"r") - cntA = a.readlines() - cntB = etpOutput - cntA = string.join(cntA) - cntB = string.join(cntB) - a.close() - md5A = md5.new() - md5B = md5.new() - md5A.update(cntA) - md5B.update(cntB) - - if (md5A.digest() == md5B.digest()) and (not enzymeRequestBump): - etpOutfilePath = None - else: - # add 1 to: packagename-1.2.3-r1-etpX.etp - newFileCounter = int(possibleOldFile.split("-")[len(possibleOldFile.split("-"))-1].split(etpConst['extension'])[0].split(etpConst['extension'][1:])[1]) - newFileCounter += 1 - etpOutfilePath = re.subn(ETP_REVISION_CONST,str(newFileCounter), etpOutfilePath)[0] - except OSError: - etpOutfilePath = possibleOldFile - - return etpOutput, etpOutfilePath