diff --git a/handlers/enzyme b/handlers/enzyme index 2b7e8d2b8..ab340974c 100644 --- a/handlers/enzyme +++ b/handlers/enzyme @@ -1,19 +1,24 @@ #!/usr/bin/python -# Copyright Fabio Erculiani - Sabayon Linux 2007 +''' + # DESCRIPTION: + # Portage tree manager and package builder following specified schemas -# DESCRIPTION: -# prerequisites: -# - a sabayon chroot -# - maintenance time (you should see if there's something that needs to be done manually) -# features: -# - extended distcc support -# - daily reports -# this application manages the portage tree and builds .tbz2 files: -# EXAMPLE (if you want to build a new version of openoffice with all its dependencies): -# # enzyme openoffice -# ---> openoffice package will be build and put into the entropy store directory -# there will be some options, like: -# --merge-bin: will install the packages on the system (maybe by default) + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' import os import sys @@ -21,8 +26,59 @@ import string sys.path.append('../libraries') import entropyTools +import enzymeTools from entropyConstants import * # CONSTANTS APPNAME = "enzyme" APPVERSION = "1.0" +def print_help(): + print "* info * : Sabayon Linux "+APPNAME+" (C - 2007)" + print + print "* usage * : "+APPNAME+" " + print "* opts * : --help\t\tthis output" + print "* opts * : --version\t\tprint version" + print + print "* info * : tools available: " + print "* info * : \tworld\t to build all the possible new packages" + print "* info * : \tsync\t to just sync portage tree" + print "* info * : \tdata-cleanup\t to clean all the old packages or stale files" + print "* info * : \tdata-reset\t to remove all and start over" + print + +options = sys.argv[1:] + +# print version +if (string.join(options).find("--version") != -1) or (string.join(options).find(" -V") != -1): + entropyTools.print_generic(APPNAME+": "+APPVERSION) + sys.exit(0) + +# print help +if len(options) < 1 or string.join(options).find("--help") != -1 or string.join(options).find(" -h") != -1: + print_help() + if len(options) < 1: + entropyTools.print_error("not enough parameters") + sys.exit(1) + +if (not entropyTools.isRoot()): + entropyTools.print_error("you must be root in order to run "+APPNAME) + sys.exit(1) + +# world tool +if (options[0] == "world"): + #enzymeTools.world() + print "world management not yet integrated" + sys.exit(0) +# sync tool +elif (options[0] == "sync"): + enzymeTools.sync() + sys.exit(0) + +# add other options here +# ... +# ... +# tbz2 tool + +else: + if (entropyTools.checkAtom()): + print "atom management not yet integrated" \ No newline at end of file diff --git a/handlers/reagent b/handlers/reagent index 06b70fd85..e7ec5b5bb 100644 --- a/handlers/reagent +++ b/handlers/reagent @@ -1,9 +1,25 @@ #!/usr/bin/python -# Copyright Fabio Erculiani - Sabayon Linux 2007 +''' + # DESCRIPTION: + # this application gets a .tbz2 file as input and creates the dependency binary metafile + # that another application could handle using portage and binpackages extensions directly -# DESCRIPTION: -# this application gets a .tbz2 file as input and creates the dependency binary metafile -# that another application could handle using portage and binpackages extensions directly + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' import os import sys @@ -11,6 +27,7 @@ import string sys.path.append('../libraries') import entropyTools +import reagentTools from entropyConstants import * # CONSTANTS @@ -41,9 +58,12 @@ if len(options) < 1 or string.join(options).find("--help") != -1 or string.join( entropyTools.print_error("not enough parameters") sys.exit(1) +if (not entropyTools.isRoot()): + print_error("you must be root in order to run "+APPNAME) + # digest tool (creates a digest on the specified directory) if (options[0].find("digest") != -1): - entropyTools.createDigest(options[1]) + reagentTools.createDigest(options[1]) sys.exit(0) # add other options here # ... @@ -67,10 +87,10 @@ if (not validFile): entropyTools.print_info("extracting data and calculating info for: "+tbz2File.split("/")[len(tbz2File.split("/"))-1]+" please wait...") # fill all the info -etpData = entropyTools.extractPkgData(tbz2File) +etpData = reagentTools.extractPkgData(tbz2File) # look where I can store the file and return its path -etpOutput, etpOutfilePath = entropyTools.allocateFile(etpData) +etpOutput, etpOutfilePath = reagentTools.allocateFile(etpData) if etpOutfilePath is not None: entropyTools.print_info("writing: "+etpOutfilePath) @@ -79,7 +99,7 @@ if etpOutfilePath is not None: f.flush() f.close() # digesting directory - entropyTools.createDigest(etpOutfilePath) + reagentTools.createDigest(etpOutfilePath) else: entropyTools.print_info("not generating a new .etp file, it's not needed") diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 36e8beb07..17903b562 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -1,8 +1,24 @@ #!/usr/bin/python -# Copyright Fabio Erculiani - Sabayon Linux 2007 +''' + # DESCRIPTION: + # Variables container -# DESCRIPTION: -# Variables container + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' # Specifications of the content of .etp file # THIS IS THE KEY PART OF ENTROPY BINARY PACKAGES MANAGEMENT @@ -46,8 +62,8 @@ ETP_REVISION_CONST = "%ETPREV%" ETP_DIR = "/var/lib/entropy" ETP_TMPDIR = "/tmp" ETP_REPODIR = "/repository"+"/"+ETP_ARCH_CONST +ETP_PORTDIR = "/portage" ETP_DBDIR = "/database"+"/"+ETP_ARCH_CONST -ETP_UPDIR = "/upload"+"/"+ETP_ARCH_CONST ETP_STOREDIR = "/store"+"/"+ETP_ARCH_CONST ETP_CONF_DIR = "/etc/entropy" ETP_HEADER_TEXT = "# Entropy specifications file (released under the GPLv2)\n" @@ -58,7 +74,8 @@ etpConst = { 'packagesbindir': ETP_DIR+ETP_REPODIR, # etpConst['packagesbindir'] --> repository where the packages will be stored 'packagesdatabasedir': ETP_DIR+ETP_DBDIR, # etpConst['packagesdatabasedir'] --> repository where .etp files will be stored 'packagesstoredir': ETP_DIR+ETP_DBDIR, # etpConst['packagesstoredir'] --> directory where .tbz2 files are stored waiting for being processed by entropy-specifications-generator - 'packagessuploaddir': ETP_DIR+ETP_UPDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror + 'packagessuploaddir': ETP_DIR+ETP_STOREDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror + 'portagetreedir': ETP_DIR+ETP_PORTDIR, # directory where is stored our local portage tree 'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration 'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file 'enzymeconf': ETP_CONF_DIR+"/enzyme.conf", # enzym.conf file @@ -143,4 +160,11 @@ dbRDEPEND = "RDEPEND" dbPDEPEND = "PDEPEND" dbNEEDED = "NEEDED" dbOR = "|or|" -dbKEYWORDS = "KEYWORDS" \ No newline at end of file +dbKEYWORDS = "KEYWORDS" + +# Portage variables reference +# vdbVARIABLE --> $VARIABLE +vdbPORTDIR = "PORTDIR" + +# Portage commands +cdbEMERGE = "emerge" \ No newline at end of file diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index b814e262e..7cfa34a39 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -1,17 +1,45 @@ #!/usr/bin/python -# Copyright Fabio Erculiani - Sabayon Linux 2007 +''' + # DESCRIPTION: + # generic tools for all the handlers applications -# DESCRIPTION: -# generic tools for all the handlers applications + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' import portage import portage_const -from portage_dep import isvalidatom, isjustname, dep_getkey +from portage_dep import isvalidatom, isjustname + +# colours support +import output +from output import bold, colorize, green, red, yellow from entropyConstants import * -import commands import re +def isRoot(): + import getpass + if (getpass.getuser() == "root"): + return True + return False + +def getPortageEnv(var): + return portage.config(clone=portage.settings).environ()[var] + # resolve atoms automagically (best, not current!) # sys-libs/application --> sys-libs/application-1.2.3-r1 def getBestAtom(atom): @@ -44,6 +72,11 @@ def getInstalledAtom(atom): else: return atom +def checkAtom(atom): + if (isvalidatom(atom) == 1): + return True + return False + def removeSpaceAtTheEnd(string): if string.endswith(" "): return string[:len(string)-1] @@ -60,35 +93,10 @@ def md5sum(filepath): block = readfile.read(1024) return m.hexdigest() -def createDigest(path): - if path.endswith(etpConst['extension']): - # remove file name and keep the rest of the path - _path = path.split("/")[:len(path.split("/"))-1] - path = "" - for i in _path: - if (i): - path += "/"+i - if (not os.path.isdir(path)): - print_error(path+" does not exist") - sys.exit(102) - digestContent = os.listdir(path) - # only .etp files - _digestContent = digestContent - digestContent = [] - for i in _digestContent: - if i.endswith(etpConst['extension']): - digestContent.append(i) - if (not digestContent[0].endswith(etpConst['extension'])): - print_error(path+" does not contain "+etpConst['extension']+" files") - sys.exit(103) - print_info("digesting files in "+path) - digestOut = [] - for i in digestContent: - digestOut.append("MD5 "+md5sum(path+"/"+i)+" "+i+"\n") - f = open(path+"/"+etpConst['digestfile'],"w") - f.writelines(digestOut) - f.flush() - f.close() +# Tool to run commands +def spawnCommand(command): + rc = os.system(command) + return rc def print_error(msg): print "* erro * : "+msg @@ -100,390 +108,4 @@ def print_warning(msg): print "* warn * : "+msg def print_generic(msg): # here we'll wrap any nice formatting - print msg - -# This function extracts all the info from a .tbz2 file and returns them -def extractPkgData(package): - - tbz2File = package - - package = package.split(".tbz2") - package = package[0].split("-") - pkgname = "" - pkglen = len(package) - if package[pkglen-1].startswith("r"): - pkgver = package[pkglen-2]+"-"+package[pkglen-1] - pkglen -= 2 - else: - pkgver = package[len(package)-1] - pkglen -= 1 - for i in range(pkglen): - if i == pkglen-1: - pkgname += package[i] - else: - pkgname += package[i]+"-" - pkgname = pkgname.split("/")[len(pkgname.split("/"))-1] - - # Fill Package name and version - etpData['name'] = pkgname - etpData['version'] = pkgver - - # .tbz2 md5 - etpData['digest'] = md5sum(tbz2File) - # local path to the file - etpData['packagepath'] = tbz2File - - import xpak - tbz2 = xpak.tbz2(tbz2File) - tbz2TmpDir = etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']+"/" - tbz2.decompose(tbz2TmpDir) - - # Fill chost - f = open(tbz2TmpDir+dbCHOST,"r") - etpData['chost'] = f.readline().strip() - f.close() - - # Fill description - try: - f = open(tbz2TmpDir+dbDESCRIPTION,"r") - etpData['description'] = f.readline().strip() - f.close() - except IOError: - etpData['description'] = "" - - # Fill homepage - try: - f = open(tbz2TmpDir+dbHOMEPAGE,"r") - etpData['homepage'] = f.readline().strip() - f.close() - except IOError: - etpData['homepage'] = "" - - # Fill url - for i in etpSources['packagesuri']: - etpData['download'] += translateArch(i+etpData['name']+"-"+etpData['version']+".tbz2",etpData['chost'])+" " - if (not etpData['download']): - print_error("no 'packages|' specified in "+etpConst['repositoriesconf']) - sys.exit(101) - etpData['download'] = removeSpaceAtTheEnd(etpData['download']) - - # Fill category - f = open(tbz2TmpDir+dbCATEGORY,"r") - etpData['category'] = f.readline().strip() - f.close() - - # Fill CFLAGS - try: - f = open(tbz2TmpDir+dbCFLAGS,"r") - etpData['cflags'] = f.readline().strip() - f.close() - except IOError: - etpData['cflags'] = "" - - # Fill CXXFLAGS - try: - f = open(tbz2TmpDir+dbCXXFLAGS,"r") - etpData['cxxflags'] = f.readline().strip() - f.close() - except IOError: - etpData['cxxflags'] = "" - - # Fill license - try: - f = open(tbz2TmpDir+dbLICENSE,"r") - etpData['license'] = f.readline().strip() - f.close() - except IOError: - etpData['license'] = "" - - # Fill sources - try: - f = open(tbz2TmpDir+dbSRC_URI,"r") - tmpSources = f.readline().strip().split() - f.close() - for atom in tmpSources: - if atom.endswith("?"): - etpData['sources'] += "="+atom[:len(atom)-1]+"|" - elif (not atom.startswith("(")) and (not atom.startswith(")")): - etpData['sources'] += atom+" " - except IOError: - etpData['sources'] = "" - - # manage etpData['sources'] to create etpData['mirrorlinks'] - # =mirror://openoffice|link1|link2|link3 - tmpMirrorList = etpData['sources'].split() - for i in tmpMirrorList: - if i.startswith("mirror://"): - # parse what mirror I need - x = i.split("/")[2] - mirrorlist = portage.thirdpartymirrors[x] - mirrorURI = "mirror://"+x - out = "="+mirrorURI+"|" - for mirror in mirrorlist: - out += mirror+"|" - if out.endswith("|"): - out = out[:len(out)-1] - etpData['mirrorlinks'] += out+" " - etpData['mirrorlinks'] = removeSpaceAtTheEnd(etpData['mirrorlinks']) - - # Fill USE - f = open(tbz2TmpDir+dbUSE,"r") - tmpUSE = f.readline().strip() - f.close() - try: - f = open(tbz2TmpDir+dbIUSE,"r") - tmpIUSE = f.readline().strip().split() - f.close() - except IOError: - tmpIUSE = "" - - for i in tmpIUSE: - if tmpUSE.find(i) != -1: - etpData['useflags'] += i+" " - else: - etpData['useflags'] += "-"+i+" " - - # cleanup - tmpUSE = etpData['useflags'].split() - tmpUSE = list(set(tmpUSE)) - etpData['useflags'] = '' - for i in tmpUSE: - etpData['useflags'] += i+" " - etpData['useflags'] = removeSpaceAtTheEnd(etpData['useflags']) - - # fill KEYWORDS - try: - f = open(tbz2TmpDir+dbKEYWORDS,"r") - etpData['keywords'] = f.readline().strip() - f.close() - except IOError: - etpData['keywords'] = "" - - # fill ARCHs - pkgArchs = etpData['keywords'] - for i in ETP_ARCHS: - if pkgArchs.find(i) != -1 and (pkgArchs.find("-"+i) == -1): # in case we find something like -amd64... - etpData['binkeywords'] += i+" " - - etpData['binkeywords'] = removeSpaceAtTheEnd(etpData['binkeywords']) - - # Fill dependencies - # to fill dependencies we use *DEPEND files - f = open(tbz2TmpDir+dbDEPEND,"r") - roughDependencies = f.readline().strip() - f.close() - f = open(tbz2TmpDir+dbRDEPEND,"r") - roughDependencies += " "+f.readline().strip() - f.close() - f = open(tbz2TmpDir+dbPDEPEND,"r") - roughDependencies += " "+f.readline().strip() - f.close() - roughDependencies = roughDependencies.split() - - # variables filled - # etpData['dependencies'] - - useMatch = False - openParenthesis = 0 - openOr = False - useFlagQuestion = False - for atom in roughDependencies: - - if atom.endswith("?"): - # we need to see if that useflag is enabled - useFlag = atom.split("?")[0] - useFlagQuestion = True - for i in etpData['useflags'].split(): - if i.startswith("!"): - if (i != useFlag): - useMatch = True - break - else: - if (i == useFlag): - useMatch = True - break - - if atom.startswith("("): - openParenthesis += 1 - - if atom.startswith(")"): - if (openOr): - # remove last "_or_" from etpData['dependencies'] - openOr = False - if etpData['dependencies'].endswith(dbOR): - etpData['dependencies'] = etpData['dependencies'][:len(etpData['dependencies'])-len(dbOR)] - etpData['dependencies'] += " " - openParenthesis -= 1 - if (openParenthesis == 0): - useFlagQuestion = False - useMatch = False - - if atom.startswith("||"): - openOr = True - - if atom.find("/") != -1 and (not atom.startswith("!")) and (not atom.endswith("?")): - # it's a package name /-??? - if ((useFlagQuestion) and (useMatch)) or ((not useFlagQuestion) and (not useMatch)): - # check if there's an OR - etpData['dependencies'] += atom - if (openOr): - etpData['dependencies'] += dbOR - else: - etpData['dependencies'] += " " - - if atom.startswith("!") and (not atom.endswith("?")): - if ((useFlagQuestion) and (useMatch)) or ((not useFlagQuestion) and (not useMatch)): - etpData['conflicts'] += atom - if (openOr): - etpData['conflicts'] += dbOR - else: - etpData['conflicts'] += " " - - - # format properly - tmpConflicts = list(set(etpData['conflicts'].split())) - etpData['conflicts'] = '' - for i in tmpConflicts: - i = i[1:] # remove "!" - etpData['conflicts'] += i+" " - etpData['conflicts'] = removeSpaceAtTheEnd(etpData['conflicts']) - - tmpDeps = list(set(etpData['dependencies'].split())) - etpData['dependencies'] = '' - for i in tmpDeps: - etpData['dependencies'] += i+" " - etpData['dependencies'] = removeSpaceAtTheEnd(etpData['dependencies']) - - # etpData['rdependencies'] - # Now we need to add environmental dependencies - # Notes (take the example of mplayer that needed a newer libcaca release): - # - we can use (from /var/db) "NEEDED" file to catch all the needed libraries to run the binary package - # - we can use (from /var/db) "CONTENTS" to rapidly search the NEEDED files in the file above - # return all the collected info - - # start collecting needed libraries - try: - f = open(tbz2TmpDir+"/"+dbNEEDED,"r") - includedBins = f.readlines() - f.close() - except IOError: - includedBins = "" - - neededLibraries = [] - # filter the first word - for line in includedBins: - line = line.strip().split() - line = line[0] - depLibs = commands.getoutput("ldd "+line).split("\n") - for i in depLibs: - i = i.strip() - if i.find("=>") != -1: - i = i.split("=>")[1] - # format properly - if i.startswith(" "): - i = i[1:] - if i.startswith("//"): - i = i[1:] - i = i.split()[0] - neededLibraries.append(i) - neededLibraries = list(set(neededLibraries)) - - runtimeNeededPackages = [] - runtimeNeededPackagesXT = [] - for i in neededLibraries: - if i.startswith("/"): # filter garbage - pkgs = commands.getoutput(pFindLibraryXT+i).split("\n") - if (pkgs[0] != ""): - for y in pkgs: - runtimeNeededPackagesXT.append(y) - y = dep_getkey(y) - runtimeNeededPackages.append(y) - - runtimeNeededPackages = list(set(runtimeNeededPackages)) - runtimeNeededPackagesXT = list(set(runtimeNeededPackagesXT)) - - # now keep only the ones not available in etpData['dependencies'] - for i in runtimeNeededPackages: - if etpData['dependencies'].find(i) == -1: - # filter itself - if (i != etpData['category']+"/"+etpData['name']): - etpData['rundependencies'] += i+" " - - for i in runtimeNeededPackagesXT: - x = dep_getkey(i) - if etpData['dependencies'].find(x) == -1: - # filter itself - if (x != etpData['category']+"/"+etpData['name']): - etpData['rundependenciesXT'] += i+" " - - # format properly - etpData['rundependencies'] = removeSpaceAtTheEnd(etpData['rundependencies']) - etpData['rundependenciesXT'] = removeSpaceAtTheEnd(etpData['rundependenciesXT']) - - # write API info - etpData['etpapi'] = ETP_API - - return etpData - -# This function generates the right path for putting the .etp file -# and take count of already available ones bumping version only if needed -def allocateFile(etpData): - - # this will be the first thing to return - etpOutput = [] - # append header - etpOutput.append(ETP_HEADER_TEXT) - for i in etpData: - if (etpData[i]): - etpOutput.append(i+": "+etpData[i]+"\n") - - # locate directory structure - etpOutfileDir = etpConst['packagesdatabasedir']+"/"+etpData['category']+"/"+etpData['name'] - etpOutfileDir = translateArch(etpOutfileDir,etpData['chost']) - etpOutfileName = etpData['name']+"-"+etpData['version']+"-etp"+ETP_REVISION_CONST+etpConst['extension'] - etpOutfilePath = etpOutfileDir+"/"+etpOutfileName - - # we've the directory, then create it - if (not os.path.isdir(etpOutfileDir)): - try: - os.makedirs(etpOutfileDir) - except OSError: - pass - # it's a brand new dir - etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] - else: # directory already exists, check for already available files - alreadyAvailableFiles = [] - for i in range(MAX_ETP_REVISION_COUNT+1): - testfile = re.subn(ETP_REVISION_CONST,str(i), etpOutfilePath)[0] - if (os.path.isfile(testfile)): - alreadyAvailableFiles.append(testfile) - if (alreadyAvailableFiles == []): - etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] - else: - # grab the last one - possibleOldFile = alreadyAvailableFiles[len(alreadyAvailableFiles)-1] - # now compares both to see if they're equal or not - try: - import md5 - import string - a = open(possibleOldFile,"r") - cntA = a.readlines() - cntB = etpOutput - cntA = string.join(cntA) - cntB = string.join(cntB) - a.close() - md5A = md5.new() - md5B = md5.new() - md5A.update(cntA) - md5B.update(cntB) - if md5A.digest() == md5B.digest(): - etpOutfilePath = None - else: - # add 1 to: packagename-1.2.3-r1-etpX.etp - newFileCounter = int(possibleOldFile.split("-")[len(possibleOldFile.split("-"))-1].split(etpConst['extension'])[0].split(etpConst['extension'][1:])[1]) - newFileCounter += 1 - etpOutfilePath = re.subn(ETP_REVISION_CONST,str(newFileCounter), etpOutfilePath)[0] - except OSError: - etpOutfilePath = possibleOldFile - - return etpOutput, etpOutfilePath \ No newline at end of file + print msg \ No newline at end of file diff --git a/libraries/enzymeTools.py b/libraries/enzymeTools.py new file mode 100644 index 000000000..cc41170e7 --- /dev/null +++ b/libraries/enzymeTools.py @@ -0,0 +1,58 @@ +#!/usr/bin/python +''' + # DESCRIPTION: + # generic tools for enzyme application + + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' + + +import portage +import portage_const + +from entropyConstants import * +from entropyTools import * + +# Stolen from Porthole 0.5.0 - thanks for your help :-) + +def getSyncTime(): + """gets and returns the timestamp info saved during + the last portage tree sync""" + lastSync = None + try: + f = open(etpConst['portagetreedir'] + "/metadata/timestamp") + data = f.read() + f.close() + if data: + try: + lastSync = (str(data).decode('utf_8').encode("utf_8",'replace')) + except: + try: + lastSync = (str(data).decode('iso-8859-1').encode('utf_8', 'replace')) + except: + print_warning("getSyncTime(): unknown encoding") + else: + print_warning("getSyncTime(): nothing to read") + except: + print_warning("getSyncTime(): empty Portage tree (first run?) or no timestamp to read") + + +# fetch the latest updates from Gentoo rsync mirrors +def sync(): + print_info(green("syncing the Portage tree at: "+etpConst['portagetreedir'])) + entropyTools.spawnCommand(vdbPORTDIR+"="+etpConst['portagetreedir']+" "+cdbEMERGE+" --sync") + \ No newline at end of file diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py new file mode 100644 index 000000000..3269988ef --- /dev/null +++ b/libraries/reagentTools.py @@ -0,0 +1,448 @@ +#!/usr/bin/python +''' + # DESCRIPTION: + # generic tools for reagent application + + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' + +import portage +import portage_const +from portage_dep import dep_getkey + +from entropyConstants import * +from entropyTools import * +import commands +import re + +# Create the manifest file inside the directory provided by +# 'path' and hash all the *.etpConst['extension'] files +def createDigest(path): + if path.endswith(etpConst['extension']): + # remove file name and keep the rest of the path + _path = path.split("/")[:len(path.split("/"))-1] + path = "" + for i in _path: + if (i): + path += "/"+i + if (not os.path.isdir(path)): + print_error(path+" does not exist") + sys.exit(102) + digestContent = os.listdir(path) + # only .etp files + _digestContent = digestContent + digestContent = [] + for i in _digestContent: + if i.endswith(etpConst['extension']): + digestContent.append(i) + if (not digestContent[0].endswith(etpConst['extension'])): + print_error(path+" does not contain "+etpConst['extension']+" files") + sys.exit(103) + print_info("digesting files in "+path) + digestOut = [] + for i in digestContent: + digestOut.append("MD5 "+md5sum(path+"/"+i)+" "+i+"\n") + f = open(path+"/"+etpConst['digestfile'],"w") + f.writelines(digestOut) + f.flush() + f.close() + +# This function extracts all the info from a .tbz2 file and returns them +def extractPkgData(package): + + tbz2File = package + + package = package.split(".tbz2") + package = package[0].split("-") + pkgname = "" + pkglen = len(package) + if package[pkglen-1].startswith("r"): + pkgver = package[pkglen-2]+"-"+package[pkglen-1] + pkglen -= 2 + else: + pkgver = package[len(package)-1] + pkglen -= 1 + for i in range(pkglen): + if i == pkglen-1: + pkgname += package[i] + else: + pkgname += package[i]+"-" + pkgname = pkgname.split("/")[len(pkgname.split("/"))-1] + + # Fill Package name and version + etpData['name'] = pkgname + etpData['version'] = pkgver + + # .tbz2 md5 + etpData['digest'] = md5sum(tbz2File) + # local path to the file + etpData['packagepath'] = tbz2File + + import xpak + tbz2 = xpak.tbz2(tbz2File) + tbz2TmpDir = etpConst['packagestmpdir']+"/"+etpData['name']+"-"+etpData['version']+"/" + tbz2.decompose(tbz2TmpDir) + + # Fill chost + f = open(tbz2TmpDir+dbCHOST,"r") + etpData['chost'] = f.readline().strip() + f.close() + + # Fill description + try: + f = open(tbz2TmpDir+dbDESCRIPTION,"r") + etpData['description'] = f.readline().strip() + f.close() + except IOError: + etpData['description'] = "" + + # Fill homepage + try: + f = open(tbz2TmpDir+dbHOMEPAGE,"r") + etpData['homepage'] = f.readline().strip() + f.close() + except IOError: + etpData['homepage'] = "" + + # Fill url + for i in etpSources['packagesuri']: + etpData['download'] += translateArch(i+etpData['name']+"-"+etpData['version']+".tbz2",etpData['chost'])+" " + if (not etpData['download']): + print_error("no 'packages|' specified in "+etpConst['repositoriesconf']) + sys.exit(101) + etpData['download'] = removeSpaceAtTheEnd(etpData['download']) + + # Fill category + f = open(tbz2TmpDir+dbCATEGORY,"r") + etpData['category'] = f.readline().strip() + f.close() + + # Fill CFLAGS + try: + f = open(tbz2TmpDir+dbCFLAGS,"r") + etpData['cflags'] = f.readline().strip() + f.close() + except IOError: + etpData['cflags'] = "" + + # Fill CXXFLAGS + try: + f = open(tbz2TmpDir+dbCXXFLAGS,"r") + etpData['cxxflags'] = f.readline().strip() + f.close() + except IOError: + etpData['cxxflags'] = "" + + # Fill license + try: + f = open(tbz2TmpDir+dbLICENSE,"r") + etpData['license'] = f.readline().strip() + f.close() + except IOError: + etpData['license'] = "" + + # Fill sources + try: + f = open(tbz2TmpDir+dbSRC_URI,"r") + tmpSources = f.readline().strip().split() + f.close() + for atom in tmpSources: + if atom.endswith("?"): + etpData['sources'] += "="+atom[:len(atom)-1]+"|" + elif (not atom.startswith("(")) and (not atom.startswith(")")): + etpData['sources'] += atom+" " + except IOError: + etpData['sources'] = "" + + # manage etpData['sources'] to create etpData['mirrorlinks'] + # =mirror://openoffice|link1|link2|link3 + tmpMirrorList = etpData['sources'].split() + for i in tmpMirrorList: + if i.startswith("mirror://"): + # parse what mirror I need + x = i.split("/")[2] + mirrorlist = portage.thirdpartymirrors[x] + mirrorURI = "mirror://"+x + out = "="+mirrorURI+"|" + for mirror in mirrorlist: + out += mirror+"|" + if out.endswith("|"): + out = out[:len(out)-1] + etpData['mirrorlinks'] += out+" " + etpData['mirrorlinks'] = removeSpaceAtTheEnd(etpData['mirrorlinks']) + + # Fill USE + f = open(tbz2TmpDir+dbUSE,"r") + tmpUSE = f.readline().strip() + f.close() + try: + f = open(tbz2TmpDir+dbIUSE,"r") + tmpIUSE = f.readline().strip().split() + f.close() + except IOError: + tmpIUSE = "" + + for i in tmpIUSE: + if tmpUSE.find(i) != -1: + etpData['useflags'] += i+" " + else: + etpData['useflags'] += "-"+i+" " + + # cleanup + tmpUSE = etpData['useflags'].split() + tmpUSE = list(set(tmpUSE)) + etpData['useflags'] = '' + for i in tmpUSE: + etpData['useflags'] += i+" " + etpData['useflags'] = removeSpaceAtTheEnd(etpData['useflags']) + + # fill KEYWORDS + try: + f = open(tbz2TmpDir+dbKEYWORDS,"r") + etpData['keywords'] = f.readline().strip() + f.close() + except IOError: + etpData['keywords'] = "" + + # fill ARCHs + pkgArchs = etpData['keywords'] + for i in ETP_ARCHS: + if pkgArchs.find(i) != -1 and (pkgArchs.find("-"+i) == -1): # in case we find something like -amd64... + etpData['binkeywords'] += i+" " + + etpData['binkeywords'] = removeSpaceAtTheEnd(etpData['binkeywords']) + + # Fill dependencies + # to fill dependencies we use *DEPEND files + f = open(tbz2TmpDir+dbDEPEND,"r") + roughDependencies = f.readline().strip() + f.close() + f = open(tbz2TmpDir+dbRDEPEND,"r") + roughDependencies += " "+f.readline().strip() + f.close() + f = open(tbz2TmpDir+dbPDEPEND,"r") + roughDependencies += " "+f.readline().strip() + f.close() + roughDependencies = roughDependencies.split() + + # variables filled + # etpData['dependencies'] + + useMatch = False + openParenthesis = 0 + openOr = False + useFlagQuestion = False + for atom in roughDependencies: + + if atom.endswith("?"): + # we need to see if that useflag is enabled + useFlag = atom.split("?")[0] + useFlagQuestion = True + for i in etpData['useflags'].split(): + if i.startswith("!"): + if (i != useFlag): + useMatch = True + break + else: + if (i == useFlag): + useMatch = True + break + + if atom.startswith("("): + openParenthesis += 1 + + if atom.startswith(")"): + if (openOr): + # remove last "_or_" from etpData['dependencies'] + openOr = False + if etpData['dependencies'].endswith(dbOR): + etpData['dependencies'] = etpData['dependencies'][:len(etpData['dependencies'])-len(dbOR)] + etpData['dependencies'] += " " + openParenthesis -= 1 + if (openParenthesis == 0): + useFlagQuestion = False + useMatch = False + + if atom.startswith("||"): + openOr = True + + if atom.find("/") != -1 and (not atom.startswith("!")) and (not atom.endswith("?")): + # it's a package name /-??? + if ((useFlagQuestion) and (useMatch)) or ((not useFlagQuestion) and (not useMatch)): + # check if there's an OR + etpData['dependencies'] += atom + if (openOr): + etpData['dependencies'] += dbOR + else: + etpData['dependencies'] += " " + + if atom.startswith("!") and (not atom.endswith("?")): + if ((useFlagQuestion) and (useMatch)) or ((not useFlagQuestion) and (not useMatch)): + etpData['conflicts'] += atom + if (openOr): + etpData['conflicts'] += dbOR + else: + etpData['conflicts'] += " " + + + # format properly + tmpConflicts = list(set(etpData['conflicts'].split())) + etpData['conflicts'] = '' + for i in tmpConflicts: + i = i[1:] # remove "!" + etpData['conflicts'] += i+" " + etpData['conflicts'] = removeSpaceAtTheEnd(etpData['conflicts']) + + tmpDeps = list(set(etpData['dependencies'].split())) + etpData['dependencies'] = '' + for i in tmpDeps: + etpData['dependencies'] += i+" " + etpData['dependencies'] = removeSpaceAtTheEnd(etpData['dependencies']) + + # etpData['rdependencies'] + # Now we need to add environmental dependencies + # Notes (take the example of mplayer that needed a newer libcaca release): + # - we can use (from /var/db) "NEEDED" file to catch all the needed libraries to run the binary package + # - we can use (from /var/db) "CONTENTS" to rapidly search the NEEDED files in the file above + # return all the collected info + + # start collecting needed libraries + try: + f = open(tbz2TmpDir+"/"+dbNEEDED,"r") + includedBins = f.readlines() + f.close() + except IOError: + includedBins = "" + + neededLibraries = [] + # filter the first word + for line in includedBins: + line = line.strip().split() + line = line[0] + depLibs = commands.getoutput("ldd "+line).split("\n") + for i in depLibs: + i = i.strip() + if i.find("=>") != -1: + i = i.split("=>")[1] + # format properly + if i.startswith(" "): + i = i[1:] + if i.startswith("//"): + i = i[1:] + i = i.split()[0] + neededLibraries.append(i) + neededLibraries = list(set(neededLibraries)) + + runtimeNeededPackages = [] + runtimeNeededPackagesXT = [] + for i in neededLibraries: + if i.startswith("/"): # filter garbage + pkgs = commands.getoutput(pFindLibraryXT+i).split("\n") + if (pkgs[0] != ""): + for y in pkgs: + runtimeNeededPackagesXT.append(y) + y = dep_getkey(y) + runtimeNeededPackages.append(y) + + runtimeNeededPackages = list(set(runtimeNeededPackages)) + runtimeNeededPackagesXT = list(set(runtimeNeededPackagesXT)) + + # now keep only the ones not available in etpData['dependencies'] + for i in runtimeNeededPackages: + if etpData['dependencies'].find(i) == -1: + # filter itself + if (i != etpData['category']+"/"+etpData['name']): + etpData['rundependencies'] += i+" " + + for i in runtimeNeededPackagesXT: + x = dep_getkey(i) + if etpData['dependencies'].find(x) == -1: + # filter itself + if (x != etpData['category']+"/"+etpData['name']): + etpData['rundependenciesXT'] += i+" " + + # format properly + etpData['rundependencies'] = removeSpaceAtTheEnd(etpData['rundependencies']) + etpData['rundependenciesXT'] = removeSpaceAtTheEnd(etpData['rundependenciesXT']) + + # write API info + etpData['etpapi'] = ETP_API + + return etpData + +# This function generates the right path for putting the .etp file +# and take count of already available ones bumping version only if needed +def allocateFile(etpData): + + # this will be the first thing to return + etpOutput = [] + # append header + etpOutput.append(ETP_HEADER_TEXT) + for i in etpData: + if (etpData[i]): + etpOutput.append(i+": "+etpData[i]+"\n") + + # locate directory structure + etpOutfileDir = etpConst['packagesdatabasedir']+"/"+etpData['category']+"/"+etpData['name'] + etpOutfileDir = translateArch(etpOutfileDir,etpData['chost']) + etpOutfileName = etpData['name']+"-"+etpData['version']+"-etp"+ETP_REVISION_CONST+etpConst['extension'] + etpOutfilePath = etpOutfileDir+"/"+etpOutfileName + + # we've the directory, then create it + if (not os.path.isdir(etpOutfileDir)): + try: + os.makedirs(etpOutfileDir) + except OSError: + pass + # it's a brand new dir + etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] + else: # directory already exists, check for already available files + alreadyAvailableFiles = [] + for i in range(MAX_ETP_REVISION_COUNT+1): + testfile = re.subn(ETP_REVISION_CONST,str(i), etpOutfilePath)[0] + if (os.path.isfile(testfile)): + alreadyAvailableFiles.append(testfile) + if (alreadyAvailableFiles == []): + etpOutfilePath = re.subn(ETP_REVISION_CONST,"1", etpOutfilePath)[0] + else: + # grab the last one + possibleOldFile = alreadyAvailableFiles[len(alreadyAvailableFiles)-1] + # now compares both to see if they're equal or not + try: + import md5 + import string + a = open(possibleOldFile,"r") + cntA = a.readlines() + cntB = etpOutput + cntA = string.join(cntA) + cntB = string.join(cntB) + a.close() + md5A = md5.new() + md5B = md5.new() + md5A.update(cntA) + md5B.update(cntB) + if md5A.digest() == md5B.digest(): + etpOutfilePath = None + else: + # add 1 to: packagename-1.2.3-r1-etpX.etp + newFileCounter = int(possibleOldFile.split("-")[len(possibleOldFile.split("-"))-1].split(etpConst['extension'])[0].split(etpConst['extension'][1:])[1]) + newFileCounter += 1 + etpOutfilePath = re.subn(ETP_REVISION_CONST,str(newFileCounter), etpOutfilePath)[0] + except OSError: + etpOutfilePath = possibleOldFile + + return etpOutput, etpOutfilePath \ No newline at end of file