From 42b5c2d2af9bdaaaebb7f435651af389e96c7dac Mon Sep 17 00:00:00 2001 From: lxnay Date: Sun, 25 Mar 2007 16:03:33 +0000 Subject: [PATCH] added FTP server management, put the basis for the activator stack git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@192 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- conf/activator.conf | 15 +++-- handlers/activator | 4 +- handlers/enzyme | 1 + libraries/activatorTools.py | 16 ++++- libraries/entropyConstants.py | 31 +++++++++ libraries/entropyTools.py | 122 +++++++++++++++++++++++++++++++++- libraries/enzymeTools.py | 14 +++- 7 files changed, 191 insertions(+), 12 deletions(-) diff --git a/conf/activator.conf b/conf/activator.conf index 4ccb43371..c766a13ef 100644 --- a/conf/activator.conf +++ b/conf/activator.conf @@ -1,14 +1,19 @@ # Project Entropy 1.0 activator configuration file # Main mirror configuration -# this is the URI for the main mirror -# It can be HTTP,FTP,RSYNC +# this is the URI for the mirrors +# will be used in order +# It can be FTP,RSYNC # # example: -# main-mirror-upload|ftp://username:password@address +# mirror-upload|ftp://username:password@address # main-mirror-download|ftp://address # # note: the packages/%ARCH% part is added by Entropy # -main-mirror-upload|ftp://username:password@address -main-mirror-download|ftp://address +mirror-upload|ftp://username:password@address +mirror-upload|ftp://username:password@address +mirror-upload|ftp://username:password@address +mirror-download|ftp://address +mirror-download|http://address +mirror-download|rsync://address diff --git a/handlers/activator b/handlers/activator index aba2cfbd2..e8f4e02a6 100644 --- a/handlers/activator +++ b/handlers/activator @@ -43,8 +43,8 @@ def print_help(): entropyTools.print_info(" --verbose\t\tprint debugging info") entropyTools.print_info(" --nocolor\t\tdisable colorized output") entropyTools.print_info(entropyTools.blue("Tools available: ")) - entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t\t to sync all the upload mirrors")) - entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t\t to download all the binary packages (will overwrite)") + entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t to sync all the upload mirrors")) + entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t to download all the binary packages (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples") diff --git a/handlers/enzyme b/handlers/enzyme index b11879e22..f478980b7 100644 --- a/handlers/enzyme +++ b/handlers/enzyme @@ -50,6 +50,7 @@ def print_help(): entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t just ask before doing what should be done") entropyTools.print_info(" \t\t"+entropyTools.red("--repackage-installed")+"\t creates binaries of all the installed packages") entropyTools.print_info(" \t\t"+entropyTools.red("--skipfirst")+"\t\t skip the first package in the packages list") + entropyTools.print_info(" \t\t"+entropyTools.red("--skip=n")+"\t\t skip N packages") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("build"))+entropyTools.yellow("\t\t to build all the packages specified in ")) entropyTools.print_info(" \t\t"+entropyTools.red("--force-rebuild")+"\t\t force the building of the package, nevertheless") entropyTools.print_info(" \t\t"+entropyTools.red("--force-repackage")+"\t force the repackaging of all the possible package") diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 171d873f3..db0860680 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -31,4 +31,18 @@ import commands import string def sync(options): - print "hello!" \ No newline at end of file + print "hello!" + + for uri in etpConst['activatoruploaduris']: + ftp = activatorFTP(uri) + print "Listing the content of: "+ftp.getFTPHost() + print "at port: "+str(ftp.getFTPPort()) + print "in dir: "+ftp.getFTPDir() + print ftp.listFTPdir() + #print "deleting file: XML-XSLT-0.48.tbz2" + #rc = ftp.deleteFile("XML-XSLT-0.48.tbz2") + #print rc + #print "uploading file..." + #rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2") + #print str(rc) + ftp.closeFTPConnection() \ No newline at end of file diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 8d86d8823..b1dc7752d 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -96,6 +96,9 @@ etpConst = { 'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration 'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file 'enzymeconf': ETP_CONF_DIR+"/enzyme.conf", # enzyme.conf file + 'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file + 'activatoruploaduris': [],# list of URIs that activator can use to upload files (parsed from activator.conf) + 'activatordownloaduris': [],# list of URIs that activator can use to fetch data 'digestfile': "Manifest", # file that contains md5 hashes 'extension': ".etp", # entropy files extension 'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path where we'll have to append the packages|uri part. @@ -230,6 +233,34 @@ for i in ovlst: _ovlst.append(etpConst['overlaysdir']+"/"+i) etpConst['overlays'] = string.join(_ovlst," ") +# activator section +if (not os.path.isfile(etpConst['activatorconf'])): + print "ERROR: "+etpConst['activatorconf']+" does not exist" + sys.exit(50) +else: + try: + if (os.stat(etpConst['activatorconf'])[0] != 33152): + os.chmod(etpConst['activatorconf'],0600) + except: + print "ERROR: cannot chmod 0600 file: "+etpConst['activatorconf'] + sys.exit(50) + # fill etpConst['activatoruploaduris'] and etpConst['activatordownloaduris'] + f = open(etpConst['activatorconf'],"r") + actconffile = f.readlines() + f.close() + for line in actconffile: + line = line.strip() + if line.startswith("mirror-upload|") and (len(line.split("mirror-upload|")) == 2): + uri = line.split("mirror-upload|")[1] + if uri.endswith("/"): + uri = uri[:len(uri)-1] + etpConst['activatoruploaduris'].append(uri) + if line.startswith("mirror-download|") and (len(line.split("mirror-download|")) == 2): + uri = line.split("mirror-download|")[1] + if uri.endswith("/"): + uri = uri[:len(uri)-1] + etpConst['activatordownloaduris'].append(uri) + # Portage /var/db///* # you never know if gentoo devs change these things dbDESCRIPTION = "DESCRIPTION" diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 44bc7ec26..e52fa4e50 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -124,7 +124,7 @@ def calculateAtomUSEFlags(atom): result = commands.getoutput(cmd).split("\n") useparm = "" for line in result: - if line.startswith("[ebuild"): + if line.startswith("[ebuild") and (line.find("USE=") != -1): useparm = line.split('USE="')[len(line.split('USE="'))-1].split('"')[0].strip() useparm = useparm.split() _useparm = [] @@ -349,7 +349,7 @@ def parseElogFile(atom): logline = True continue # disable all the others - elif line.startswith("INFO:"): + elif line.startswith("INFO:") or line.startswith("LOG:"): logline = False continue if (logline) and (line.strip() != ""): @@ -674,6 +674,124 @@ def getPortageAppDbPath(): return rc+"/" return rc +# ------ BEGIN: activator tools ------ + +class activatorFTP: + + # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org + # this must be run before calling the other functions + def __init__(self, ftpuri): + + from ftplib import FTP + + self.ftpuri = ftpuri + + self.ftphost = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1] + self.ftphost = self.ftphost.split("@")[len(self.ftphost.split("@"))-1] + self.ftphost = self.ftphost.split("/")[0] + self.ftphost = self.ftphost.split(":")[0] + + self.ftpuser = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1].split(":")[0] + if (self.ftpuser == ""): + self.ftpuser = "anonymous@" + self.ftppassword = "anonymous" + else: + self.ftppassword = ftpuri.split("@")[:len(ftpuri.split("@"))-1] + if len(self.ftppassword) > 1: + import string + self.ftppassword = string.join(self.ftppassword,"@") + self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] + if (self.ftppassword == ""): + self.ftppassword = "anonymous" + else: + self.ftppassword = self.ftppassword[0] + self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] + if (self.ftppassword == ""): + self.ftppassword = "anonymous" + + self.ftpport = ftpuri.split(":")[len(ftpuri.split(":"))-1] + try: + self.ftpport = int(self.ftpport) + except: + self.ftpport = 21 + + self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1] + self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1] + self.ftpdir = self.ftpdir.split(":")[0] + self.ftpdir = self.ftpdir+translateArchFromUname(etpConst['binaryurirelativepath']) + if self.ftpdir.endswith("/"): + self.ftpdir = self.ftpdir[:len(self.ftpdir)-1] + + print self.ftpdir + + self.ftpconn = FTP(self.ftphost) + self.ftpconn.login(self.ftpuser,self.ftppassword) + # change to our dir + self.ftpconn.cwd(self.ftpdir) + + def getFTPHost(self): + return self.ftphost + + def getFTPPort(self): + return self.ftpport + + def getFTPDir(self): + return self.ftpdir + + def getCWD(self): + return self.ftpconn.pwd() + + def setCWD(self,dir): + self.ftpconn.cwd(dir) + + # list files and directory of a FTP + # @returns a list + def listFTPdir(self): + # directory is: self.ftpdir + try: + rc = self.ftpconn.nlst() + _rc = [] + for i in rc: + _rc.append(i.split("/")[len(i.split("/"))-1]) + rc = _rc + except: + return [] + return rc + + def deleteFile(self,file): + try: + rc = self.ftpconn.delete(self.ftpdir+"/"+file) + if rc.startswith("250"): + return True + else: + return False + except: + return False + + def uploadFile(self,file,ascii = False): + if (not ascii): + f = open(file) + file = file.split("/")[len(file.split("/"))-1] + rc = self.ftpconn.storbinary("STOR "+file,f) + return rc + else: + f = open(file) + file = file.split("/")[len(file.split("/"))-1] + rc = self.ftpconn.storlines("STOR "+file,f) + return rc + + # also used to move files + def renameFile(self,fromfile,tofile) + self.ftpconn.rename(fromfile,tofile) + + def closeFTPConnection(self): + self.ftpconn.quit() + + + # FIXME: add upload/delete/mv commands + +# ------ END: activator tools ------ + # get a list, returns a sorted list def alphaSorter(seq): def stripter(s, goodchrs): diff --git a/libraries/enzymeTools.py b/libraries/enzymeTools.py index 79246b6a3..774dab5c9 100644 --- a/libraries/enzymeTools.py +++ b/libraries/enzymeTools.py @@ -237,13 +237,13 @@ def build(atoms): if len(toBeBuilt) > 1: cleanatomlist = [] for atom in toBeBuilt: - if (not atom.startswith(">")) and (not atom.startswith("<")) and (not atom.startswith("=")) and (not isjustname(atom)): + if (not atom.startswith(">")) and (not atom.startswith("<")) and (not atom.startswith("~")) and (not atom.startswith("=")) and (not isjustname(atom)): cleanatomlist.append("="+atom) else: cleanatomlist.append(atom) atoms = string.join(cleanatomlist," ") else: - if atoms[0].startswith(">") or atoms[0].startswith("<") or atoms[0].startswith("=") or isjustname(atoms[0]): + if atoms[0].startswith(">") or atoms[0].startswith("<") or atoms[0].startswith("~") or atoms[0].startswith("=") or isjustname(atoms[0]): atoms = atoms[0] else: atoms = "="+atoms[0] @@ -538,6 +538,7 @@ def world(options): enzymeRequestPretend = False enzymeRequestJustRepackageWorld = False enzymeRequestSkipfirst = False + enzymeRequestSkipN = False for i in myopts: if ( i == "--verbose" ) or ( i == "-v" ): enzymeRequestVerbose = True @@ -549,6 +550,13 @@ def world(options): enzymeRequestPretend = True elif ( i == "--skipfirst" ): enzymeRequestSkipfirst = True + elif ( i.startswith("--skip=") ): + enzymeRequestSkipN = True + skip_number = i.split("--skip=")[len(i.split("--skip="))-1] + try: + skipN = int(skip_number) + except: + skipN = 1 elif ( i == "--repackage-installed" ): enzymeRequestJustRepackageWorld = True elif ( i == "--deep" ): @@ -601,6 +609,8 @@ def world(options): if (enzymeRequestSkipfirst): deplist = deplist[1:] + elif (enzymeRequestSkipN): + deplist = deplist[skipN:] # composing the request atoms = []