added FTP server management, put the basis for the activator stack

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@192 cd1c1023-2f26-0410-ae45-c471fc1f0318

conf/activator.conf

@@ -1,14 +1,19 @@
 # Project Entropy 1.0 activator configuration file
 
 # Main mirror configuration
-# this is the URI for the main mirror
-# It can be HTTP,FTP,RSYNC
+# this is the URI for the mirrors
+# will be used in order
+# It can be FTP,RSYNC
 #
 #   example:
-#   main-mirror-upload|ftp://username:password@address
+#   mirror-upload|ftp://username:password@address
 #   main-mirror-download|ftp://address
 #
 #   note: the packages/%ARCH% part is added by Entropy
 #
-main-mirror-upload|ftp://username:password@address
-main-mirror-download|ftp://address
+mirror-upload|ftp://username:password@address
+mirror-upload|ftp://username:password@address
+mirror-upload|ftp://username:password@address
+mirror-download|ftp://address
+mirror-download|http://address
+mirror-download|rsync://address

handlers/activator

@@ -43,8 +43,8 @@ def print_help():
     entropyTools.print_info("  --verbose\t\tprint debugging info")
     entropyTools.print_info("  --nocolor\t\tdisable colorized output")
     entropyTools.print_info(entropyTools.blue("Tools available: "))
-    entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t\t to sync all the upload mirrors"))
-    entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t\t to download all the binary packages (will overwrite)")
+    entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t to sync all the upload mirrors"))
+    entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t to download all the binary packages (will overwrite)")
     entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)")
     entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)")
     entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples")

handlers/enzyme

@@ -50,6 +50,7 @@ def print_help():
     entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t just ask before doing what should be done")
     entropyTools.print_info(" \t\t"+entropyTools.red("--repackage-installed")+"\t creates binaries of all the installed packages")
     entropyTools.print_info(" \t\t"+entropyTools.red("--skipfirst")+"\t\t skip the first package in the packages list")
+    entropyTools.print_info(" \t\t"+entropyTools.red("--skip=n")+"\t\t skip N packages")
     entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("build"))+entropyTools.yellow("\t\t to build all the packages specified in <atom(s)>"))
     entropyTools.print_info(" \t\t"+entropyTools.red("--force-rebuild")+"\t\t force the building of the package, nevertheless")
     entropyTools.print_info(" \t\t"+entropyTools.red("--force-repackage")+"\t force the repackaging of all the possible package")

libraries/activatorTools.py

@@ -31,4 +31,18 @@ import commands
 import string
 
 def sync(options):
-    print "hello!"
+    print "hello!"
+
+    for uri in etpConst['activatoruploaduris']:
+	ftp = activatorFTP(uri)
+	print "Listing the content of: "+ftp.getFTPHost()
+	print "at port: "+str(ftp.getFTPPort())
+	print "in dir: "+ftp.getFTPDir()
+	print ftp.listFTPdir()
+	#print "deleting file: XML-XSLT-0.48.tbz2"
+	#rc = ftp.deleteFile("XML-XSLT-0.48.tbz2")
+	#print rc
+	#print "uploading file..."
+	#rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2")
+	#print str(rc)
+	ftp.closeFTPConnection()

libraries/entropyConstants.py

@@ -96,6 +96,9 @@ etpConst = {
     'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration
     'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file
     'enzymeconf': ETP_CONF_DIR+"/enzyme.conf", # enzyme.conf file
+    'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file
+    'activatoruploaduris': [],# list of URIs that activator can use to upload files (parsed from activator.conf)
+    'activatordownloaduris': [],# list of URIs that activator can use to fetch data
     'digestfile': "Manifest", # file that contains md5 hashes
     'extension': ".etp", # entropy files extension
     'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path where we'll have to append the packages|uri part.
@@ -230,6 +233,34 @@ for i in ovlst:
 	_ovlst.append(etpConst['overlaysdir']+"/"+i)
 etpConst['overlays'] = string.join(_ovlst," ")
 
+# activator section
+if (not os.path.isfile(etpConst['activatorconf'])):
+    print "ERROR: "+etpConst['activatorconf']+" does not exist"
+    sys.exit(50)
+else:
+    try:
+	if (os.stat(etpConst['activatorconf'])[0] != 33152):
+	    os.chmod(etpConst['activatorconf'],0600)
+    except:
+	print "ERROR: cannot chmod 0600 file: "+etpConst['activatorconf']
+	sys.exit(50)
+    # fill etpConst['activatoruploaduris'] and etpConst['activatordownloaduris']
+    f = open(etpConst['activatorconf'],"r")
+    actconffile = f.readlines()
+    f.close()
+    for line in actconffile:
+	line = line.strip()
+	if line.startswith("mirror-upload|") and (len(line.split("mirror-upload|")) == 2):
+	    uri = line.split("mirror-upload|")[1]
+	    if uri.endswith("/"):
+		uri = uri[:len(uri)-1]
+	    etpConst['activatoruploaduris'].append(uri)
+	if line.startswith("mirror-download|") and (len(line.split("mirror-download|")) == 2):
+	    uri = line.split("mirror-download|")[1]
+	    if uri.endswith("/"):
+		uri = uri[:len(uri)-1]
+	    etpConst['activatordownloaduris'].append(uri)
+
 # Portage /var/db/<pkgcat>/<pkgname-pkgver>/*
 # you never know if gentoo devs change these things
 dbDESCRIPTION = "DESCRIPTION"

libraries/entropyTools.py

@@ -124,7 +124,7 @@ def calculateAtomUSEFlags(atom):
     result = commands.getoutput(cmd).split("\n")
     useparm = ""
     for line in result:
-	if line.startswith("[ebuild"):
+	if line.startswith("[ebuild") and (line.find("USE=") != -1):
 	    useparm = line.split('USE="')[len(line.split('USE="'))-1].split('"')[0].strip()
     useparm = useparm.split()
     _useparm = []
@@ -349,7 +349,7 @@ def parseElogFile(atom):
 		logline = True
 		continue
 		# disable all the others
-	    elif line.startswith("INFO:"):
+	    elif line.startswith("INFO:") or line.startswith("LOG:"):
 		logline = False
 		continue
 	    if (logline) and (line.strip() != ""):
@@ -674,6 +674,124 @@ def getPortageAppDbPath():
 	return rc+"/"
     return rc
 
+# ------ BEGIN: activator tools ------
+
+class activatorFTP:
+
+    # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org
+    # this must be run before calling the other functions
+    def __init__(self, ftpuri):
+	
+	from ftplib import FTP
+	
+	self.ftpuri = ftpuri
+	
+	self.ftphost = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1]
+	self.ftphost = self.ftphost.split("@")[len(self.ftphost.split("@"))-1]
+	self.ftphost = self.ftphost.split("/")[0]
+	self.ftphost = self.ftphost.split(":")[0]
+	
+	self.ftpuser = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1].split(":")[0]
+	if (self.ftpuser == ""):
+	    self.ftpuser = "anonymous@"
+	    self.ftppassword = "anonymous"
+	else:
+	    self.ftppassword = ftpuri.split("@")[:len(ftpuri.split("@"))-1]
+	    if len(self.ftppassword) > 1:
+		import string
+		self.ftppassword = string.join(self.ftppassword,"@")
+		self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1]
+		if (self.ftppassword == ""):
+		    self.ftppassword = "anonymous"
+	    else:
+		self.ftppassword = self.ftppassword[0]
+		self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1]
+		if (self.ftppassword == ""):
+		    self.ftppassword = "anonymous"
+	
+	self.ftpport = ftpuri.split(":")[len(ftpuri.split(":"))-1]
+	try:
+	    self.ftpport = int(self.ftpport)
+	except:
+	    self.ftpport = 21
+	
+	self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1]
+	self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1]
+	self.ftpdir = self.ftpdir.split(":")[0]
+	self.ftpdir = self.ftpdir+translateArchFromUname(etpConst['binaryurirelativepath'])
+	if self.ftpdir.endswith("/"):
+	    self.ftpdir = self.ftpdir[:len(self.ftpdir)-1]
+
+	print self.ftpdir
+
+	self.ftpconn = FTP(self.ftphost)
+	self.ftpconn.login(self.ftpuser,self.ftppassword)
+	# change to our dir
+	self.ftpconn.cwd(self.ftpdir)
+
+    def getFTPHost(self):
+	return self.ftphost
+
+    def getFTPPort(self):
+	return self.ftpport
+
+    def getFTPDir(self):
+	return self.ftpdir
+
+    def getCWD(self):
+	return self.ftpconn.pwd()
+
+    def setCWD(self,dir):
+	self.ftpconn.cwd(dir)
+
+    # list files and directory of a FTP
+    # @returns a list
+    def listFTPdir(self):
+	# directory is: self.ftpdir
+	try:
+	    rc = self.ftpconn.nlst()
+	    _rc = []
+	    for i in rc:
+		_rc.append(i.split("/")[len(i.split("/"))-1])
+	    rc = _rc
+	except:
+	    return []
+	return rc
+
+    def deleteFile(self,file):
+	try:
+	    rc = self.ftpconn.delete(self.ftpdir+"/"+file)
+	    if rc.startswith("250"):
+		return True
+	    else:
+		return False
+	except:
+	    return False
+
+    def uploadFile(self,file,ascii = False):
+	if (not ascii):
+	    f = open(file)
+	    file = file.split("/")[len(file.split("/"))-1]
+	    rc = self.ftpconn.storbinary("STOR "+file,f)
+	    return rc
+	else:
+	    f = open(file)
+	    file = file.split("/")[len(file.split("/"))-1]
+	    rc = self.ftpconn.storlines("STOR "+file,f)
+	    return rc
+
+    # also used to move files
+    def renameFile(self,fromfile,tofile)
+	self.ftpconn.rename(fromfile,tofile)
+
+    def closeFTPConnection(self):
+	self.ftpconn.quit()
+
+
+    # FIXME: add upload/delete/mv commands
+
+# ------ END: activator tools ------
+
 # get a list, returns a sorted list
 def alphaSorter(seq):
     def stripter(s, goodchrs):

libraries/enzymeTools.py

@@ -237,13 +237,13 @@ def build(atoms):
         if len(toBeBuilt) > 1:
 	    cleanatomlist = []
 	    for atom in toBeBuilt:
-	        if (not atom.startswith(">")) and (not atom.startswith("<")) and (not atom.startswith("=")) and (not isjustname(atom)):
+	        if (not atom.startswith(">")) and (not atom.startswith("<")) and (not atom.startswith("~")) and (not atom.startswith("=")) and (not isjustname(atom)):
 		    cleanatomlist.append("="+atom)
 	        else:
 		    cleanatomlist.append(atom)
             atoms = string.join(cleanatomlist," ")
         else:
-	    if atoms[0].startswith(">") or atoms[0].startswith("<") or atoms[0].startswith("=") or isjustname(atoms[0]):
+	    if atoms[0].startswith(">") or atoms[0].startswith("<") or atoms[0].startswith("~") or atoms[0].startswith("=") or isjustname(atoms[0]):
 	        atoms = atoms[0]
 	    else:
 	        atoms = "="+atoms[0]
@@ -538,6 +538,7 @@ def world(options):
     enzymeRequestPretend = False
     enzymeRequestJustRepackageWorld = False
     enzymeRequestSkipfirst = False
+    enzymeRequestSkipN = False
     for i in myopts:
         if ( i == "--verbose" ) or ( i == "-v" ):
 	    enzymeRequestVerbose = True
@@ -549,6 +550,13 @@ def world(options):
 	    enzymeRequestPretend = True
 	elif ( i == "--skipfirst" ):
 	    enzymeRequestSkipfirst = True
+	elif ( i.startswith("--skip=") ):
+	    enzymeRequestSkipN = True
+	    skip_number = i.split("--skip=")[len(i.split("--skip="))-1]
+	    try:
+		skipN = int(skip_number)
+	    except:
+		skipN = 1
 	elif ( i == "--repackage-installed" ):
 	    enzymeRequestJustRepackageWorld = True
 	elif ( i == "--deep" ):
@@ -601,6 +609,8 @@ def world(options):
 	
 	if (enzymeRequestSkipfirst):
 	    deplist = deplist[1:]
+	elif (enzymeRequestSkipN):
+	    deplist = deplist[skipN:]
 	
 	# composing the request
 	atoms = []