improved FTP support, started to define activator edges

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@195 cd1c1023-2f26-0410-ae45-c471fc1f0318

TODO

@@ -3,7 +3,8 @@ TODO list (for developers only):
 - activator should care about the removal of the old packages
 - build() and world(), on enzyme, add the support for whitelist+cron
 - build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form)
-- enzyme, add search module?
+- should reagent keep in sync its database directory ?
+- FTP client: add download()
 - Sabayon Linux USE flags: remove all server related use flags
 
 Project Status:

conf/activator.conf

@@ -12,8 +12,4 @@
 #   note: the packages/%ARCH% part is added by Entropy
 #
 mirror-upload|ftp://username:password@address
-mirror-upload|ftp://username:password@address
-mirror-upload|ftp://username:password@address
 mirror-download|ftp://address
-mirror-download|http://address
-mirror-download|rsync://address

handlers/activator

@@ -46,7 +46,6 @@ def print_help():
     entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t to sync all the upload mirrors"))
     entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t to download all the binary packages (will overwrite)")
     entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)")
-    entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)")
     entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples")
 
 options = sys.argv[1:]

libraries/activatorTools.py

@@ -31,18 +31,51 @@ import commands
 import string
 
 def sync(options):
-    print "hello!"
 
+    # translate %ARCH%
+    etpConst['packagessuploaddir'] = translateArchFromUname(etpConst['packagessuploaddir'])
+    etpConst['packagesdatabasedir'] = translateArchFromUname(etpConst['packagesdatabasedir'])
+    etpConst['packagesbindir'] = translateArchFromUname(etpConst['packagesbindir'])
+    etpConst['binaryurirelativepath'] = translateArchFromUname(etpConst['binaryurirelativepath'])
+    etpConst['etpurirelativepath'] = translateArchFromUname(etpConst['etpurirelativepath'])
+
+    print_info(green(" * ")+red("Collecting local binary packages..."),back = True)
+    localtbz2counter = 0
+    localTbz2Files = []
+    for file in os.listdir(etpConst['packagessuploaddir']):
+	if file.endswith(".tbz2"):
+	    localTbz2Files.append([ file , getFileTimeStamp(etpConst['packagessuploaddir']+"/"+file) ])
+	    localtbz2counter += 1
+    print_info(green(" * ")+red("Packages directory:\t")+bold(str(localtbz2counter))+red(" packages ready for the upload."))
+
+    # INFO: the Entropy repository will be, synced to the latest on the server, compressed, uploaded and kept there?
+    print_info(green(" * ")+red("Collecting local Entropy repository entries..."),back = True)
+    localtetpcounter = 0
+    localEtpFiles = []
+    for (dir, sub, files) in os.walk(etpConst['packagesdatabasedir']):
+	localEtpFiles.append(dir)
+	for file in files:
+	    localEtpFiles.append(dir+"/"+file)
+	    if file.endswith(etpConst['extension']):
+		localtetpcounter += 1
+    print_info(green(" * ")+red("Entropy directory:\t")+bold(str(localtetpcounter))+red(" specification files available."))
+
+    # packages relative uri: etpConst['binaryurirelativepath']
+    # entropy relative uri : etpConst['etpurirelativepath']
+    #print "deleting file: XML-XSLT-0.48.tbz2"
+    #rc = ftp.deleteFile("XML-XSLT-0.48.tbz2")
+    #print rc
+    #print "uploading file..."
+    #rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2")
+    #print str(rc)
+
+    # For each URI do the same thing
     for uri in etpConst['activatoruploaduris']:
 	ftp = activatorFTP(uri)
 	print "Listing the content of: "+ftp.getFTPHost()
 	print "at port: "+str(ftp.getFTPPort())
 	print "in dir: "+ftp.getFTPDir()
 	print ftp.listFTPdir()
-	#print "deleting file: XML-XSLT-0.48.tbz2"
-	#rc = ftp.deleteFile("XML-XSLT-0.48.tbz2")
-	#print rc
-	#print "uploading file..."
-	#rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2")
-	#print str(rc)
-	ftp.closeFTPConnection()
+	print ftp.spawnFTPCommand("mdtm index.htm")
+	ftp.closeFTPConnection()
+

libraries/entropyConstants.py

@@ -103,7 +103,8 @@ etpConst = {
     'activatordownloaduris': [],# list of URIs that activator can use to fetch data
     'digestfile': "Manifest", # file that contains md5 hashes
     'extension': ".etp", # entropy files extension
-    'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path where we'll have to append the packages|uri part.
+    'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository.
+    'etpurirelativepath': "/database/"+ETP_ARCH_CONST+"/", # Relative remote path for the .etp repository.
     'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit
 }
 

libraries/entropyTools.py

@@ -761,12 +761,9 @@ class activatorFTP:
 	self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1]
 	self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1]
 	self.ftpdir = self.ftpdir.split(":")[0]
-	self.ftpdir = self.ftpdir+translateArchFromUname(etpConst['binaryurirelativepath'])
 	if self.ftpdir.endswith("/"):
 	    self.ftpdir = self.ftpdir[:len(self.ftpdir)-1]
 
-	print self.ftpdir
-
 	self.ftpconn = FTP(self.ftphost)
 	self.ftpconn.login(self.ftpuser,self.ftppassword)
 	# change to our dir
@@ -787,6 +784,14 @@ class activatorFTP:
     def setCWD(self,dir):
 	self.ftpconn.cwd(dir)
 
+    def getFileMtime(self,path):
+	rc = self.ftpconn.sendcmd("mdtm "+path)
+	return rc.split()[len(rc.split())-1]
+
+    def spawnFTPCommand(self,cmd):
+	rc = self.ftpconn.sendcmd(cmd)
+	return rc
+
     # list files and directory of a FTP
     # @returns a list
     def listFTPdir(self):
@@ -823,6 +828,19 @@ class activatorFTP:
 	    rc = self.ftpconn.storlines("STOR "+file,f)
 	    return rc
 
+    def downloadFile(self,filepath,downloaddir,ascii = False):
+	file = filepath.split("/")[len(filepath.split("/"))-1]
+	if (not ascii):
+	    f = open(downloaddir+"/"+file,"wb")
+	    self.ftpconn.retrbinary('RETR '+file,f.write)
+	    f.flush()
+	    f.close()
+	else:
+	    f = open(downloaddir+"/"+file,"w")
+	    self.ftpconn.retrlines('RETR '+file,f.write)
+	    f.flush()
+	    f.close()
+
     # also used to move files
     def renameFile(self,fromfile,tofile):
 	self.ftpconn.rename(fromfile,tofile)
@@ -871,6 +889,23 @@ def packageSearch(keyword):
 
     return SearchDirs
 
+
+def getFileUnixMtime(path):
+    return os.path.getmtime(path)
+
+def getFileTimeStamp(path):
+    from datetime import datetime
+    # used in this way for convenience
+    unixtime = os.path.getmtime(path)
+    humantime = datetime.fromtimestamp(unixtime)
+    # format properly
+    humantime = str(humantime)
+    outputtime = ""
+    for chr in humantime:
+	if chr != "-" and chr != " " and chr != ":":
+	    outputtime += chr
+    return outputtime
+
 # get a list, returns a sorted list
 def alphaSorter(seq):
     def stripter(s, goodchrs):