From bf3200ec6daa5f024e5b7d5a8784013e818b5b1f Mon Sep 17 00:00:00 2001 From: lxnay Date: Mon, 26 Mar 2007 08:11:08 +0000 Subject: [PATCH] improved FTP support, started to define activator edges git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@195 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 3 ++- conf/activator.conf | 4 --- handlers/activator | 1 - libraries/activatorTools.py | 49 +++++++++++++++++++++++++++++------ libraries/entropyConstants.py | 3 ++- libraries/entropyTools.py | 41 ++++++++++++++++++++++++++--- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/TODO b/TODO index 768d5300f..3d27c44d5 100644 --- a/TODO +++ b/TODO @@ -3,7 +3,8 @@ TODO list (for developers only): - activator should care about the removal of the old packages - build() and world(), on enzyme, add the support for whitelist+cron - build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form) -- enzyme, add search module? +- should reagent keep in sync its database directory ? +- FTP client: add download() - Sabayon Linux USE flags: remove all server related use flags Project Status: diff --git a/conf/activator.conf b/conf/activator.conf index c766a13ef..48143524a 100644 --- a/conf/activator.conf +++ b/conf/activator.conf @@ -12,8 +12,4 @@ # note: the packages/%ARCH% part is added by Entropy # mirror-upload|ftp://username:password@address -mirror-upload|ftp://username:password@address -mirror-upload|ftp://username:password@address mirror-download|ftp://address -mirror-download|http://address -mirror-download|rsync://address diff --git a/handlers/activator b/handlers/activator index e8f4e02a6..c6e3ae669 100644 --- a/handlers/activator +++ b/handlers/activator @@ -46,7 +46,6 @@ def print_help(): entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("sync"))+entropyTools.yellow("\t to sync all the upload mirrors")) entropyTools.print_info(" \t\t"+entropyTools.red("--download-packages")+"\t to download all the binary packages (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)") - entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples") options = sys.argv[1:] diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index db0860680..8ebf5d5ed 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -31,18 +31,51 @@ import commands import string def sync(options): - print "hello!" + # translate %ARCH% + etpConst['packagessuploaddir'] = translateArchFromUname(etpConst['packagessuploaddir']) + etpConst['packagesdatabasedir'] = translateArchFromUname(etpConst['packagesdatabasedir']) + etpConst['packagesbindir'] = translateArchFromUname(etpConst['packagesbindir']) + etpConst['binaryurirelativepath'] = translateArchFromUname(etpConst['binaryurirelativepath']) + etpConst['etpurirelativepath'] = translateArchFromUname(etpConst['etpurirelativepath']) + + print_info(green(" * ")+red("Collecting local binary packages..."),back = True) + localtbz2counter = 0 + localTbz2Files = [] + for file in os.listdir(etpConst['packagessuploaddir']): + if file.endswith(".tbz2"): + localTbz2Files.append([ file , getFileTimeStamp(etpConst['packagessuploaddir']+"/"+file) ]) + localtbz2counter += 1 + print_info(green(" * ")+red("Packages directory:\t")+bold(str(localtbz2counter))+red(" packages ready for the upload.")) + + # INFO: the Entropy repository will be, synced to the latest on the server, compressed, uploaded and kept there? + print_info(green(" * ")+red("Collecting local Entropy repository entries..."),back = True) + localtetpcounter = 0 + localEtpFiles = [] + for (dir, sub, files) in os.walk(etpConst['packagesdatabasedir']): + localEtpFiles.append(dir) + for file in files: + localEtpFiles.append(dir+"/"+file) + if file.endswith(etpConst['extension']): + localtetpcounter += 1 + print_info(green(" * ")+red("Entropy directory:\t")+bold(str(localtetpcounter))+red(" specification files available.")) + + # packages relative uri: etpConst['binaryurirelativepath'] + # entropy relative uri : etpConst['etpurirelativepath'] + #print "deleting file: XML-XSLT-0.48.tbz2" + #rc = ftp.deleteFile("XML-XSLT-0.48.tbz2") + #print rc + #print "uploading file..." + #rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2") + #print str(rc) + + # For each URI do the same thing for uri in etpConst['activatoruploaduris']: ftp = activatorFTP(uri) print "Listing the content of: "+ftp.getFTPHost() print "at port: "+str(ftp.getFTPPort()) print "in dir: "+ftp.getFTPDir() print ftp.listFTPdir() - #print "deleting file: XML-XSLT-0.48.tbz2" - #rc = ftp.deleteFile("XML-XSLT-0.48.tbz2") - #print rc - #print "uploading file..." - #rc = ftp.uploadFile("/var/lib/entropy/store/x86/alsa-lib-1.0.14_rc3.tbz2") - #print str(rc) - ftp.closeFTPConnection() \ No newline at end of file + print ftp.spawnFTPCommand("mdtm index.htm") + ftp.closeFTPConnection() + diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 54f8f9015..b5964c586 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -103,7 +103,8 @@ etpConst = { 'activatordownloaduris': [],# list of URIs that activator can use to fetch data 'digestfile': "Manifest", # file that contains md5 hashes 'extension': ".etp", # entropy files extension - 'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path where we'll have to append the packages|uri part. + 'binaryurirelativepath': "/packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. + 'etpurirelativepath': "/database/"+ETP_ARCH_CONST+"/", # Relative remote path for the .etp repository. 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit } diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 73e69ce2f..62e9807dc 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -761,12 +761,9 @@ class activatorFTP: self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1] self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1] self.ftpdir = self.ftpdir.split(":")[0] - self.ftpdir = self.ftpdir+translateArchFromUname(etpConst['binaryurirelativepath']) if self.ftpdir.endswith("/"): self.ftpdir = self.ftpdir[:len(self.ftpdir)-1] - print self.ftpdir - self.ftpconn = FTP(self.ftphost) self.ftpconn.login(self.ftpuser,self.ftppassword) # change to our dir @@ -787,6 +784,14 @@ class activatorFTP: def setCWD(self,dir): self.ftpconn.cwd(dir) + def getFileMtime(self,path): + rc = self.ftpconn.sendcmd("mdtm "+path) + return rc.split()[len(rc.split())-1] + + def spawnFTPCommand(self,cmd): + rc = self.ftpconn.sendcmd(cmd) + return rc + # list files and directory of a FTP # @returns a list def listFTPdir(self): @@ -823,6 +828,19 @@ class activatorFTP: rc = self.ftpconn.storlines("STOR "+file,f) return rc + def downloadFile(self,filepath,downloaddir,ascii = False): + file = filepath.split("/")[len(filepath.split("/"))-1] + if (not ascii): + f = open(downloaddir+"/"+file,"wb") + self.ftpconn.retrbinary('RETR '+file,f.write) + f.flush() + f.close() + else: + f = open(downloaddir+"/"+file,"w") + self.ftpconn.retrlines('RETR '+file,f.write) + f.flush() + f.close() + # also used to move files def renameFile(self,fromfile,tofile): self.ftpconn.rename(fromfile,tofile) @@ -871,6 +889,23 @@ def packageSearch(keyword): return SearchDirs + +def getFileUnixMtime(path): + return os.path.getmtime(path) + +def getFileTimeStamp(path): + from datetime import datetime + # used in this way for convenience + unixtime = os.path.getmtime(path) + humantime = datetime.fromtimestamp(unixtime) + # format properly + humantime = str(humantime) + outputtime = "" + for chr in humantime: + if chr != "-" and chr != " " and chr != ":": + outputtime += chr + return outputtime + # get a list, returns a sorted list def alphaSorter(seq): def stripter(s, goodchrs):