From e690fab8c56445cfde8c8a18e4790c41439b00f2 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Wed, 26 Sep 2007 08:28:29 +0000 Subject: [PATCH] add file transfer speed git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@522 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- client/equoTools.py | 11 ++++++++-- libraries/activatorTools.py | 16 +++++++------- libraries/entropyConstants.py | 26 ++++++---------------- libraries/entropyTools.py | 18 ++++++++++++++- libraries/remoteTools.py | 41 ++++++++++++++++++++++++++++------- 5 files changed, 74 insertions(+), 38 deletions(-) diff --git a/client/equoTools.py b/client/equoTools.py index 7160fcce8..6b96e321f 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -709,7 +709,7 @@ def generateDependencyTree(atomInfo, emptydeps = False, deepdeps = False): atom = atomMatch(dep) deps = getDependencies(atom) if (not emptydeps): - deps, xxx = filterSatisfiedDependencies(deps, deepdeps = deepdeps) #FIXME add deepdeps + deps, xxx = filterSatisfiedDependencies(deps, deepdeps = deepdeps) unsatisfiedDeps += deps[:] closeClientDatabase(clientDbconn) @@ -1002,6 +1002,8 @@ def fetchFileOnMirrors(repository, filename, digest = False): print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - file not available on this mirror.") elif rc == -2: print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - wrong checksum.") + elif rc == -3: + print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - not found.") else: print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - unknown reason.") try: @@ -1026,6 +1028,8 @@ def fetchFile(url, digest = False): fetchChecksum = downloadData(url,filepath) except: return -1 + if fetchChecksum == -3: + return -3 if (digest != False): #print digest+" <--> "+fetchChecksum if (fetchChecksum != digest): @@ -2327,7 +2331,10 @@ def stepExecutor(step,infoDict, clientDbconn = None): print_info(red(" ## ")+blue("Installing package: ")+red(os.path.basename(infoDict['download']))) output = installFile(infoDict, clientDbconn) if output != 0: - errormsg = red("An error occured while trying to install the package. Check if you have enough disk space on your hard disk. Error "+str(output)) + if output == 512: + errormsg = red("You are running out of disk space. I bet, you're probably Michele. Error 512") + else: + errormsg = red("An error occured while trying to install the package. Check if your hard disk is healthy. Error "+str(output)) print_error(errormsg) elif step == "remove": diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index c1040f1cc..981709ffd 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -1039,14 +1039,6 @@ def uploadDatabase(uris): # remove the gzip os.remove(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasefilegzip']) - print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True) - # uploading revision file - rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True) - if (rc == True): - print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed.")) - else: - print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) - # generate digest hexdigest = md5sum(etpConst['etpdatabasefilepath']) f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],"w") @@ -1064,6 +1056,14 @@ def uploadDatabase(uris): entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.") print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) + print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True) + # uploading revision file + rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True) + if (rc == True): + print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed.")) + else: + print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) + # close connection ftp.closeConnection() # unlock database diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 9f6f9c5b4..b6b57871a 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -429,6 +429,13 @@ etpHandlers = { 'errorsend': "http://svn.sabayonlinux.org/entropy/handlers/error_report.php?arch="+ETP_ARCH_CONST+"&stacktrace=", } +### file transfer settings +etpFileTransfer = { + 'datatransfer': 0, + 'oldgather': 0, + 'gather': 0, + 'transferpollingtime': float(1)/2 # 0.5secs = 2Hz +} # Create paths if not os.path.isdir(etpConst['entropyworkdir']): @@ -576,27 +583,8 @@ dbOR = "|or|" dbKEYWORDS = "KEYWORDS" dbCONTENTS = "CONTENTS" dbCOUNTER = "COUNTER" -dbPORTAGE_ELOG_OPTS = 'PORTAGE_ELOG_CLASSES="warn info log" PORTAGE_ELOG_SYSTEM="save" PORT_LOGDIR="'+etpConst['logdir']+'"' # Portage variables reference # vdbVARIABLE --> $VARIABLE vdbPORTDIR = "PORTDIR" vdbPORTDIR_OVERLAY = "PORTDIR_OVERLAY" - -# Portage & misc commands -cdbEMERGE = "emerge" -cdbRunEmerge = vdbPORTDIR+"='"+etpConst['portagetreedir']+"' "+vdbPORTDIR_OVERLAY+"='"+etpConst['overlays']+"' "+cdbEMERGE -cdbStartDistcc = "/etc/init.d/distccd start --nodeps" -cdbStopDistcc = "/etc/init.d/distccd stop --nodeps" -cdbStatusDistcc = "/etc/init.d/distccd status" - -# Portage options -odbBuild = " -b " -odbNodeps = " --nodeps " - -if (commands.getoutput("q -V").find("portage-utils") != -1): - pFindLibrary = "qfile -qC " - pFindLibraryXT = "qfile -qeC " -else: - pFindLibrary = "equery belongs -n " - pFindLibraryXT = "equery belongs -en " \ No newline at end of file diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 84948da62..ba4563018 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -29,6 +29,7 @@ import sys import random import commands import urlparse +import threading, time # Instantiate the databaseStatus: import databaseTools @@ -56,6 +57,22 @@ def isRoot(): return True return False +class TimeScheduled(threading.Thread): + def __init__(self, function, delay): + threading.Thread.__init__(self) + self.function = function + self.delay = delay + def run(self): + self.alive = 1 + while self.alive: + self.function() + try: + time.sleep(self.delay) + except: + pass + def kill(self): + self.alive = 0 + def applicationLockCheck(option = None, gentle = False): if (etpConst['applicationlock']): print_error(red("Another instance of Equo is running. Action: ")+bold(str(option))+red(" denied.")) @@ -70,7 +87,6 @@ def getRandomNumber(): return int(str(random.random())[2:7]) def countdown(secs=5,what="Counting...", back = False): - import time if secs: if back: sys.stdout.write(what) diff --git a/libraries/remoteTools.py b/libraries/remoteTools.py index a48879225..3ba82ce7b 100644 --- a/libraries/remoteTools.py +++ b/libraries/remoteTools.py @@ -67,14 +67,23 @@ def getRemotePackageChecksum(serverName,filename, branch): # HTTP/FTP equo/download functions ################################################### -def downloadData(url,pathToSave, bufferSize = 8192, checksum = True): +def downloadData(url,pathToSave, bufferSize = 8192, checksum = True, showSpeed = True): remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadFile: called.") + + # start scheduler + if (showSpeed): + etpFileTransfer['datatransfer'] = 0 + etpFileTransfer['oldgather'] = 0 + etpFileTransfer['gather'] = 0 + speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransfer['transferpollingtime']) + speedUpdater.start() + rc = "-1" try: remotefile = urllib2.urlopen(url) except Exception, e: remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"downloadFile: Exception caught for: "+str(url)+" -> "+str(e)) - return "-3" + rc = "-3" try: maxsize = remotefile.headers.get("content-length") except: @@ -84,16 +93,21 @@ def downloadData(url,pathToSave, bufferSize = 8192, checksum = True): rsx = "x" while rsx != '': rsx = remotefile.read(bufferSize) - __downloadFileCommitData(localfile,rsx,maxsize = maxsize) + __downloadFileCommitData(localfile, rsx, maxsize = maxsize, showSpeed = showSpeed) localfile.flush() localfile.close() #print_info("",back = True) if checksum: # return digest - return entropyTools.md5sum(pathToSave) + rc = entropyTools.md5sum(pathToSave) else: # return -2 - return "-2" + rc = "-2" + + if (showSpeed): + speedUpdater.kill() + + return rc # Get the content of an online page # @returns content: if the file exists @@ -134,7 +148,7 @@ def reportApplicationError(errorstring): ################################################### # HTTP/FTP equo INTERNAL FUNCTIONS ################################################### -def __downloadFileCommitData(f, buf, output = True, maxsize = 0): +def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = True): # writing file buffer f.write(buf) # update progress @@ -160,10 +174,21 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0): diffbarsize = barsize-curbarsize for y in range(diffbarsize): bartext += " " - bartext += "]" + if (showSpeed): + etpFileTransfer['gather'] = f.tell() + bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransfer['datatransfer']))+"/sec" + else: + bartext += "]" average = str(average) if len(average) < 2: average = " "+average - currentText += " "+average+"% "+bartext + currentText += " <-> "+average+"% "+bartext # print ! print_info(currentText,back = True) + + +def __updateSpeedInfo(): + diff = etpFileTransfer['gather'] - etpFileTransfer['oldgather'] + # we have the diff size + etpFileTransfer['datatransfer'] = diff / etpFileTransfer['transferpollingtime'] + etpFileTransfer['oldgather'] = etpFileTransfer['gather'] \ No newline at end of file