sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add file transfer speed

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@522 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2007-09-26 08:28:29 +00:00
parent 32f3c024a6
commit e690fab8c5
5 changed files with 74 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/equoTools.py
+9 -2
View File
@@ -709,7 +709,7 @@ def generateDependencyTree(atomInfo, emptydeps = False, deepdeps = False):
atom = atomMatch(dep)
deps = getDependencies(atom)
if (not emptydeps):
deps, xxx = filterSatisfiedDependencies(deps, deepdeps = deepdeps) #FIXME add deepdeps
deps, xxx = filterSatisfiedDependencies(deps, deepdeps = deepdeps)
unsatisfiedDeps += deps[:]
closeClientDatabase(clientDbconn)
@@ -1002,6 +1002,8 @@ def fetchFileOnMirrors(repository, filename, digest = False):
print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - file not available on this mirror.")
elif rc == -2:
print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - wrong checksum.")
elif rc == -3:
print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - not found.")
else:
print_info(red(" ## ")+mirrorCountText+blue("Error downloading from: ")+red(spliturl(url)[1])+" - unknown reason.")
try:
@@ -1026,6 +1028,8 @@ def fetchFile(url, digest = False):
fetchChecksum = downloadData(url,filepath)
except:
return -1
if fetchChecksum == -3:
return -3
if (digest != False):
#print digest+" <--> "+fetchChecksum
if (fetchChecksum != digest):
@@ -2327,7 +2331,10 @@ def stepExecutor(step,infoDict, clientDbconn = None):
print_info(red(" ## ")+blue("Installing package: ")+red(os.path.basename(infoDict['download'])))
output = installFile(infoDict, clientDbconn)
if output != 0:
errormsg = red("An error occured while trying to install the package. Check if you have enough disk space on your hard disk. Error "+str(output))
if output == 512:
errormsg = red("You are running out of disk space. I bet, you're probably Michele. Error 512")
else:
errormsg = red("An error occured while trying to install the package. Check if your hard disk is healthy. Error "+str(output))
print_error(errormsg)
elif step == "remove":
libraries/activatorTools.py
+8 -8
View File
@@ -1039,14 +1039,6 @@ def uploadDatabase(uris):
# remove the gzip
os.remove(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasefilegzip'])
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
# uploading revision file
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True)
if (rc == True):
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed."))
else:
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# generate digest
hexdigest = md5sum(etpConst['etpdatabasefilepath'])
f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],"w")
@@ -1064,6 +1056,14 @@ def uploadDatabase(uris):
entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.")
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
# uploading revision file
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True)
if (rc == True):
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed."))
else:
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# close connection
ftp.closeConnection()
# unlock database
libraries/entropyConstants.py
+7 -19
View File
@@ -429,6 +429,13 @@ etpHandlers = {
'errorsend': "http://svn.sabayonlinux.org/entropy/handlers/error_report.php?arch="+ETP_ARCH_CONST+"&stacktrace=",
}
### file transfer settings
etpFileTransfer = {
'datatransfer': 0,
'oldgather': 0,
'gather': 0,
'transferpollingtime': float(1)/2 # 0.5secs = 2Hz
}
# Create paths
if not os.path.isdir(etpConst['entropyworkdir']):
@@ -576,27 +583,8 @@ dbOR = "|or|"
dbKEYWORDS = "KEYWORDS"
dbCONTENTS = "CONTENTS"
dbCOUNTER = "COUNTER"
dbPORTAGE_ELOG_OPTS = 'PORTAGE_ELOG_CLASSES="warn info log" PORTAGE_ELOG_SYSTEM="save" PORT_LOGDIR="'+etpConst['logdir']+'"'
# Portage variables reference
# vdbVARIABLE --> $VARIABLE
vdbPORTDIR = "PORTDIR"
vdbPORTDIR_OVERLAY = "PORTDIR_OVERLAY"
# Portage & misc commands
cdbEMERGE = "emerge"
cdbRunEmerge = vdbPORTDIR+"='"+etpConst['portagetreedir']+"' "+vdbPORTDIR_OVERLAY+"='"+etpConst['overlays']+"' "+cdbEMERGE
cdbStartDistcc = "/etc/init.d/distccd start --nodeps"
cdbStopDistcc = "/etc/init.d/distccd stop --nodeps"
cdbStatusDistcc = "/etc/init.d/distccd status"
# Portage options
odbBuild = " -b "
odbNodeps = " --nodeps "
if (commands.getoutput("q -V").find("portage-utils") != -1):
pFindLibrary = "qfile -qC "
pFindLibraryXT = "qfile -qeC "
else:
pFindLibrary = "equery belongs -n "
pFindLibraryXT = "equery belongs -en "
libraries/entropyTools.py
+17 -1
View File
@@ -29,6 +29,7 @@ import sys
import random
import commands
import urlparse
import threading, time
# Instantiate the databaseStatus:
import databaseTools
@@ -56,6 +57,22 @@ def isRoot():
return True
return False
class TimeScheduled(threading.Thread):
def __init__(self, function, delay):
threading.Thread.__init__(self)
self.function = function
self.delay = delay
def run(self):
self.alive = 1
while self.alive:
self.function()
try:
time.sleep(self.delay)
except:
pass
def kill(self):
self.alive = 0
def applicationLockCheck(option = None, gentle = False):
if (etpConst['applicationlock']):
print_error(red("Another instance of Equo is running. Action: ")+bold(str(option))+red(" denied."))
@@ -70,7 +87,6 @@ def getRandomNumber():
return int(str(random.random())[2:7])
def countdown(secs=5,what="Counting...", back = False):
import time
if secs:
if back:
sys.stdout.write(what)
libraries/remoteTools.py
+33 -8
View File
@@ -67,14 +67,23 @@ def getRemotePackageChecksum(serverName,filename, branch):
# HTTP/FTP equo/download functions
###################################################
def downloadData(url,pathToSave, bufferSize = 8192, checksum = True):
def downloadData(url,pathToSave, bufferSize = 8192, checksum = True, showSpeed = True):
remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadFile: called.")
# start scheduler
if (showSpeed):
etpFileTransfer['datatransfer'] = 0
etpFileTransfer['oldgather'] = 0
etpFileTransfer['gather'] = 0
speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransfer['transferpollingtime'])
speedUpdater.start()
rc = "-1"
try:
remotefile = urllib2.urlopen(url)
except Exception, e:
remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"downloadFile: Exception caught for: "+str(url)+" -> "+str(e))
return "-3"
rc = "-3"
try:
maxsize = remotefile.headers.get("content-length")
except:
@@ -84,16 +93,21 @@ def downloadData(url,pathToSave, bufferSize = 8192, checksum = True):
rsx = "x"
while rsx != '':
rsx = remotefile.read(bufferSize)
__downloadFileCommitData(localfile,rsx,maxsize = maxsize)
__downloadFileCommitData(localfile, rsx, maxsize = maxsize, showSpeed = showSpeed)
localfile.flush()
localfile.close()
#print_info("",back = True)
if checksum:
# return digest
return entropyTools.md5sum(pathToSave)
rc = entropyTools.md5sum(pathToSave)
else:
# return -2
return "-2"
rc = "-2"
if (showSpeed):
speedUpdater.kill()
return rc
# Get the content of an online page
# @returns content: if the file exists
@@ -134,7 +148,7 @@ def reportApplicationError(errorstring):
###################################################
# HTTP/FTP equo INTERNAL FUNCTIONS
###################################################
def __downloadFileCommitData(f, buf, output = True, maxsize = 0):
def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = True):
# writing file buffer
f.write(buf)
# update progress
@@ -160,10 +174,21 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0):
diffbarsize = barsize-curbarsize
for y in range(diffbarsize):
bartext += " "
bartext += "]"
if (showSpeed):
etpFileTransfer['gather'] = f.tell()
bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransfer['datatransfer']))+"/sec"
else:
bartext += "]"
average = str(average)
if len(average) < 2:
average = " "+average
currentText += " "+average+"% "+bartext
currentText += " <-> "+average+"% "+bartext
# print !
print_info(currentText,back = True)
def __updateSpeedInfo():
diff = etpFileTransfer['gather'] - etpFileTransfer['oldgather']
# we have the diff size
etpFileTransfer['datatransfer'] = diff / etpFileTransfer['transferpollingtime']
etpFileTransfer['oldgather'] = etpFileTransfer['gather']