diff --git a/TODO b/TODO index cae98d80b..8427fe377 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,7 @@ TODO list: - - move rssTools to entropy.py - GUI - server part rewrite ala EquoInterface - Community repositories - - move smartTools to entropy.py ? - find a way to better handle real smartapps deps - add deptest check for every reagent run - implement client db sanity check diff --git a/client/equo b/client/equo index 6a99e9bf1..ed44759c0 100644 --- a/client/equo +++ b/client/equo @@ -252,8 +252,8 @@ try: elif (options[0] == "smart"): rc = 0 if len(options) > 1: - import smartTools - rc = smartTools.smart(options[1:]) + import text_smart + rc = text_smart.smart(options[1:]) elif (options[0] == "conf"): import text_configuration diff --git a/libraries/smartTools.py b/client/text_smart.py similarity index 98% rename from libraries/smartTools.py rename to client/text_smart.py index 952da312e..6ca3b7321 100644 --- a/libraries/smartTools.py +++ b/client/text_smart.py @@ -19,7 +19,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ''' -import os import shutil from entropyConstants import * from outputTools import * @@ -95,11 +94,10 @@ def QuickpkgHandler(mypackages, savedir = None): # print the list if (not etpUi['quiet']) or (etpUi['ask']): print_info(darkgreen(" * ")+red("This is the list of the packages that would be quickpkg'd:")) pkgInfo = {} - pkgData = {} for pkg in packages: atom = text_ui.Equo.clientDbconn.retrieveAtom(pkg[0]) - pkgInfo[pkg] = atom - pkgData[pkg] = text_ui.Equo.clientDbconn.getPackageData(pkg[0]) + pkgInfo[pkg]['atom'] = atom + pkgInfo[pkg]['idpackage'] = pkg[0] print_info(brown("\t[")+red("from:")+bold("installed")+brown("]")+" - "+atom) if (not etpUi['quiet']) or (etpUi['ask']): @@ -108,8 +106,9 @@ def QuickpkgHandler(mypackages, savedir = None): return 0 for pkg in packages: - if not etpUi['quiet']: print_info(brown(" * ")+red("Compressing: ")+darkgreen(pkgInfo[pkg])) - resultfile = text_ui.Equo.entropyTools.quickpkg(pkgdata = pkgData[pkg], dirpath = savedir) + if not etpUi['quiet']: print_info(brown(" * ")+red("Compressing: ")+darkgreen(pkgInfo[pkg]['atom'])) + pkgdata = text_ui.Equo.clientDbconn.getPackageData(pkgInfo[pkg]['idpackage']) + resultfile = text_ui.Equo.entropyTools.quickpkg(pkgdata = pkgdata, dirpath = savedir) if resultfile == None: if not etpUi['quiet']: print_error(darkred(" * ")+red("Error creating package for: ")+bold(pkgInfo[pkg])+darkred(". Cannot continue.")) return 3 diff --git a/client/text_ui.py b/client/text_ui.py index 83fa09309..181808dab 100644 --- a/client/text_ui.py +++ b/client/text_ui.py @@ -438,6 +438,16 @@ def installPackages(packages = [], atomsdata = [], deps = True, emptydeps = Fals if dl < 0: pkgsize = dbconn.retrieveSize(packageInfo[0]) downloadSize += int(pkgsize) + else: + try: + f = open(etpConst['entropyworkdir']+"/"+pkgfile,"r") + f.seek(0,2) + currsize = f.tell() + pkgsize = dbconn.retrieveSize(packageInfo[0]) + downloadSize += int(pkgsize)-int(currsize) + f.close() + except: + pass # get installed package data installedVer = '0' diff --git a/libraries/entropy.py b/libraries/entropy.py index b49ccf638..cb67ba7cc 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -23,6 +23,7 @@ import shutil import commands import urllib2 +import socket import random from entropyConstants import * from outputTools import * @@ -1369,37 +1370,39 @@ class EquoInterface(TextInterface): @input package: url -> HTTP/FTP url, digest -> md5 hash of the file @output: -1 = download error (cannot find the file), -2 = digest error, 0 = all fine ''' - def fetch_file(self, url, digest = None): + def fetch_file(self, url, digest = None, resume = True): # remove old filename = os.path.basename(url) filepath = etpConst['packagesbindir']+"/"+etpConst['branch']+"/"+filename - if os.path.exists(filepath): - os.remove(filepath) - + # load class - fetchConn = self.urlFetcher(url, filepath) + fetchConn = self.urlFetcher(url, filepath, resume = resume) fetchConn.progress = self.progress # start to download data_transfer = 0 + resumed = False try: fetchChecksum = fetchConn.download() data_transfer = fetchConn.datatransfer + resumed = fetchConn.resumed except KeyboardInterrupt: - return -4, data_transfer + return -4, data_transfer, resumed + except NameError: + raise except: - return -1, data_transfer + return -1, data_transfer, resumed if fetchChecksum == "-3": - return -3, data_transfer + return -3, data_transfer, resumed del fetchConn if (digest): if (fetchChecksum != digest): # not properly downloaded - return -2, data_transfer + return -2, data_transfer, resumed else: - return 0, data_transfer - return 0, data_transfer + return 0, data_transfer, resumed + return 0, data_transfer, resumed def add_failing_mirror(self, mirrorname,increment = 1): item = etpRemoteFailures.get(mirrorname) @@ -1464,48 +1467,75 @@ class EquoInterface(TextInterface): remaining.remove(uri) continue - # now fetch the new one - self.updateProgress( - mirrorCountText+blue("Downloading from: ")+red(self.entropyTools.spliturl(url)[1]), - importance = 1, - type = "warning", - header = red(" ## ") - ) - rc, data_transfer = self.fetch_file(url, digest) - if rc == 0: - self.updateProgress( - mirrorCountText+blue("Successfully downloaded from: ")+red(self.entropyTools.spliturl(url)[1])+blue(" at "+str(self.entropyTools.bytesIntoHuman(data_transfer))+"/sec"), - importance = 1, - type = "info", - header = red(" ## ") - ) - return 0 - else: - error_message = mirrorCountText+blue("Error downloading from: ")+red(self.entropyTools.spliturl(url)[1]) - # something bad happened - if rc == -1: - error_message += " - file not available on this mirror." - elif rc == -2: - self.add_failing_mirror(uri,1) - error_message += " - wrong checksum." - elif rc == -3: - self.add_failing_mirror(uri,2) - error_message += " - not found." - elif rc == -4: - error_message += blue(" - discarded download.") - else: - self.add_failing_mirror(uri, 5) - error_message += " - unknown reason." - self.updateProgress( - error_message, - importance = 1, - type = "warning", - header = red(" ## ") - ) - if rc == -4: # user discarded fetch - return 1 - remaining.remove(uri) + do_resume = True + while 1: + try: + # now fetch the new one + self.updateProgress( + mirrorCountText+blue("Downloading from: ")+red(self.entropyTools.spliturl(url)[1]), + importance = 1, + type = "warning", + header = red(" ## ") + ) + rc, data_transfer, resumed = self.fetch_file(url, digest, do_resume) + if rc == 0: + self.updateProgress( + mirrorCountText+blue("Successfully downloaded from: ")+red(self.entropyTools.spliturl(url)[1])+blue(" at "+str(self.entropyTools.bytesIntoHuman(data_transfer))+"/sec"), + importance = 1, + type = "info", + header = red(" ## ") + ) + return 0 + elif resumed: + do_resume = False + continue + else: + error_message = mirrorCountText+blue("Error downloading from: ")+red(self.entropyTools.spliturl(url)[1]) + # something bad happened + if rc == -1: + error_message += " - file not available on this mirror." + elif rc == -2: + self.add_failing_mirror(uri,1) + error_message += " - wrong checksum." + elif rc == -3: + self.add_failing_mirror(uri,2) + error_message += " - not found." + elif rc == -4: + error_message += blue(" - discarded download.") + else: + self.add_failing_mirror(uri, 5) + error_message += " - unknown reason." + self.updateProgress( + error_message, + importance = 1, + type = "warning", + header = red(" ## ") + ) + if rc == -4: # user discarded fetch + return 1 + remaining.remove(uri) + break + except KeyboardInterrupt: + break + except: + raise + def quickpkg(self, atomstring, savedir = None): + if savedir == None: + savedir = etpConst['packagestmpdir'] + if not os.path.isdir(etpConst['packagestmpdir']): + os.makedirs(etpConst['packagestmpdir']) + # match package + match = self.clientDbconn.atomMatch(atomstring) + if match[0] == -1: + return -1,None,None + atom = self.clientDbconn.atomMatch(match[0]) + pkgdata = self.clientDbconn.getPackageData(match[0]) + resultfile = self.entropyTools.quickpkg(pkgdata = pkgdata, dirpath = savedir) + if resultfile == None: + return -1,atom,None + else: + return 0,atom,resultfile def Package(self): conn = PackageInterface(EquoInstance = self) @@ -2749,7 +2779,14 @@ class PackageInterface: self.infoDict['steps'].append("checksum") # if file exists, first checksum then fetch if os.path.isfile(os.path.join(etpConst['entropyworkdir'],self.infoDict['download'])): - self.infoDict['steps'].reverse() + # check size first + repo_size = dbconn.retrieveSize(idpackage) + f = open(os.path.join(etpConst['entropyworkdir'],self.infoDict['download']),"r") + f.seek(0,2) + disk_size = f.tell() + f.close() + if repo_size == disk_size: + self.infoDict['steps'].reverse() class FileUpdatesInterface: @@ -3339,13 +3376,11 @@ class FtpInterface: import entropyTools self.entropyTools = entropyTools - import socket - self.socket = socket import ftplib self.ftplib = ftplib # import FTP modules - self.socket.setdefaulttimeout(60) + socket.setdefaulttimeout(60) self.ftpuri = ftpuri self.ftphost = self.entropyTools.extractFTPHostFromUri(self.ftpuri) @@ -3400,7 +3435,7 @@ class FtpInterface: # this can be used in case of exceptions def reconnectHost(self): # import FTP modules - self.socket.setdefaulttimeout(60) + socket.setdefaulttimeout(60) counter = 10 while 1: counter -= 1 @@ -3645,25 +3680,38 @@ class FtpInterface: ''' class urlFetcher: - def __init__(self, url, pathToSave, checksum = True, showSpeed = True): + def __init__(self, url, pathToSave, checksum = True, showSpeed = True, resume = True): self.url = url + self.resume = resume self.url = self.encodeUrl(self.url) self.pathToSave = pathToSave self.checksum = checksum + self.resumed = False self.showSpeed = showSpeed self.bufferSize = 8192 self.status = None self.remotefile = None - self.localfile = None self.downloadedsize = 0 self.average = 0 self.remotesize = 0 # transfer status data - self.gather = 0 + self.startingposition = 0 self.datatransfer = 0 + self.time_remaining = "(infinite)" self.elapsed = etpFileTransfer['elapsed'] self.transferpollingtime = etpFileTransfer['transferpollingtime'] + import entropyTools + self.entropyTools = entropyTools + + # resume support + if os.path.isfile(self.pathToSave) and os.access(self.pathToSave,os.R_OK) and self.resume: + self.localfile = open(self.pathToSave,"awb") + self.localfile.seek(0,2) + self.startingposition = int(self.localfile.tell()) + self.resumed = True + else: + self.localfile = open(self.pathToSave,"wb") # setup proxy, doing here because config is dynamic if etpConst['proxy']: @@ -3678,7 +3726,7 @@ class urlFetcher: def download(self): if self.showSpeed: - self.speedUpdater = entropyTools.TimeScheduled( + self.speedUpdater = self.entropyTools.TimeScheduled( self.updateSpeedInfo, self.transferpollingtime ) @@ -3688,11 +3736,27 @@ class urlFetcher: # set timeout socket.setdefaulttimeout(60) - # go download slave! + # get file size if available + try: + self.remotefile = urllib2.urlopen(self.url) + self.remotesize = int(self.remotefile.headers.get("content-length")) + self.remotefile.close() + except: + pass # handle user stupidity try: - self.remotefile = urllib2.urlopen(self.url) + request = self.url + if ((self.startingposition > 0) and (self.remotesize > 0)) and (self.startingposition < self.remotesize): + try: + request = urllib2.Request(self.url, headers = { "Range" : "bytes=" + str(self.startingposition) + "-" + str(self.remotesize) }) + except: + pass + elif (self.startingposition == self.remotesize): + return self.prepare_return() + else: + self.localfile = open(self.pathToSave,"wb") + self.remotefile = urllib2.urlopen(request) except KeyboardInterrupt: self.close() raise @@ -3701,16 +3765,9 @@ class urlFetcher: self.status = "-3" return self.status - # get file size if available - try: - self.remotesize = self.remotefile.headers.get("content-length") - except: - pass - if self.remotesize > 0: self.remotesize = float(int(self.remotesize))/1024 - self.localfile = open(self.pathToSave,"w") rsx = "x" while rsx != '': rsx = self.remotefile.read(self.bufferSize) @@ -3723,8 +3780,12 @@ class urlFetcher: # kill thread self.close() + return self.prepare_return() + + + def prepare_return(self): if self.checksum: - self.status = entropyTools.md5sum(self.pathToSave) + self.status = self.entropyTools.md5sum(self.pathToSave) return self.status else: self.status = "-2" @@ -3741,7 +3802,7 @@ class urlFetcher: # reimplemented from TextInterface def updateProgress(self): - currentText = darkred(" <-> Downloading: ")+darkgreen(str(round(float(self.downloadedsize)/1024,1)))+"/"+red(str(round(self.remotesize,1)))+" kB" + currentText = darkred(" <-> Downloading: ")+darkgreen(str(round(float(self.downloadedsize)/1024,1)))+"/"+red(str(round(self.remotesize,1))) + " kB" # create progress bar barsize = 10 bartext = "[" @@ -3757,8 +3818,7 @@ class urlFetcher: for y in range(diffbarsize): bartext += " " if (self.showSpeed): - self.gather = self.downloadedsize - bartext += "] => "+str(entropyTools.bytesIntoHuman(self.datatransfer))+"/sec" + bartext += "] => "+str(self.entropyTools.bytesIntoHuman(self.datatransfer))+"/sec ~ ETA: "+str(self.time_remaining) else: bartext += "]" average = str(self.average) @@ -3776,7 +3836,12 @@ class urlFetcher: def updateSpeedInfo(self): self.elapsed += self.transferpollingtime # we have the diff size - self.datatransfer = self.gather / self.elapsed + self.datatransfer = (self.downloadedsize-self.startingposition) / self.elapsed + try: + self.time_remaining = int(round((int(round(self.remotesize*1024,0))-int(round(self.downloadedsize,0)))/self.datatransfer,0)) + self.time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.time_remaining) + except: + self.time_remaining = "(infinite)" class rssFeed: @@ -3976,4 +4041,4 @@ class rssFeed: f = open(self.file,"w") f.writelines(doc.toprettyxml(indent=" ")) f.flush() - f.close() \ No newline at end of file + f.close() diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 3207482cc..f8e670c76 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -169,11 +169,11 @@ def report_application_error(errorstring): urllib2.install_opener(opener) xfile = urllib2.urlopen(url) result = xfile.readlines() - socket.setdefaulttimeout(2) + socket.setdefaulttimeout(2) return result except: - socket.setdefaulttimeout(2) - return False + socket.setdefaulttimeout(2) + return False def ebeep(count = 5): for x in range(count): @@ -181,12 +181,12 @@ def ebeep(count = 5): def applicationLockCheck(option = None, gentle = False): if (etpConst['applicationlock']): - print_error(red("Another instance of Equo is running. Action: ")+bold(str(option))+red(" denied.")) - print_error(red("If I am lying (maybe). Please remove ")+bold(etpConst['pidfile'])) - if (not gentle): - sys.exit(10) - else: - return True + print_error(red("Another instance of Equo is running. Action: ")+bold(str(option))+red(" denied.")) + print_error(red("If I am lying (maybe). Please remove ")+bold(etpConst['pidfile'])) + if (not gentle): + sys.exit(10) + else: + return True return False def getRandomNumber(): @@ -195,22 +195,22 @@ def getRandomNumber(): def countdown(secs=5,what="Counting...", back = False): if secs: - if back: - sys.stdout.write(red(">> ")+what) - else: - print what + if back: + sys.stdout.write(red(">> ")+what) + else: + print what for i in range(secs)[::-1]: sys.stdout.write(red(str(i+1)+" ")) sys.stdout.flush() - time.sleep(1) + time.sleep(1) def spinner(rotations, interval, message=''): - for x in xrange(rotations): - writechar(message + '|/-\\'[x%4] + '\r') - time.sleep(interval) - writechar(' ') - for i in xrange(len(message)): print ' ', - writechar('\r') + for x in xrange(rotations): + writechar(message + '|/-\\'[x%4] + '\r') + time.sleep(interval) + writechar(' ') + for i in xrange(len(message)): print ' ', + writechar('\r') def md5sum(filepath): import md5 @@ -239,7 +239,7 @@ def md5sum_directory(directory): m.update(block) block = readfile.read(1024) return m.hexdigest() - + def unpackGzip(gzipfilepath): import gzip filepath = gzipfilepath[:-3] # remove .gz @@ -1364,6 +1364,38 @@ def convertUnixTimeToHumanTime(unixtime): humantime = str(datetime.fromtimestamp(unixtime)) return humantime +def convertSecondsToFancyOutput(seconds): + + mysecs = seconds + myminutes = 0 + myhours = 0 + mydays = 0 + + while mysecs >= 60: + mysecs -= 60 + myminutes += 1 + + while myminutes >= 60: + myminutes -= 60 + myhours += 1 + + while myhours >= 24: + myhours -= 24 + mydays += 1 + + output = [] + output.append(str(mysecs)+"s") + if myminutes > 0 or myhours > 0: + output.append(str(myminutes)+"m") + if myhours > 0 or mydays > 0: + output.append(str(myhours)+"h") + if mydays > 0: + output.append(str(mydays)+"d") + output.reverse() + return ':'.join(output) + + + # get a list, returns a sorted list def alphaSorter(seq): def stripter(s, goodchrs):