From 898bd4aeed6fd8fc002e8ded073470e4364bc55a Mon Sep 17 00:00:00 2001 From: lxnay Date: Thu, 12 Mar 2009 23:49:19 +0000 Subject: [PATCH] Entropy/TODO: - update TODO Entropy/Server: - merge server config files into one (server.conf), reduce complexity Entropy/Transceivers: - FtpInterface add ETA and current up/down speed information git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@3157 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- conf/activator.conf | 12 --- conf/reagent.conf | 49 ----------- conf/server.conf.example | 46 ++++++++++ docs/TODO | 3 - libraries/entropy/const.py | 100 +++++++++------------- libraries/entropy/qa.py | 4 +- libraries/entropy/transceivers.py | 138 ++++++++++++++++++++++-------- 7 files changed, 192 insertions(+), 160 deletions(-) delete mode 100644 conf/activator.conf delete mode 100644 conf/reagent.conf diff --git a/conf/activator.conf b/conf/activator.conf deleted file mode 100644 index 1caea40a7..000000000 --- a/conf/activator.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Project Entropy 1.0 activator configuration file - -# Database format: -# bz2 or gz -database-format|bz2 - -# Log level -# 0: No Logging -# 1: Normal Logging -# 2: Verbose Logging -loglevel|1 - diff --git a/conf/reagent.conf b/conf/reagent.conf deleted file mode 100644 index 58a312a5d..000000000 --- a/conf/reagent.conf +++ /dev/null @@ -1,49 +0,0 @@ -# Project Entropy 1.0 reagent configuration file - -# Log level -# 0: No Logging -# 1: Normal Logging -# 2: Verbose Logging -loglevel|1 - -# Online RSS support to track changes of the things happened on the database. -# Here are basic settings, like the ability to enable/disable this feature or set the name of the feed. -# Feed will be uploaded on the same packages.db.[bz2|gz] directory -# -# syntax for rss-feed: -# rss-feed| -# -rss-feed|enable - -# syntax for rss-name: -# rss-name| -# -rss-name|packages.rss - -# This option allows to specify a base URL for the and RSS entry. -# If you have the entropy web portal online, just add its path to index.py which -# will be wrapped to add ?search=... -# syntax for rss-base-url: -# rss-base-url| -# -rss-base-url|http://packages.sabayonlinux.org/ -# This is just the website url that will be added to the RSS -rss-website-url|http://www.sabayonlinux.org/ - -# Maximum RSS entries, oldest will be dropped -# syntax for max-rss-entries: -# max-rss-entries| -# -max-rss-entries|10000 - -# Maximum RSS entries for the light version, oldest will be dropped -# syntax for max-rss-light-entries: -# max-rss-light-entries| -# -# max-rss-light-entries|100 - -# RSS Editor E-mail (corresponding to managingEditor tag) -# syntax for max-rss-entries: -# managing-editor| -# -managing-editor| \ No newline at end of file diff --git a/conf/server.conf.example b/conf/server.conf.example index 6e542f9b0..c7782b3ea 100644 --- a/conf/server.conf.example +++ b/conf/server.conf.example @@ -44,3 +44,49 @@ officialserverrepositoryid|sabayonlinux.org #| repository|sabayonlinux.org|Sabayon Linux Official Repository|ftp://user:pass@111.111.111.111/|http://svn.sabayonlinux.org/entropy|svn.sabayonlinux.org#1026,1027 # # repository| + +# Database format used by EAPI1 packages: +# bz2 or gz +database-format|bz2 + +# Online RSS support to track changes of the things happened on the database. +# Here are basic settings, like the ability to enable/disable this feature or set the name of the feed. +# Feed will be uploaded on the same packages.db.[bz2|gz] directory +# +# syntax for rss-feed: +# rss-feed| +# +rss-feed|enable + +# syntax for rss-name: +# rss-name| +# +rss-name|packages.rss + +# This option allows to specify a base URL for the and RSS entry. +# If you have the entropy web portal online, just add its path to index.py which +# will be wrapped to add ?search=... +# syntax for rss-base-url: +# rss-base-url| +# +rss-base-url|http://packages.sabayonlinux.org/ +# This is just the website url that will be added to the RSS +rss-website-url|http://www.sabayonlinux.org/ + +# Maximum RSS entries, oldest will be dropped +# syntax for max-rss-entries: +# max-rss-entries| +# +max-rss-entries|10000 + +# Maximum RSS entries for the light version, oldest will be dropped +# syntax for max-rss-light-entries: +# max-rss-light-entries| +# +# max-rss-light-entries|100 + +# RSS Editor E-mail (corresponding to managingEditor tag) +# syntax for max-rss-entries: +# managing-editor| +# +managing-editor| \ No newline at end of file diff --git a/docs/TODO b/docs/TODO index 848745fb8..534346108 100644 --- a/docs/TODO +++ b/docs/TODO @@ -2,9 +2,6 @@ TODO list Proposed for Entropy 1.0 (before and after) (requires API changes, perhaps): - - merge all the server config files to server.conf? - - iron out new FtpInterface - - FtpInterface (upload ETA) - Entropy codebase => camelCase to underscore - Spritz codebase refactoring (fileupdates: show changes button using internal viewer) - Add all button doesn't effectively add all the packages to the queue or diff --git a/libraries/entropy/const.py b/libraries/entropy/const.py index 07c863d9e..80a555de9 100644 --- a/libraries/entropy/const.py +++ b/libraries/entropy/const.py @@ -164,9 +164,7 @@ def initConfig_entropyConstants(rootdir): const_configureLockPaths() initConfig_clientConstants() # server stuff - const_readReagentSettings() const_readServerSettings() - const_readActivatorSettings() # reflow back settings etpConst.update(backed_up_settings) @@ -233,9 +231,7 @@ def const_defaultSettings(rootdir): 'confsetsdirname': ETP_SETSDIRNAME, # just the dirname 'entropyconf': ETP_CONF_DIR+"/entropy.conf", # entropy.conf file 'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file - 'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file 'serverconf': ETP_CONF_DIR+"/server.conf", # server.conf file (generic server side settings) - 'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file 'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file 'socketconf': ETP_CONF_DIR+"/socket.conf", # socket.conf file 'packagesrelativepath': "packages/"+ETP_ARCH_CONST+"/", # user by client interfaces @@ -981,58 +977,6 @@ def const_configureLockPaths(): } -def const_readActivatorSettings(): - - if os.path.isfile(etpConst['activatorconf']): - - f = open(etpConst['activatorconf'],"r") - actconffile = f.readlines() - f.close() - for line in actconffile: - line = line.strip() - if line.startswith("database-format|") and (len(line.split("database-format|")) == 2): - fmt = line.split("database-format|")[1] - if fmt in etpConst['etpdatabasesupportedcformats']: - etpConst['etpdatabasefileformat'] = fmt - -def const_readReagentSettings(): - - if (os.path.isfile(etpConst['reagentconf'])): - f = open(etpConst['reagentconf'],"r") - reagentconf = f.readlines() - f.close() - for line in reagentconf: - - if line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): - feed = line.split("rss-feed|")[1] - if feed in ("enable","enabled","true","1"): - etpConst['rss-feed'] = True - elif feed in ("disable","disabled","false","0","no",): - etpConst['rss-feed'] = False - elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2): - feedname = line.split("rss-name|")[1].strip() - etpConst['rss-name'] = feedname - elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2): - etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip() - if not etpConst['rss-base-url'][-1] == "/": - etpConst['rss-base-url'] += "/" - elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2): - etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip() - elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2): - etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip() - elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2): - try: - entries = int(line.split("max-rss-entries|")[1].strip()) - etpConst['rss-max-entries'] = entries - except ValueError: - pass - elif line.startswith("max-rss-light-entries|") and (len(line.split("max-rss-light-entries|")) == 2): - try: - entries = int(line.split("max-rss-light-entries|")[1].strip()) - etpConst['rss-light-max-entries'] = entries - except ValueError: - pass - def const_readServerSettings(): if not os.access(etpConst['serverconf'],os.R_OK): @@ -1041,7 +985,7 @@ def const_readServerSettings(): etpConst['server_repositories'].clear() f = open(etpConst['serverconf'],"r") - serverconf = f.readlines() + serverconf = [x.strip() for x in f.readlines()] f.close() for line in serverconf: @@ -1067,7 +1011,6 @@ def const_readServerSettings(): pass elif line.startswith("repository|") and (len(line.split("|")) in [5,6]): - repoid, repodata = const_extractServerRepositoryParameters(line) if repoid in etpConst['server_repositories']: # just update mirrors @@ -1075,6 +1018,47 @@ def const_readServerSettings(): else: etpConst['server_repositories'][repoid] = repodata.copy() + elif line.startswith("database-format|") and (len(line.split("database-format|")) == 2): + fmt = line.split("database-format|")[1] + if fmt in etpConst['etpdatabasesupportedcformats']: + etpConst['etpdatabasefileformat'] = fmt + + elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): + feed = line.split("rss-feed|")[1] + if feed in ("enable","enabled","true","1"): + etpConst['rss-feed'] = True + elif feed in ("disable","disabled","false","0","no",): + etpConst['rss-feed'] = False + + elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2): + feedname = line.split("rss-name|")[1].strip() + etpConst['rss-name'] = feedname + + elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2): + etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip() + if not etpConst['rss-base-url'][-1] == "/": + etpConst['rss-base-url'] += "/" + + elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2): + etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip() + + elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2): + etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip() + + elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2): + try: + entries = int(line.split("max-rss-entries|")[1].strip()) + etpConst['rss-max-entries'] = entries + except (ValueError,IndexError,): + pass + + elif line.startswith("max-rss-light-entries|") and (len(line.split("max-rss-light-entries|")) == 2): + try: + entries = int(line.split("max-rss-light-entries|")[1].strip()) + etpConst['rss-light-max-entries'] = entries + except (ValueError,IndexError,): + pass + const_configureServerRepoPaths() def const_extractServerRepositoryParameters(repostring): diff --git a/libraries/entropy/qa.py b/libraries/entropy/qa.py index e16a06874..1790e1a02 100644 --- a/libraries/entropy/qa.py +++ b/libraries/entropy/qa.py @@ -128,7 +128,9 @@ class QAInterface: return broken - def scan_missing_dependencies(self, idpackages, dbconn, ask = True, self_check = False, repo = etpConst['officialrepositoryid'], black_list = None, black_list_adder = None): + def scan_missing_dependencies(self, idpackages, dbconn, ask = True, + self_check = False, repo = etpConst['officialrepositoryid'], + black_list = None, black_list_adder = None): if not isinstance(black_list,set): black_list = set() diff --git a/libraries/entropy/transceivers.py b/libraries/entropy/transceivers.py index 4b1b686d9..b7ab170c7 100644 --- a/libraries/entropy/transceivers.py +++ b/libraries/entropy/transceivers.py @@ -589,11 +589,10 @@ class FtpInterface: self.socket, self.ftplib, self.entropyTools = socket, ftplib, entropyTools self.Entropy = OutputInterface self.__verbose = verbose - self.__oldprogress = 0.0 - self.__filesize = 0 - self.__filekbcount = 0 + self.__init_vars() self.socket.setdefaulttimeout(60) self.__ftpuri = ftpuri + self.__speed_updater = None self.__currentdir = '.' self.__ftphost = self.entropyTools.extractFTPHostFromUri(self.__ftpuri) self.__ftpuser, self.__ftppassword, self.__ftpport, self.__ftpdir = self.entropyTools.extract_ftp_data(ftpuri) @@ -636,6 +635,17 @@ class FtpInterface: ) self.set_cwd(self.__ftpdir, dodir = True) + def __init_vars(self): + self.__oldprogress = 0.0 + self.__filesize = 0 + self.__filekbcount = 0 + self.__transfersize = 0 + self.__startingposition = 0 + self.__elapsed = 0.0 + self.__time_remaining_secs = 0 + self.__time_remaining = "(%s)" % (_("infinite"),) + self.__transferpollingtime = float(1)/4 + def set_basedir(self): return self.set_cwd(self.__ftpdir) @@ -780,14 +790,18 @@ class FtpInterface: return "226" return rc - self.__oldprogress = 0.0 - def up_file_up_progress(buf): self.updateProgress(len(buf)) - for i in range(10): # ten tries - filename = file.split("/")[len(file.split("/"))-1] + tries = 0 + while tries < 10: + + tries += 1 + filename = os.path.basename(file) + self.__init_vars() + self.__start_speed_counter() try: + with open(file,"r") as f: self.__filesize = round(float(self.entropyTools.get_file_size(file))/1024,1) @@ -809,13 +823,13 @@ class FtpInterface: return False except Exception, e: # connection reset by peer + self.entropyTools.printTraceback() - self.Entropy.updateProgress(" ", importance = 0, type = "info") mytxt = red("%s: %s, %s... #%s") % ( _("Upload issue"), e, _("retrying"), - i+1, + tries+1, ) self.Entropy.updateProgress( mytxt, @@ -829,9 +843,10 @@ class FtpInterface: if self.is_file_available(filename+".tmp"): self.delete_file(filename+".tmp") - def download_file(self, filename, downloaddir, ascii = False): + finally: + self.__stop_speed_counter() - self.__oldprogress = 0.0 + def download_file(self, filename, downloaddir, ascii = False): def df_up(buf): # writing file buffer @@ -852,30 +867,56 @@ class FtpInterface: percent = True ) - # look if the file exist - if self.is_file_available(filename): - self.__filekbcount = 0 - # get the file size - self.__filesize = self.get_file_size_compat(filename) - if (self.__filesize): - self.__filesize = round(float(int(self.__filesize))/1024,1) - if (self.__filesize == 0): - self.__filesize = 1 - else: - self.__filesize = 0 - if not ascii: - f = open(downloaddir+"/"+filename,"wb") - rc = self.__ftpconn.retrbinary('RETR '+filename, df_up, 1024) - else: - f = open(downloaddir+"/"+filename,"w") - rc = self.__ftpconn.retrlines('RETR '+filename, f.write) - f.flush() - f.close() - if rc.find("226") != -1: # upload complete - return True - return False - else: - return None + tries = 10 + while tries: + tries -= 1 + + self.__init_vars() + self.__start_speed_counter() + try: + + # look if the file exist + if self.is_file_available(filename): + self.__filekbcount = 0 + # get the file size + self.__filesize = self.get_file_size_compat(filename) + if (self.__filesize): + self.__filesize = round(float(int(self.__filesize))/1024,1) + if (self.__filesize == 0): + self.__filesize = 1 + else: + self.__filesize = 0 + if not ascii: + f = open(downloaddir+"/"+filename,"wb") + rc = self.__ftpconn.retrbinary('RETR '+filename, df_up, 1024) + else: + f = open(downloaddir+"/"+filename,"w") + rc = self.__ftpconn.retrlines('RETR '+filename, f.write) + f.flush() + f.close() + if rc.find("226") != -1: # upload complete + return True + return False + + except Exception, e: # connection reset by peer + + self.entropyTools.printTraceback() + mytxt = red("%s: %s, %s... #%s") % ( + _("Download issue"), + e, + _("retrying"), + i+1, + ) + self.Entropy.updateProgress( + mytxt, + importance = 1, + type = "warning", + header = " " + ) + self.reconnect_host() # reconnect + + finally: + self.__stop_speed_counter() # also used to move files def rename_file(self, fromfile, tofile): @@ -911,9 +952,31 @@ class FtpInterface: # timeout, who cares! pass + def __start_speed_counter(self): + self.__speed_updater = TimeScheduled( + self.__transferpollingtime, + self.__update_speed, + ) + self.__speed_updater.start() + + def __stop_speed_counter(self): + if self.__speed_updater != None: + self.__speed_updater.kill() + + def __update_speed(self): + self.__elapsed += self.__transferpollingtime + # we have the diff size + self.__datatransfer = (self.__filesize-self.__startingposition) / self.__elapsed + try: + self.__time_remaining_secs = int(round((int(round(self.__filesize*1024,0))-int(round(self.__transfersize,0)))/self.__datatransfer,0)) + self.__time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.__time_remaining_secs) + except: + self.__time_remaining = "(%s)" % (_("infinite"),) + def updateProgress(self, buf_len): # get the buffer size self.__filekbcount += float(buf_len)/1024 + self.__transfersize += buf_len # create percentage myUploadPercentage = 100.0 if self.__filesize >= 1: @@ -926,10 +989,11 @@ class FtpInterface: myUploadPercentage = str(myUploadPercentage)+"%" # create text - mytxt = _("Upload status") + mytxt = _("Transfer status") currentText = brown(" <-> %s: " % (mytxt,)) + \ darkgreen(str(myUploadSize)) + "/" + red(str(self.__filesize)) + " kB " + \ - brown("[") + str(myUploadPercentage) + brown("]") + brown("[") + str(myUploadPercentage) + brown("]") + " " + self.__time_remaining + \ + self.entropyTools.bytesIntoHuman(self.__datatransfer) + "/"+_("sec") # WARN: re-enabled updateProgress, this may cause slowdowns # print_info(currentText, back = True) self.Entropy.updateProgress(currentText, back = True)