sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Entropy/TODO:

Browse Source 
- update TODO
Entropy/Server:
- merge server config files into one (server.conf), reduce complexity
Entropy/Transceivers:
- FtpInterface add ETA and current up/down speed information


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@3157 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2009-03-12 23:49:19 +00:00
parent d4d0ecbaec
commit 898bd4aeed
7 changed files with 192 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conf/activator.conf
-12
View File
@@ -1,12 +0,0 @@
# Project Entropy 1.0 activator configuration file
# Database format:
# bz2 or gz
database-format|bz2
# Log level
# 0: No Logging
# 1: Normal Logging
# 2: Verbose Logging
loglevel|1
conf/reagent.conf
-49
View File
@@ -1,49 +0,0 @@
# Project Entropy 1.0 reagent configuration file
# Log level
# 0: No Logging
# 1: Normal Logging
# 2: Verbose Logging
loglevel|1
# Online RSS support to track changes of the things happened on the database.
# Here are basic settings, like the ability to enable/disable this feature or set the name of the feed.
# Feed will be uploaded on the same packages.db.[bz2|gz] directory
#
# syntax for rss-feed:
# rss-feed|<enable/disable>
#
rss-feed|enable
# syntax for rss-name:
# rss-name|<name.rss>
#
rss-name|packages.rss
# This option allows to specify a base URL for the <guid> and <link> RSS entry.
# If you have the entropy web portal online, just add its path to index.py which
# will be wrapped to add ?search=...
# syntax for rss-base-url:
# rss-base-url|<URL>
#
rss-base-url|http://packages.sabayonlinux.org/
# This is just the website url that will be added to the RSS
rss-website-url|http://www.sabayonlinux.org/
# Maximum RSS entries, oldest will be dropped
# syntax for max-rss-entries:
# max-rss-entries|<number>
#
max-rss-entries|10000
# Maximum RSS entries for the light version, oldest will be dropped
# syntax for max-rss-light-entries:
# max-rss-light-entries|<number>
#
# max-rss-light-entries|100
# RSS Editor E-mail (corresponding to managingEditor tag)
# syntax for max-rss-entries:
# managing-editor|<email>
#
managing-editor|
conf/server.conf.example
+46
View File
@@ -44,3 +44,49 @@ officialserverrepositoryid|sabayonlinux.org
#| repository|sabayonlinux.org|Sabayon Linux Official Repository|ftp://user:pass@111.111.111.111/|http://svn.sabayonlinux.org/entropy|svn.sabayonlinux.org#1026,1027
#
# repository|
# Database format used by EAPI1 packages:
# bz2 or gz
database-format|bz2
# Online RSS support to track changes of the things happened on the database.
# Here are basic settings, like the ability to enable/disable this feature or set the name of the feed.
# Feed will be uploaded on the same packages.db.[bz2|gz] directory
#
# syntax for rss-feed:
# rss-feed|<enable/disable>
#
rss-feed|enable
# syntax for rss-name:
# rss-name|<name.rss>
#
rss-name|packages.rss
# This option allows to specify a base URL for the <guid> and <link> RSS entry.
# If you have the entropy web portal online, just add its path to index.py which
# will be wrapped to add ?search=...
# syntax for rss-base-url:
# rss-base-url|<URL>
#
rss-base-url|http://packages.sabayonlinux.org/
# This is just the website url that will be added to the RSS
rss-website-url|http://www.sabayonlinux.org/
# Maximum RSS entries, oldest will be dropped
# syntax for max-rss-entries:
# max-rss-entries|<number>
#
max-rss-entries|10000
# Maximum RSS entries for the light version, oldest will be dropped
# syntax for max-rss-light-entries:
# max-rss-light-entries|<number>
#
# max-rss-light-entries|100
# RSS Editor E-mail (corresponding to managingEditor tag)
# syntax for max-rss-entries:
# managing-editor|<email>
#
managing-editor|
docs/TODO
-3
View File
@@ -2,9 +2,6 @@ TODO list
Proposed for Entropy 1.0 (before and after) (requires API changes, perhaps):
- merge all the server config files to server.conf?
- iron out new FtpInterface
- FtpInterface (upload ETA)
- Entropy codebase => camelCase to underscore
- Spritz codebase refactoring (fileupdates: show changes button using internal viewer)
- Add all button doesn't effectively add all the packages to the queue or
libraries/entropy/const.py
+42 -58
View File
@@ -164,9 +164,7 @@ def initConfig_entropyConstants(rootdir):
const_configureLockPaths()
initConfig_clientConstants()
# server stuff
const_readReagentSettings()
const_readServerSettings()
const_readActivatorSettings()
# reflow back settings
etpConst.update(backed_up_settings)
@@ -233,9 +231,7 @@ def const_defaultSettings(rootdir):
'confsetsdirname': ETP_SETSDIRNAME, # just the dirname
'entropyconf': ETP_CONF_DIR+"/entropy.conf", # entropy.conf file
'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file
'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file
'serverconf': ETP_CONF_DIR+"/server.conf", # server.conf file (generic server side settings)
'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file
'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file
'socketconf': ETP_CONF_DIR+"/socket.conf", # socket.conf file
'packagesrelativepath': "packages/"+ETP_ARCH_CONST+"/", # user by client interfaces
@@ -981,58 +977,6 @@ def const_configureLockPaths():
}
def const_readActivatorSettings():
if os.path.isfile(etpConst['activatorconf']):
f = open(etpConst['activatorconf'],"r")
actconffile = f.readlines()
f.close()
for line in actconffile:
line = line.strip()
if line.startswith("database-format|") and (len(line.split("database-format|")) == 2):
fmt = line.split("database-format|")[1]
if fmt in etpConst['etpdatabasesupportedcformats']:
etpConst['etpdatabasefileformat'] = fmt
def const_readReagentSettings():
if (os.path.isfile(etpConst['reagentconf'])):
f = open(etpConst['reagentconf'],"r")
reagentconf = f.readlines()
f.close()
for line in reagentconf:
if line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2):
feed = line.split("rss-feed|")[1]
if feed in ("enable","enabled","true","1"):
etpConst['rss-feed'] = True
elif feed in ("disable","disabled","false","0","no",):
etpConst['rss-feed'] = False
elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2):
feedname = line.split("rss-name|")[1].strip()
etpConst['rss-name'] = feedname
elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2):
etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip()
if not etpConst['rss-base-url'][-1] == "/":
etpConst['rss-base-url'] += "/"
elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2):
etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip()
elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2):
etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip()
elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2):
try:
entries = int(line.split("max-rss-entries|")[1].strip())
etpConst['rss-max-entries'] = entries
except ValueError:
pass
elif line.startswith("max-rss-light-entries|") and (len(line.split("max-rss-light-entries|")) == 2):
try:
entries = int(line.split("max-rss-light-entries|")[1].strip())
etpConst['rss-light-max-entries'] = entries
except ValueError:
pass
def const_readServerSettings():
if not os.access(etpConst['serverconf'],os.R_OK):
@@ -1041,7 +985,7 @@ def const_readServerSettings():
etpConst['server_repositories'].clear()
f = open(etpConst['serverconf'],"r")
serverconf = f.readlines()
serverconf = [x.strip() for x in f.readlines()]
f.close()
for line in serverconf:
@@ -1067,7 +1011,6 @@ def const_readServerSettings():
pass
elif line.startswith("repository|") and (len(line.split("|")) in [5,6]):
repoid, repodata = const_extractServerRepositoryParameters(line)
if repoid in etpConst['server_repositories']:
# just update mirrors
@@ -1075,6 +1018,47 @@ def const_readServerSettings():
else:
etpConst['server_repositories'][repoid] = repodata.copy()
elif line.startswith("database-format|") and (len(line.split("database-format|")) == 2):
fmt = line.split("database-format|")[1]
if fmt in etpConst['etpdatabasesupportedcformats']:
etpConst['etpdatabasefileformat'] = fmt
elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2):
feed = line.split("rss-feed|")[1]
if feed in ("enable","enabled","true","1"):
etpConst['rss-feed'] = True
elif feed in ("disable","disabled","false","0","no",):
etpConst['rss-feed'] = False
elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2):
feedname = line.split("rss-name|")[1].strip()
etpConst['rss-name'] = feedname
elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2):
etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip()
if not etpConst['rss-base-url'][-1] == "/":
etpConst['rss-base-url'] += "/"
elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2):
etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip()
elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2):
etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip()
elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2):
try:
entries = int(line.split("max-rss-entries|")[1].strip())
etpConst['rss-max-entries'] = entries
except (ValueError,IndexError,):
pass
elif line.startswith("max-rss-light-entries|") and (len(line.split("max-rss-light-entries|")) == 2):
try:
entries = int(line.split("max-rss-light-entries|")[1].strip())
etpConst['rss-light-max-entries'] = entries
except (ValueError,IndexError,):
pass
const_configureServerRepoPaths()
def const_extractServerRepositoryParameters(repostring):
libraries/entropy/qa.py
+3 -1
View File
@@ -128,7 +128,9 @@ class QAInterface:
return broken
def scan_missing_dependencies(self, idpackages, dbconn, ask = True, self_check = False, repo = etpConst['officialrepositoryid'], black_list = None, black_list_adder = None):
def scan_missing_dependencies(self, idpackages, dbconn, ask = True,
self_check = False, repo = etpConst['officialrepositoryid'],
black_list = None, black_list_adder = None):
if not isinstance(black_list,set):
black_list = set()
libraries/entropy/transceivers.py
+101 -37
View File
@@ -589,11 +589,10 @@ class FtpInterface:
self.socket, self.ftplib, self.entropyTools = socket, ftplib, entropyTools
self.Entropy = OutputInterface
self.__verbose = verbose
self.__oldprogress = 0.0
self.__filesize = 0
self.__filekbcount = 0
self.__init_vars()
self.socket.setdefaulttimeout(60)
self.__ftpuri = ftpuri
self.__speed_updater = None
self.__currentdir = '.'
self.__ftphost = self.entropyTools.extractFTPHostFromUri(self.__ftpuri)
self.__ftpuser, self.__ftppassword, self.__ftpport, self.__ftpdir = self.entropyTools.extract_ftp_data(ftpuri)
@@ -636,6 +635,17 @@ class FtpInterface:
)
self.set_cwd(self.__ftpdir, dodir = True)
def __init_vars(self):
self.__oldprogress = 0.0
self.__filesize = 0
self.__filekbcount = 0
self.__transfersize = 0
self.__startingposition = 0
self.__elapsed = 0.0
self.__time_remaining_secs = 0
self.__time_remaining = "(%s)" % (_("infinite"),)
self.__transferpollingtime = float(1)/4
def set_basedir(self):
return self.set_cwd(self.__ftpdir)
@@ -780,14 +790,18 @@ class FtpInterface:
return "226"
return rc
self.__oldprogress = 0.0
def up_file_up_progress(buf):
self.updateProgress(len(buf))
for i in range(10): # ten tries
filename = file.split("/")[len(file.split("/"))-1]
tries = 0
while tries < 10:
tries += 1
filename = os.path.basename(file)
self.__init_vars()
self.__start_speed_counter()
try:
with open(file,"r") as f:
self.__filesize = round(float(self.entropyTools.get_file_size(file))/1024,1)
@@ -809,13 +823,13 @@ class FtpInterface:
return False
except Exception, e: # connection reset by peer
self.entropyTools.printTraceback()
self.Entropy.updateProgress(" ", importance = 0, type = "info")
mytxt = red("%s: %s, %s... #%s") % (
_("Upload issue"),
e,
_("retrying"),
i+1,
tries+1,
)
self.Entropy.updateProgress(
mytxt,
@@ -829,9 +843,10 @@ class FtpInterface:
if self.is_file_available(filename+".tmp"):
self.delete_file(filename+".tmp")
def download_file(self, filename, downloaddir, ascii = False):
finally:
self.__stop_speed_counter()
self.__oldprogress = 0.0
def download_file(self, filename, downloaddir, ascii = False):
def df_up(buf):
# writing file buffer
@@ -852,30 +867,56 @@ class FtpInterface:
percent = True
)
# look if the file exist
if self.is_file_available(filename):
self.__filekbcount = 0
# get the file size
self.__filesize = self.get_file_size_compat(filename)
if (self.__filesize):
self.__filesize = round(float(int(self.__filesize))/1024,1)
if (self.__filesize == 0):
self.__filesize = 1
else:
self.__filesize = 0
if not ascii:
f = open(downloaddir+"/"+filename,"wb")
rc = self.__ftpconn.retrbinary('RETR '+filename, df_up, 1024)
else:
f = open(downloaddir+"/"+filename,"w")
rc = self.__ftpconn.retrlines('RETR '+filename, f.write)
f.flush()
f.close()
if rc.find("226") != -1: # upload complete
return True
return False
else:
return None
tries = 10
while tries:
tries -= 1
self.__init_vars()
self.__start_speed_counter()
try:
# look if the file exist
if self.is_file_available(filename):
self.__filekbcount = 0
# get the file size
self.__filesize = self.get_file_size_compat(filename)
if (self.__filesize):
self.__filesize = round(float(int(self.__filesize))/1024,1)
if (self.__filesize == 0):
self.__filesize = 1
else:
self.__filesize = 0
if not ascii:
f = open(downloaddir+"/"+filename,"wb")
rc = self.__ftpconn.retrbinary('RETR '+filename, df_up, 1024)
else:
f = open(downloaddir+"/"+filename,"w")
rc = self.__ftpconn.retrlines('RETR '+filename, f.write)
f.flush()
f.close()
if rc.find("226") != -1: # upload complete
return True
return False
except Exception, e: # connection reset by peer
self.entropyTools.printTraceback()
mytxt = red("%s: %s, %s... #%s") % (
_("Download issue"),
e,
_("retrying"),
i+1,
)
self.Entropy.updateProgress(
mytxt,
importance = 1,
type = "warning",
header = " "
)
self.reconnect_host() # reconnect
finally:
self.__stop_speed_counter()
# also used to move files
def rename_file(self, fromfile, tofile):
@@ -911,9 +952,31 @@ class FtpInterface:
# timeout, who cares!
pass
def __start_speed_counter(self):
self.__speed_updater = TimeScheduled(
self.__transferpollingtime,
self.__update_speed,
)
self.__speed_updater.start()
def __stop_speed_counter(self):
if self.__speed_updater != None:
self.__speed_updater.kill()
def __update_speed(self):
self.__elapsed += self.__transferpollingtime
# we have the diff size
self.__datatransfer = (self.__filesize-self.__startingposition) / self.__elapsed
try:
self.__time_remaining_secs = int(round((int(round(self.__filesize*1024,0))-int(round(self.__transfersize,0)))/self.__datatransfer,0))
self.__time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.__time_remaining_secs)
except:
self.__time_remaining = "(%s)" % (_("infinite"),)
def updateProgress(self, buf_len):
# get the buffer size
self.__filekbcount += float(buf_len)/1024
self.__transfersize += buf_len
# create percentage
myUploadPercentage = 100.0
if self.__filesize >= 1:
@@ -926,10 +989,11 @@ class FtpInterface:
myUploadPercentage = str(myUploadPercentage)+"%"
# create text
mytxt = _("Upload status")
mytxt = _("Transfer status")
currentText = brown(" <-> %s: " % (mytxt,)) + \
darkgreen(str(myUploadSize)) + "/" + red(str(self.__filesize)) + " kB " + \
brown("[") + str(myUploadPercentage) + brown("]")
brown("[") + str(myUploadPercentage) + brown("]") + " " + self.__time_remaining + \
self.entropyTools.bytesIntoHuman(self.__datatransfer) + "/"+_("sec")
# WARN: re-enabled updateProgress, this may cause slowdowns
# print_info(currentText, back = True)
self.Entropy.updateProgress(currentText, back = True)