diff --git a/conf/remote.conf b/conf/remote.conf new file mode 100644 index 000000000..ed7ebdeb1 --- /dev/null +++ b/conf/remote.conf @@ -0,0 +1,8 @@ +# Project Entropy 1.0 Remote handlers configuration file + +# Log level +# 0: No Logging +# 1: Normal Logging +# 2: Verbose Logging +loglevel|2 + diff --git a/handlers/md5sum.php b/handlers/md5sum.php new file mode 100644 index 000000000..c7b503da2 --- /dev/null +++ b/handlers/md5sum.php @@ -0,0 +1,7 @@ + \ No newline at end of file diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 04461dddc..b70a434a0 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -33,11 +33,14 @@ import commands import string import time + # Logging initialization import logTools activatorLog = logTools.LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]") # example: activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFuncton: called.") +import remoteTools + def sync(options, justTidy = False): activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: called with justTidy -> "+str(justTidy)) @@ -191,7 +194,7 @@ def packages(options): print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) ftp = mirrorTools.handlerFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) - remotePackages = ftp.listFTPdir() + remotePackages = ftp.listDir() remotePackagesInfo = ftp.getRoughList() ftp.closeFTPConnection() @@ -491,14 +494,67 @@ def packages(options): uploadItem = etpConst['packagessuploaddir']+"/"+item[0] else: uploadItem = etpConst['packagesbindir']+"/"+item[0] - rc = ftp.uploadFile(uploadItem) + + ckOk = False + while not ckOk: + rc = ftp.uploadFile(uploadItem) + # verify upload using remoteTools + print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True) + ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0]) + if (ck == None): + print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported")) + ckOk = True + else: + if (ck == False): + # file does not exist??? + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading...")) + else: + if len(ck) == 32: + # valid checksum, checking + ckres = compareMd5(uploadItem,ck) + if (ckres): + print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been uploaded correctly.")) + ckOk = True + else: + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" has NOT been uploaded correctly. Reuploading...")) + else: + # hum, what the hell is this checksum!?!?!?! + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum. Reuploading...")) + if not os.path.isfile(uploadItem+etpConst['packageshashfileext']): hashfile = createHashFile(uploadItem) else: hashfile = uploadItem+etpConst['packageshashfileext'] + # upload md5 hash - rcmd5 = ftp.uploadFile(hashfile,ascii = True) - + print_info(counterInfo+red(" Uploading checksum of ")+bold(item[0]) + red(" to ") + bold(extractFTPHostFromUri(uri)) +red(" ...")) + ckOk = False + while not ckOk: + rcmd5 = ftp.uploadFile(hashfile,ascii = True) + # verify upload using remoteTools + print_info(counterInfo+red(" -> Verifying ")+green(item[0]+etpConst['packageshashfileext'])+bold(" checksum")+red(" (if supported)"), back = True) + ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0]) + if (ck == None): + print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0]+etpConst['packageshashfileext'])+bold(" not supported")) + ckOk = True + else: + if (ck == False): + # file does not exist??? + print_warning(counterInfo+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not exist remotely. Reuploading...")) + else: + if len(ck) == 32: + # valid checksum, checking + ckres = compareMd5(hashfile,ck) + if (ckres): + print_info(counterInfo+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" has been uploaded correctly.")) + ckOk = True + else: + print_warning(counterInfo+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" has NOT been uploaded correctly. Reuploading...")) + else: + # hum, what the hell is this checksum!?!?!?! + print_warning(counterInfo+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not have a proper checksum. Reuploading...")) + + # now check if (rc) and (rcmd5): successfulUploadCounter += 1 print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) @@ -521,11 +577,37 @@ def packages(options): continue print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ...")) - # FIXME: test if the .md5 got downloaded - if item[0].endswith(etpConst['packageshashfileext']): - rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True) - else: - rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") + ckOk = False + while not ckOk: + + if item[0].endswith(etpConst['packageshashfileext']): + rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True) + else: + rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") + + # verify upload using remoteTools + print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True) + ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0]) + if (ck == None): + print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported")) + ckOk = True + else: + if (ck == False): + # file does not exist??? + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading...")) + else: + if len(ck) == 32: + # valid checksum, checking + filepath = etpConst['packagesbindir']+"/"+item[0] + ckres = compareMd5(filepath,ck) + if (ckres): + print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been uploaded correctly.")) + ckOk = True + else: + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" has NOT been uploaded correctly. Reuploading...")) + else: + # hum, what the hell is this checksum!?!?!?! + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum. Reuploading...")) if (rc): successfulDownloadCounter += 1 @@ -542,9 +624,11 @@ def packages(options): # trap exceptions, failed to upload/download someting? except Exception, e: - print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Continuing if possible...")) - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.") + print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Showing traceback:")) + import traceback + traceback.print_stack() + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.") activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: cannot properly syncronize "+extractFTPHostFromUri(uri)+". Trying to continue if possible.") # print warning cannot sync uri diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index ce484fdce..2578b7d08 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -26,6 +26,7 @@ import string import random import sys + # Specifications of the content of etpData # THIS IS THE KEY PART OF ENTROPY BINARY PACKAGES MANAGEMENT # DO NOT EDIT THIS UNLESS YOU KNOW WHAT YOU'RE DOING !! @@ -158,6 +159,7 @@ etpConst = { 'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file 'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # Source Package Manager backend configuration (Portage now) 'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file + 'remoteconf': ETP_CONF_DIR+"/remote.conf", # remote.conf file 'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf) 'activatordownloaduris': [], # list of URIs that activator can use to fetch data 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. @@ -174,6 +176,7 @@ etpConst = { 'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info) 'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info) + 'remoteloglevel': 1, # Remote handlers (/handlers) log level (default: 1 - see remote.conf for more info) 'enzymeloglevel': 1 , # Enzyme log level (default: 1 - see enzyme.conf for more info) 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) @@ -183,6 +186,7 @@ etpConst = { 'syslogdir': ETP_SYSLOG_DIR, # Entropy system tools log directory 'mirrorslogfile': ETP_SYSLOG_DIR+"/mirrors.log", # Mirrors operations log file + 'remotelogfile': ETP_SYSLOG_DIR+"/remote.log", # Mirrors operations log file 'spmbackendlogfile': ETP_SYSLOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file 'databaselogfile': ETP_SYSLOG_DIR+"/database.log", # Database operations log file 'enzymelogfile': ETP_SYSLOG_DIR+"/enzyme.log", # Enzyme operations log file @@ -203,6 +207,11 @@ etpConst = { 'postinstallscript': "postinstall.sh", # used by the client to run some post-install actions } +# Handlers used by entropy to run and retrieve data remotely, using php helpers +etpHandlers = { + 'md5sum': "md5sum.php?arch="+ETP_ARCH_CONST+"&package=", # md5sum handler +} + # Create paths if not os.path.isdir(ETP_DIR): import getpass @@ -470,7 +479,39 @@ else: print "WARNING: invalid loglevel in: "+etpConst['mirrorsconf'] import time time.sleep(5) - + +# remote section +etpRemoteSupport = {} +if (not os.path.isfile(etpConst['remoteconf'])): + print "ERROR: "+etpConst['remoteconf']+" does not exist" + sys.exit(50) +else: + f = open(etpConst['remoteconf'],"r") + databaseconf = f.readlines() + f.close() + for line in databaseconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['remoteconf'] + sys.exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['remoteloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['remoteconf'] + import time + time.sleep(5) + + if line.startswith("httphandler|") and (len(line.split("|")) > 2): + servername = line.split("httphandler|")[1] + url = line.split("httphandler|")[2] + if not url.endswith("/"): + url = url+"/" + etpRemoteSupport[servername] = url + + # spmbackend section if (not os.path.isfile(etpConst['spmbackendconf'])): print "ERROR: "+etpConst['spmbackendconf']+" does not exist" diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 214c6a36d..9badeb902 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -41,6 +41,16 @@ entropyLog = logTools.LogFile(level=etpConst['entropyloglevel'],filename = etpCo # EXIT STATUSES: 100-199 +global __etp_debug +__etp_debug = False +def enableDebug(): + __etp_debug = True + import pdb + pdb.set_trace() + +def getDebug(): + return __etp_debug + def isRoot(): import getpass if (getpass.getuser() == "root"): diff --git a/libraries/mirrorTools.py b/libraries/mirrorTools.py index 734cda017..9a1a9b5c6 100644 --- a/libraries/mirrorTools.py +++ b/libraries/mirrorTools.py @@ -28,7 +28,6 @@ from outputTools import * import entropyTools import string import os -import ftplib import time # Logging initialization @@ -41,7 +40,20 @@ class handlerFTP: # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org # this must be run before calling the other functions - def __init__(self, ftpuri): + def __init__(self, ftpuri, debug = None): + + # ftp debugging + if entropyTools.getDebug(): + debug = True + else: + debug = False + # FIXME: remove this + #debug = True + + # import FTP modules + import timeoutsocket + import ftplib + timeoutsocket.setDefaultSocketTimeout(60) mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.__init__: called.") @@ -80,36 +92,24 @@ class handlerFTP: self.ftpdir = self.ftpdir[:len(self.ftpdir)-1] self.ftpconn = ftplib.FTP(self.ftphost) + # enable debug? + if debug: + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.__init__: DEBUG enabled.") + self.ftpconn.set_debuglevel(2) + self.ftpconn.login(self.ftpuser,self.ftppassword) # change to our dir #print self.ftpdir self.ftpconn.cwd(self.ftpdir) self.currentdir = self.ftpdir - def my_storbinary(self, cmd, fp, callback=None): # patched for callback - '''Store a file in line mode.''' - self.ftpconn.voidcmd('TYPE I') - CRLF = ftplib.CRLF - conn = self.ftpconn.transfercmd(cmd) - while 1: - buf = fp.readline() - if not buf: break - if buf[-2:] != CRLF: - if buf[-1] in CRLF: buf = buf[:-1] - buf = buf + CRLF - conn.sendall(buf) - if callback: callback(buf) # patched for callback - conn.close() - return self.ftpconn.voidresp() - #FTP.storbinary = my_storbinary # use the patched version - # this can be used in case of exceptions def reconnectHost(self): mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.reconnectHost: called.") - #try: - # self.closeFTPConnection() - #except: - # pass + # import FTP modules + import timeoutsocket + import ftplib + timeoutsocket.setDefaultSocketTimeout(60) self.ftpconn = ftplib.FTP(self.ftphost) self.ftpconn.login(self.ftpuser,self.ftppassword) # save curr dir @@ -117,16 +117,16 @@ class handlerFTP: #self.setCWD(self.ftpdir) self.setCWD(self.currentdir) - def getFTPHost(self): - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPHost: called -> "+self.ftphost) + def getHost(self): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getHost: called -> "+self.ftphost) return self.ftphost - def getFTPPort(self): - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpport) + def getPort(self): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getPort: called -> "+self.ftpport) return self.ftpport - def getFTPDir(self): - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpdir) + def getDir(self): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getDir: called -> "+self.ftpdir) return self.ftpdir def getCWD(self): @@ -148,16 +148,16 @@ class handlerFTP: rc = self.ftpconn.sendcmd("mdtm "+path) return rc.split()[len(rc.split())-1] - def spawnFTPCommand(self,cmd): - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnFTPCommand: called, command -> "+cmd) + def spawnCommand(self,cmd): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnCommand: called, command -> "+cmd) rc = self.ftpconn.sendcmd(cmd) - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnFTPCommand: called, rc -> "+str(rc)) + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnCommand: called, rc -> "+str(rc)) return rc # list files and directory of a FTP # @returns a list - def listFTPdir(self): - mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.listFTPdir: called.") + def listDir(self): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.listDir: called.") # directory is: self.ftpdir try: rc = self.ftpconn.nlst() @@ -217,10 +217,16 @@ class handlerFTP: if callback: callback(buf) mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.advancedStorBinary: before conn.close()") conn.close() - time.sleep(3) - rc = self.ftpconn.voidresp() mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.advancedStorBinary: after conn.close()") - return rc + + # that's another workaround + #return "226" + try: + rc = self.ftpconn.voidresp() + except Timeout: + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"handlerFTP.advancedStorBinary: timeout receiving voidresp(), reconnecting...") + self.reconnectHost() + return "226" def uploadFile(self,file,ascii = False): @@ -243,11 +249,11 @@ class handlerFTP: for i in range(10): # ten tries mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.uploadFile: try #"+str(i+1)) - + filename = file.split("/")[len(file.split("/"))-1] + try: f = open(file,"r") - filename = file.split("/")[len(file.split("/"))-1] # get file size self.myFileSize = round(float(os.stat(file)[6])/1024,1) @@ -259,7 +265,7 @@ class handlerFTP: if (ascii): rc = self.ftpconn.storlines("STOR "+filename+".tmp",f) else: - rc = self.my_storbinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress ) + rc = self.advancedStorBinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress ) # now we can rename the file with its original name self.renameFile(filename+".tmp",filename) @@ -275,8 +281,11 @@ class handlerFTP: return False except Exception, e: # connection reset by peer - mirrorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"handlerFTP.uploadFile: Caught Exception: "+str(e)+" upload issues, retrying...") - print_info(red("Upload issue, retrying... #"+str(i+1))) + mirrorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"handlerFTP.uploadFile: Caught Exception: "+str(e)+", upload issues, retrying...") + import traceback + traceback.print_stack() + print_warning("") + print_warning(red(" Upload issue: ")+bold(str(e))+red(", retrying... #"+str(i+1))) self.reconnectHost() # reconnect mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.uploadFile: after reconnectHost()") if self.isFileAvailable(filename): diff --git a/libraries/remoteTools.py b/libraries/remoteTools.py new file mode 100644 index 000000000..5da6615e4 --- /dev/null +++ b/libraries/remoteTools.py @@ -0,0 +1,60 @@ +#!/usr/bin/python +''' + # DESCRIPTION: + # Entropy Mirrors interface + + Copyright (C) 2007 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' + +# Never do "import portage" here, please use entropyTools binding +# EXIT STATUSES: 700-799 + +from entropyConstants import * +from outputTools import * +import entropyTools + +# Logging initialization +import logTools +remoteLog = logTools.LogFile(level=etpConst['remoteloglevel'],filename = etpConst['remotelogfile'], header = "[REMOTE/HTTP]") +# example: mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFuncton: called.") + +import timeoutsocket +import urllib +timeoutsocket.setDefaultSocketTimeout(60) + +# Get checksum of a package by running md5sum remotely (using php helpers) +# @returns hex: if the file exists +# @returns None: if the server does not support HTTP handlers +# @returns False: if the file is not found +def getRemotePackageChecksum(serverName,filename): + remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getRemotePackageChecksum: called.") + # etpHandlers['md5sum'] is the command + # create the request + try: + url = etpRemoteSupport[servername] + except: + # not found, does not support HTTP handlers + return None + + request = url+etpHandlers['md5sum']+filename + remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getRemotePackageChecksum: requested url -> "+request) + + # now pray the server + file = urllib.urlopen(request) + result = file.readline().strip() + return result + \ No newline at end of file diff --git a/libraries/timeoutsocket.py b/libraries/timeoutsocket.py new file mode 100644 index 000000000..be67e79db --- /dev/null +++ b/libraries/timeoutsocket.py @@ -0,0 +1,424 @@ + +#### +# Copyright 2000,2001 by Timothy O'Malley +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software +# and its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Timothy O'Malley not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR +# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. +# +#### + +"""Timeout Socket + +This module enables a timeout mechanism on all TCP connections. It +does this by inserting a shim into the socket module. After this module +has been imported, all socket creation goes through this shim. As a +result, every TCP connection will support a timeout. + +The beauty of this method is that it immediately and transparently +enables the entire python library to support timeouts on TCP sockets. +As an example, if you wanted to SMTP connections to have a 20 second +timeout: + + import timeoutsocket + import smtplib + timeoutsocket.setDefaultSocketTimeout(20) + + +The timeout applies to the socket functions that normally block on +execution: read, write, connect, and accept. If any of these +operations exceeds the specified timeout, the exception Timeout +will be raised. + +The default timeout value is set to None. As a result, importing +this module does not change the default behavior of a socket. The +timeout mechanism only activates when the timeout has been set to +a numeric value. (This behavior mimics the behavior of the +select.select() function.) + +This module implements two classes: TimeoutSocket and TimeoutFile. + +The TimeoutSocket class defines a socket-like object that attempts to +avoid the condition where a socket may block indefinitely. The +TimeoutSocket class raises a Timeout exception whenever the +current operation delays too long. + +The TimeoutFile class defines a file-like object that uses the TimeoutSocket +class. When the makefile() method of TimeoutSocket is called, it returns +an instance of a TimeoutFile. + +Each of these objects adds two methods to manage the timeout value: + + get_timeout() --> returns the timeout of the socket or file + set_timeout() --> sets the timeout of the socket or file + + +As an example, one might use the timeout feature to create httplib +connections that will timeout after 30 seconds: + + import timeoutsocket + import httplib + H = httplib.HTTP("www.python.org") + H.sock.set_timeout(30) + +Note: When used in this manner, the connect() routine may still +block because it happens before the timeout is set. To avoid +this, use the 'timeoutsocket.setDefaultSocketTimeout()' function. + +Good Luck! + +""" + +__version__ = "$Revision: 1.23 $" +__author__ = "Timothy O'Malley " + +# +# Imports +# +import select, string +import socket +if not hasattr(socket, "_no_timeoutsocket"): + _socket = socket.socket +else: + _socket = socket._no_timeoutsocket + + +# +# Set up constants to test for Connected and Blocking operations. +# We delete 'os' and 'errno' to keep our namespace clean(er). +# Thanks to Alex Martelli and G. Li for the Windows error codes. +# +import os +if os.name == "nt": + _IsConnected = ( 10022, 10056 ) + _ConnectBusy = ( 10035, ) + _AcceptBusy = ( 10035, ) +else: + import errno + _IsConnected = ( errno.EISCONN, ) + _ConnectBusy = ( errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK ) + _AcceptBusy = ( errno.EAGAIN, errno.EWOULDBLOCK ) + del errno +del os + + +# +# Default timeout value for ALL TimeoutSockets +# +_DefaultTimeout = None +def setDefaultSocketTimeout(timeout): + global _DefaultTimeout + _DefaultTimeout = timeout +def getDefaultSocketTimeout(): + return _DefaultTimeout + +# +# Exceptions for socket errors and timeouts +# +Error = socket.error +class Timeout(Exception): + pass + + +# +# Factory function +# +from socket import AF_INET, SOCK_STREAM +def timeoutsocket(family=AF_INET, type=SOCK_STREAM, proto=None): + if family != AF_INET or type != SOCK_STREAM: + if proto: + return _socket(family, type, proto) + else: + return _socket(family, type) + return TimeoutSocket( _socket(family, type), _DefaultTimeout ) +# end timeoutsocket + +# +# The TimeoutSocket class definition +# +class TimeoutSocket: + """TimeoutSocket object + Implements a socket-like object that raises Timeout whenever + an operation takes too long. + The definition of 'too long' can be changed using the + set_timeout() method. + """ + + _copies = 0 + _blocking = 1 + + def __init__(self, sock, timeout): + self._sock = sock + self._timeout = timeout + # end __init__ + + def __getattr__(self, key): + return getattr(self._sock, key) + # end __getattr__ + + def get_timeout(self): + return self._timeout + # end set_timeout + + def set_timeout(self, timeout=None): + self._timeout = timeout + # end set_timeout + + def setblocking(self, blocking): + self._blocking = blocking + return self._sock.setblocking(blocking) + # end set_timeout + + def connect_ex(self, addr): + errcode = 0 + try: + self.connect(addr) + except Error, why: + errcode = why[0] + return errcode + # end connect_ex + + def connect(self, addr, port=None, dumbhack=None): + # In case we were called as connect(host, port) + if port != None: addr = (addr, port) + + # Shortcuts + sock = self._sock + timeout = self._timeout + blocking = self._blocking + + # First, make a non-blocking call to connect + try: + sock.setblocking(0) + sock.connect(addr) + sock.setblocking(blocking) + return + except Error, why: + # Set the socket's blocking mode back + sock.setblocking(blocking) + + # If we are not blocking, re-raise + if not blocking: + raise + + # If we are already connected, then return success. + # If we got a genuine error, re-raise it. + errcode = why[0] + if dumbhack and errcode in _IsConnected: + return + elif errcode not in _ConnectBusy: + raise + + # Now, wait for the connect to happen + # ONLY if dumbhack indicates this is pass number one. + # If select raises an error, we pass it on. + # Is this the right behavior? + if not dumbhack: + r,w,e = select.select([], [sock], [], timeout) + if w: + return self.connect(addr, dumbhack=1) + + # If we get here, then we should raise Timeout + raise Timeout("Attempted connect to %s timed out." % str(addr) ) + # end connect + + def accept(self, dumbhack=None): + # Shortcuts + sock = self._sock + timeout = self._timeout + blocking = self._blocking + + # First, make a non-blocking call to accept + # If we get a valid result, then convert the + # accept'ed socket into a TimeoutSocket. + # Be carefult about the blocking mode of ourselves. + try: + sock.setblocking(0) + newsock, addr = sock.accept() + sock.setblocking(blocking) + timeoutnewsock = self.__class__(newsock, timeout) + timeoutnewsock.setblocking(blocking) + return (timeoutnewsock, addr) + except Error, why: + # Set the socket's blocking mode back + sock.setblocking(blocking) + + # If we are not supposed to block, then re-raise + if not blocking: + raise + + # If we got a genuine error, re-raise it. + errcode = why[0] + if errcode not in _AcceptBusy: + raise + + # Now, wait for the accept to happen + # ONLY if dumbhack indicates this is pass number one. + # If select raises an error, we pass it on. + # Is this the right behavior? + if not dumbhack: + r,w,e = select.select([sock], [], [], timeout) + if r: + return self.accept(dumbhack=1) + + # If we get here, then we should raise Timeout + raise Timeout("Attempted accept timed out.") + # end accept + + def send(self, data, flags=0): + sock = self._sock + if self._blocking: + r,w,e = select.select([],[sock],[], self._timeout) + if not w: + raise Timeout("Send timed out") + return sock.send(data, flags) + # end send + + def recv(self, bufsize, flags=0): + sock = self._sock + if self._blocking: + r,w,e = select.select([sock], [], [], self._timeout) + if not r: + raise Timeout("Recv timed out") + return sock.recv(bufsize, flags) + # end recv + + def makefile(self, flags="r", bufsize=-1): + self._copies = self._copies +1 + return TimeoutFile(self, flags, bufsize) + # end makefile + + def close(self): + if self._copies <= 0: + self._sock.close() + else: + self._copies = self._copies -1 + # end close + +# end TimeoutSocket + + +class TimeoutFile: + """TimeoutFile object + Implements a file-like object on top of TimeoutSocket. + """ + + def __init__(self, sock, mode="r", bufsize=4096): + self._sock = sock + self._bufsize = 4096 + if bufsize > 0: self._bufsize = bufsize + if not hasattr(sock, "_inqueue"): self._sock._inqueue = "" + + # end __init__ + + def __getattr__(self, key): + return getattr(self._sock, key) + # end __getattr__ + + def close(self): + self._sock.close() + self._sock = None + # end close + + def write(self, data): + self.send(data) + # end write + + def read(self, size=-1): + _sock = self._sock + _bufsize = self._bufsize + while 1: + datalen = len(_sock._inqueue) + if datalen >= size >= 0: + break + bufsize = _bufsize + if size > 0: + bufsize = min(bufsize, size - datalen ) + buf = self.recv(bufsize) + if not buf: + break + _sock._inqueue = _sock._inqueue + buf + data = _sock._inqueue + _sock._inqueue = "" + if size > 0 and datalen > size: + _sock._inqueue = data[size:] + data = data[:size] + return data + # end read + + def readline(self, size=-1): + _sock = self._sock + _bufsize = self._bufsize + while 1: + idx = string.find(_sock._inqueue, "\n") + if idx >= 0: + break + datalen = len(_sock._inqueue) + if datalen >= size >= 0: + break + bufsize = _bufsize + if size > 0: + bufsize = min(bufsize, size - datalen ) + buf = self.recv(bufsize) + if not buf: + break + _sock._inqueue = _sock._inqueue + buf + + data = _sock._inqueue + _sock._inqueue = "" + if idx >= 0: + idx = idx + 1 + _sock._inqueue = data[idx:] + data = data[:idx] + elif size > 0 and datalen > size: + _sock._inqueue = data[size:] + data = data[:size] + return data + # end readline + + def readlines(self, sizehint=-1): + result = [] + data = self.read() + while data: + idx = string.find(data, "\n") + if idx >= 0: + idx = idx + 1 + result.append( data[:idx] ) + data = data[idx:] + else: + result.append( data ) + data = "" + return result + # end readlines + + def flush(self): pass + +# end TimeoutFile + + +# +# Silently replace the socket() builtin function with +# our timeoutsocket() definition. +# +if not hasattr(socket, "_no_timeoutsocket"): + socket._no_timeoutsocket = socket.socket + socket.socket = timeoutsocket +del socket +socket = timeoutsocket +# Finis diff --git a/server/activator b/server/activator index 0f6364352..7ba26f015 100644 --- a/server/activator +++ b/server/activator @@ -59,11 +59,13 @@ def print_help(): options = sys.argv[1:] -# no color parsing +# preliminary options parsing _options = [] for opt in options: if (opt == "--nocolor"): entropyTools.nocolor() + elif (opt == "--debug"): + entropyTools.enableDebug() else: _options.append(opt) options = _options diff --git a/server/enzyme b/server/enzyme index ec0c2ebf0..40b15031b 100644 --- a/server/enzyme +++ b/server/enzyme @@ -86,11 +86,13 @@ def print_help(): options = sys.argv[1:] -# no color parsing +# preliminary options parsing _options = [] for opt in options: if (opt == "--nocolor"): entropyTools.nocolor() + elif (opt == "--debug"): + entropyTools.enableDebug() else: _options.append(opt) options = _options diff --git a/server/reagent b/server/reagent index b4d4be99e..358d90b5d 100644 --- a/server/reagent +++ b/server/reagent @@ -65,11 +65,13 @@ def print_help(): options = sys.argv[1:] -# no color parsing +# preliminary options parsing _options = [] for opt in options: if (opt == "--nocolor"): entropyTools.nocolor() + elif (opt == "--debug"): + entropyTools.enableDebug() else: _options.append(opt) options = _options