added http php helpers support, md5sum verification for uploaded and downloaded files, a lot of bugfixes

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@316 cd1c1023-2f26-0410-ae45-c471fc1f0318

conf/remote.conf

@@ -0,0 +1,8 @@
+# Project Entropy 1.0 Remote handlers configuration file
+
+# Log level
+# 0: No Logging
+# 1: Normal Logging
+# 2: Verbose Logging
+loglevel|2
+

handlers/md5sum.php

@@ -0,0 +1,7 @@
+<?php
+	$pkgfile = $_GET['package'];
+	$pkgarch = $_GET['arch'];
+	$pkgfile = "../packages/" . $pkgarch . "/" . $pkgfile;
+	$md5 = md5_file($pkgfile);
+	echo($md5);
+?>

libraries/activatorTools.py

@@ -33,11 +33,14 @@ import commands
 import string
 import time
 
+
 # Logging initialization
 import logTools
 activatorLog = logTools.LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]")
 # example: activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFuncton: called.")
 
+import remoteTools
+
 def sync(options, justTidy = False):
 
     activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: called with justTidy -> "+str(justTidy))
@@ -191,7 +194,7 @@ def packages(options):
 	        print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
 	        ftp = mirrorTools.handlerFTP(uri)
 	        ftp.setCWD(etpConst['binaryurirelativepath'])
-	        remotePackages = ftp.listFTPdir()
+	        remotePackages = ftp.listDir()
 	        remotePackagesInfo = ftp.getRoughList()
 	        ftp.closeFTPConnection()
 
@@ -491,14 +494,67 @@ def packages(options):
 			    uploadItem = etpConst['packagessuploaddir']+"/"+item[0]
 			else:
 			    uploadItem = etpConst['packagesbindir']+"/"+item[0]
-			rc = ftp.uploadFile(uploadItem)
+			
+			ckOk = False
+			while not ckOk:
+			    rc = ftp.uploadFile(uploadItem)
+			    # verify upload using remoteTools
+			    print_info(counterInfo+red("   -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
+			    ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0])
+			    if (ck == None):
+				print_warning(counterInfo+red("   -> Digest verification of ")+green(item[0])+bold(" not supported"))
+				ckOk = True
+			    else:
+				if (ck == False):
+				    # file does not exist???
+				    print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading..."))
+				else:
+				    if len(ck) == 32:
+					# valid checksum, checking
+					ckres = compareMd5(uploadItem,ck)
+					if (ckres):
+					    print_info(counterInfo+red("   -> Package ")+bold(item[0])+red(" has been uploaded correctly."))
+					    ckOk = True
+					else:
+					    print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" has NOT been uploaded correctly. Reuploading..."))
+				    else:
+					# hum, what the hell is this checksum!?!?!?!
+					print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" does not have a proper checksum. Reuploading..."))
+			
 			if not os.path.isfile(uploadItem+etpConst['packageshashfileext']):
 			    hashfile = createHashFile(uploadItem)
 			else:
 			    hashfile = uploadItem+etpConst['packageshashfileext']
+
 			# upload md5 hash
-			rcmd5 = ftp.uploadFile(hashfile,ascii = True)
-			
+			print_info(counterInfo+red(" Uploading checksum of ")+bold(item[0]) + red(" to ") + bold(extractFTPHostFromUri(uri)) +red(" ..."))
+			ckOk = False
+			while not ckOk:
+			    rcmd5 = ftp.uploadFile(hashfile,ascii = True)
+			    # verify upload using remoteTools
+			    print_info(counterInfo+red("   -> Verifying ")+green(item[0]+etpConst['packageshashfileext'])+bold(" checksum")+red(" (if supported)"), back = True)
+			    ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0])
+			    if (ck == None):
+				print_warning(counterInfo+red("   -> Digest verification of ")+green(item[0]+etpConst['packageshashfileext'])+bold(" not supported"))
+				ckOk = True
+			    else:
+				if (ck == False):
+				    # file does not exist???
+				    print_warning(counterInfo+red("   -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not exist remotely. Reuploading..."))
+				else:
+				    if len(ck) == 32:
+					# valid checksum, checking
+					ckres = compareMd5(hashfile,ck)
+					if (ckres):
+					    print_info(counterInfo+red("   -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" has been uploaded correctly."))
+					    ckOk = True
+					else:
+					    print_warning(counterInfo+red("   -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" has NOT been uploaded correctly. Reuploading..."))
+				    else:
+					# hum, what the hell is this checksum!?!?!?!
+					print_warning(counterInfo+red("   -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not have a proper checksum. Reuploading..."))
+
+			# now check
 			if (rc) and (rcmd5):
 			    successfulUploadCounter += 1
 		    print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
@@ -521,11 +577,37 @@ def packages(options):
 			        continue
 		        print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."))
 			
-			# FIXME: test if the .md5 got downloaded
-			if item[0].endswith(etpConst['packageshashfileext']):
-			    rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True)
-			else:
-		            rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
+			ckOk = False
+			while not ckOk:
+
+			    if item[0].endswith(etpConst['packageshashfileext']):
+				rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True)
+			    else:
+				rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
+
+			    # verify upload using remoteTools
+			    print_info(counterInfo+red("   -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
+			    ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0])
+			    if (ck == None):
+				print_warning(counterInfo+red("   -> Digest verification of ")+green(item[0])+bold(" not supported"))
+				ckOk = True
+			    else:
+				if (ck == False):
+				    # file does not exist???
+				    print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading..."))
+				else:
+				    if len(ck) == 32:
+					# valid checksum, checking
+					filepath = etpConst['packagesbindir']+"/"+item[0]
+					ckres = compareMd5(filepath,ck)
+					if (ckres):
+					    print_info(counterInfo+red("   -> Package ")+bold(item[0])+red(" has been uploaded correctly."))
+					    ckOk = True
+					else:
+					    print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" has NOT been uploaded correctly. Reuploading..."))
+				    else:
+					# hum, what the hell is this checksum!?!?!?!
+					print_warning(counterInfo+red("   -> Package ")+bold(item[0])+red(" does not have a proper checksum. Reuploading..."))
 			
 			if (rc):
 			    successfulDownloadCounter += 1
@@ -542,9 +624,11 @@ def packages(options):
 	    # trap exceptions, failed to upload/download someting?
 	    except Exception, e:
 		
-		print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Continuing if possible..."))
-		activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.")
+		print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Showing traceback:"))
+		import traceback
+		traceback.print_stack()
 		
+		activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.")
 		activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: cannot properly syncronize "+extractFTPHostFromUri(uri)+". Trying to continue if possible.")
 		
 		# print warning cannot sync uri

libraries/entropyConstants.py

@@ -26,6 +26,7 @@ import string
 import random
 import sys
 
+
 # Specifications of the content of etpData
 # THIS IS THE KEY PART OF ENTROPY BINARY PACKAGES MANAGEMENT
 # DO NOT EDIT THIS UNLESS YOU KNOW WHAT YOU'RE DOING !!
@@ -158,6 +159,7 @@ etpConst = {
     'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file
     'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # Source Package Manager backend configuration (Portage now)
     'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file
+    'remoteconf': ETP_CONF_DIR+"/remote.conf", # remote.conf file
     'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf)
     'activatordownloaduris': [], # list of URIs that activator can use to fetch data
     'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository.
@@ -174,6 +176,7 @@ etpConst = {
     
     'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info)
     'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info)
+    'remoteloglevel': 1, # Remote handlers (/handlers) log level (default: 1 - see remote.conf for more info)
     'enzymeloglevel': 1 , # Enzyme log level (default: 1 - see enzyme.conf for more info)
     'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info)
     'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info)
@@ -183,6 +186,7 @@ etpConst = {
     
     'syslogdir': ETP_SYSLOG_DIR, # Entropy system tools log directory
     'mirrorslogfile': ETP_SYSLOG_DIR+"/mirrors.log", # Mirrors operations log file
+    'remotelogfile': ETP_SYSLOG_DIR+"/remote.log", # Mirrors operations log file
     'spmbackendlogfile': ETP_SYSLOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file
     'databaselogfile': ETP_SYSLOG_DIR+"/database.log", # Database operations log file
     'enzymelogfile': ETP_SYSLOG_DIR+"/enzyme.log", # Enzyme operations log file
@@ -203,6 +207,11 @@ etpConst = {
     'postinstallscript': "postinstall.sh", # used by the client to run some post-install actions
  }
 
+# Handlers used by entropy to run and retrieve data remotely, using php helpers
+etpHandlers = {
+    'md5sum': "md5sum.php?arch="+ETP_ARCH_CONST+"&package=", # md5sum handler
+}
+
 # Create paths
 if not os.path.isdir(ETP_DIR):
     import getpass
@@ -470,7 +479,39 @@ else:
 		print "WARNING: invalid loglevel in: "+etpConst['mirrorsconf']
 		import time
 		time.sleep(5)
-		
+	
+# remote section
+etpRemoteSupport = {}
+if (not os.path.isfile(etpConst['remoteconf'])):
+    print "ERROR: "+etpConst['remoteconf']+" does not exist"
+    sys.exit(50)
+else:
+    f = open(etpConst['remoteconf'],"r")
+    databaseconf = f.readlines()
+    f.close()
+    for line in databaseconf:
+	if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2):
+	    loglevel = line.split("loglevel|")[1]
+	    try:
+		loglevel = int(loglevel)
+	    except:
+		print "ERROR: invalid loglevel in: "+etpConst['remoteconf']
+		sys.exit(51)
+	    if (loglevel > -1) and (loglevel < 3):
+	        etpConst['remoteloglevel'] = loglevel
+	    else:
+		print "WARNING: invalid loglevel in: "+etpConst['remoteconf']
+		import time
+		time.sleep(5)
+
+	if line.startswith("httphandler|") and (len(line.split("|")) > 2):
+	    servername = line.split("httphandler|")[1]
+	    url = line.split("httphandler|")[2]
+	    if not url.endswith("/"):
+		url = url+"/"
+	    etpRemoteSupport[servername] = url
+
+
 # spmbackend section
 if (not os.path.isfile(etpConst['spmbackendconf'])):
     print "ERROR: "+etpConst['spmbackendconf']+" does not exist"

libraries/entropyTools.py

@@ -41,6 +41,16 @@ entropyLog = logTools.LogFile(level=etpConst['entropyloglevel'],filename = etpCo
 
 # EXIT STATUSES: 100-199
 
+global __etp_debug
+__etp_debug = False
+def enableDebug():
+    __etp_debug = True
+    import pdb
+    pdb.set_trace()
+
+def getDebug():
+    return __etp_debug
+
 def isRoot():
     import getpass
     if (getpass.getuser() == "root"):

libraries/mirrorTools.py

@@ -28,7 +28,6 @@ from outputTools import *
 import entropyTools
 import string
 import os
-import ftplib
 import time
 
 # Logging initialization
@@ -41,7 +40,20 @@ class handlerFTP:
 
     # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org
     # this must be run before calling the other functions
-    def __init__(self, ftpuri):
+    def __init__(self, ftpuri, debug = None):
+
+	# ftp debugging
+	if entropyTools.getDebug():
+	    debug = True
+	else:
+	    debug = False
+	# FIXME: remove this
+	#debug = True
+
+        # import FTP modules
+        import timeoutsocket
+        import ftplib
+        timeoutsocket.setDefaultSocketTimeout(60)
 
 	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.__init__: called.")
 
@@ -80,36 +92,24 @@ class handlerFTP:
 	    self.ftpdir = self.ftpdir[:len(self.ftpdir)-1]
 
 	self.ftpconn = ftplib.FTP(self.ftphost)
+	# enable debug?
+	if debug:
+	    mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.__init__: DEBUG enabled.")
+	    self.ftpconn.set_debuglevel(2)
+	
 	self.ftpconn.login(self.ftpuser,self.ftppassword)
 	# change to our dir
 	#print self.ftpdir
 	self.ftpconn.cwd(self.ftpdir)
 	self.currentdir = self.ftpdir
 
-    def my_storbinary(self, cmd, fp, callback=None):   # patched for callback
-        '''Store a file in line mode.'''
-        self.ftpconn.voidcmd('TYPE I')
-	CRLF = ftplib.CRLF
-        conn = self.ftpconn.transfercmd(cmd)
-        while 1:
-            buf = fp.readline()
-            if not buf: break
-            if buf[-2:] != CRLF:
-                if buf[-1] in CRLF: buf = buf[:-1]
-                buf = buf + CRLF
-            conn.sendall(buf)
-            if callback: callback(buf)                 # patched for callback
-        conn.close()
-        return self.ftpconn.voidresp()
-	#FTP.storbinary = my_storbinary  # use the patched version
-
     # this can be used in case of exceptions
     def reconnectHost(self):
 	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.reconnectHost: called.")
-	#try:
-	#    self.closeFTPConnection()
-	#except:
-	#    pass
+        # import FTP modules
+        import timeoutsocket
+        import ftplib
+        timeoutsocket.setDefaultSocketTimeout(60)
 	self.ftpconn = ftplib.FTP(self.ftphost)
 	self.ftpconn.login(self.ftpuser,self.ftppassword)
 	# save curr dir
@@ -117,16 +117,16 @@ class handlerFTP:
 	#self.setCWD(self.ftpdir)
 	self.setCWD(self.currentdir)
 
-    def getFTPHost(self):
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPHost: called -> "+self.ftphost)
+    def getHost(self):
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getHost: called -> "+self.ftphost)
 	return self.ftphost
 
-    def getFTPPort(self):
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpport)
+    def getPort(self):
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getPort: called -> "+self.ftpport)
 	return self.ftpport
 
-    def getFTPDir(self):
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpdir)
+    def getDir(self):
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.getDir: called -> "+self.ftpdir)
 	return self.ftpdir
 
     def getCWD(self):
@@ -148,16 +148,16 @@ class handlerFTP:
 	rc = self.ftpconn.sendcmd("mdtm "+path)
 	return rc.split()[len(rc.split())-1]
 
-    def spawnFTPCommand(self,cmd):
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnFTPCommand: called, command -> "+cmd)
+    def spawnCommand(self,cmd):
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnCommand: called, command -> "+cmd)
 	rc = self.ftpconn.sendcmd(cmd)
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnFTPCommand: called, rc -> "+str(rc))
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.spawnCommand: called, rc -> "+str(rc))
 	return rc
 
     # list files and directory of a FTP
     # @returns a list
-    def listFTPdir(self):
-	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.listFTPdir: called.")
+    def listDir(self):
+	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.listDir: called.")
 	# directory is: self.ftpdir
 	try:
 	    rc = self.ftpconn.nlst()
@@ -217,10 +217,16 @@ class handlerFTP:
             if callback: callback(buf)
 	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.advancedStorBinary: before conn.close()")
         conn.close()
-	time.sleep(3)
-	rc = self.ftpconn.voidresp()
 	mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.advancedStorBinary: after conn.close()")
-	return rc
+	
+	# that's another workaround
+	#return "226"
+	try:
+	    rc = self.ftpconn.voidresp()
+	except Timeout:
+	    mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"handlerFTP.advancedStorBinary: timeout receiving voidresp(), reconnecting...")
+	    self.reconnectHost()
+	    return "226"
 
     def uploadFile(self,file,ascii = False):
 	
@@ -243,11 +249,11 @@ class handlerFTP:
 	for i in range(10): # ten tries
 	
 	    mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.uploadFile: try #"+str(i+1))
-	
+	    filename = file.split("/")[len(file.split("/"))-1]
+
 	    try:
 	    
 	        f = open(file,"r")
-	        filename = file.split("/")[len(file.split("/"))-1]
 		
 	        # get file size
 	        self.myFileSize = round(float(os.stat(file)[6])/1024,1)
@@ -259,7 +265,7 @@ class handlerFTP:
 	        if (ascii):
 		    rc = self.ftpconn.storlines("STOR "+filename+".tmp",f)
 	        else:
-		    rc = self.my_storbinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress )
+		    rc = self.advancedStorBinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress )
 		
 	        # now we can rename the file with its original name
 	        self.renameFile(filename+".tmp",filename)
@@ -275,8 +281,11 @@ class handlerFTP:
 		    return False
 	    
 	    except Exception, e: # connection reset by peer
-		mirrorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"handlerFTP.uploadFile: Caught Exception: "+str(e)+" upload issues, retrying...")
-		print_info(red("Upload issue, retrying... #"+str(i+1)))
+		mirrorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"handlerFTP.uploadFile: Caught Exception: "+str(e)+", upload issues, retrying...")
+		import traceback
+		traceback.print_stack()
+		print_warning("")
+		print_warning(red("  Upload issue: ")+bold(str(e))+red(", retrying... #"+str(i+1)))
 		self.reconnectHost() # reconnect
 		mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.uploadFile: after reconnectHost()")
 		if self.isFileAvailable(filename):

libraries/remoteTools.py

@@ -0,0 +1,60 @@
+#!/usr/bin/python
+'''
+    # DESCRIPTION:
+    # Entropy Mirrors interface
+
+    Copyright (C) 2007 Fabio Erculiani
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+'''
+
+# Never do "import portage" here, please use entropyTools binding
+# EXIT STATUSES: 700-799
+
+from entropyConstants import *
+from outputTools import *
+import entropyTools
+
+# Logging initialization
+import logTools
+remoteLog = logTools.LogFile(level=etpConst['remoteloglevel'],filename = etpConst['remotelogfile'], header = "[REMOTE/HTTP]")
+# example: mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"testFuncton: called.")
+
+import timeoutsocket
+import urllib
+timeoutsocket.setDefaultSocketTimeout(60)
+
+# Get checksum of a package by running md5sum remotely (using php helpers)
+# @returns hex: if the file exists
+# @returns None: if the server does not support HTTP handlers
+# @returns False: if the file is not found
+def getRemotePackageChecksum(serverName,filename):
+    remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getRemotePackageChecksum: called.")
+    # etpHandlers['md5sum'] is the command
+    # create the request
+    try:
+	url = etpRemoteSupport[servername]
+    except:
+	# not found, does not support HTTP handlers
+	return None
+    
+    request = url+etpHandlers['md5sum']+filename
+    remoteLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getRemotePackageChecksum: requested url -> "+request)
+    
+    # now pray the server
+    file = urllib.urlopen(request)
+    result = file.readline().strip()
+    return result
+    

libraries/timeoutsocket.py

@@ -0,0 +1,424 @@
+
+####
+# Copyright 2000,2001 by Timothy O'Malley <timo@alum.mit.edu>
+# 
+#                All Rights Reserved
+# 
+# Permission to use, copy, modify, and distribute this software
+# and its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Timothy O'Malley  not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission. 
+# 
+# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
+# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE. 
+#
+####
+
+"""Timeout Socket
+
+This module enables a timeout mechanism on all TCP connections.  It
+does this by inserting a shim into the socket module.  After this module
+has been imported, all socket creation goes through this shim.  As a
+result, every TCP connection will support a timeout.
+
+The beauty of this method is that it immediately and transparently
+enables the entire python library to support timeouts on TCP sockets.
+As an example, if you wanted to SMTP connections to have a 20 second
+timeout:
+
+    import timeoutsocket
+    import smtplib
+    timeoutsocket.setDefaultSocketTimeout(20)
+
+
+The timeout applies to the socket functions that normally block on
+execution:  read, write, connect, and accept.  If any of these 
+operations exceeds the specified timeout, the exception Timeout
+will be raised.
+
+The default timeout value is set to None.  As a result, importing
+this module does not change the default behavior of a socket.  The
+timeout mechanism only activates when the timeout has been set to
+a numeric value.  (This behavior mimics the behavior of the
+select.select() function.)
+
+This module implements two classes: TimeoutSocket and TimeoutFile.
+
+The TimeoutSocket class defines a socket-like object that attempts to
+avoid the condition where a socket may block indefinitely.  The
+TimeoutSocket class raises a Timeout exception whenever the
+current operation delays too long. 
+
+The TimeoutFile class defines a file-like object that uses the TimeoutSocket
+class.  When the makefile() method of TimeoutSocket is called, it returns
+an instance of a TimeoutFile.
+
+Each of these objects adds two methods to manage the timeout value:
+
+    get_timeout()   -->  returns the timeout of the socket or file
+    set_timeout()   -->  sets the timeout of the socket or file
+
+
+As an example, one might use the timeout feature to create httplib
+connections that will timeout after 30 seconds:
+
+    import timeoutsocket
+    import httplib
+    H = httplib.HTTP("www.python.org")
+    H.sock.set_timeout(30)
+
+Note:  When used in this manner, the connect() routine may still
+block because it happens before the timeout is set.  To avoid
+this, use the 'timeoutsocket.setDefaultSocketTimeout()' function.
+
+Good Luck!
+
+"""
+
+__version__ = "$Revision: 1.23 $"
+__author__  = "Timothy O'Malley <timo@alum.mit.edu>"
+
+#
+# Imports
+#
+import select, string
+import socket
+if not hasattr(socket, "_no_timeoutsocket"):
+    _socket = socket.socket
+else:
+    _socket = socket._no_timeoutsocket
+
+
+#
+# Set up constants to test for Connected and Blocking operations.
+# We delete 'os' and 'errno' to keep our namespace clean(er).
+# Thanks to Alex Martelli and G. Li for the Windows error codes.
+#
+import os
+if os.name == "nt":
+    _IsConnected = ( 10022, 10056 )
+    _ConnectBusy = ( 10035, )
+    _AcceptBusy  = ( 10035, )
+else:
+    import errno
+    _IsConnected = ( errno.EISCONN, )
+    _ConnectBusy = ( errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK )
+    _AcceptBusy  = ( errno.EAGAIN, errno.EWOULDBLOCK )
+    del errno
+del os
+
+
+#
+# Default timeout value for ALL TimeoutSockets
+#
+_DefaultTimeout = None
+def setDefaultSocketTimeout(timeout):
+    global _DefaultTimeout
+    _DefaultTimeout = timeout
+def getDefaultSocketTimeout():
+    return _DefaultTimeout
+
+#
+# Exceptions for socket errors and timeouts
+#
+Error = socket.error
+class Timeout(Exception):
+    pass
+
+
+#
+# Factory function
+#
+from socket import AF_INET, SOCK_STREAM
+def timeoutsocket(family=AF_INET, type=SOCK_STREAM, proto=None):
+    if family != AF_INET or type != SOCK_STREAM:
+        if proto:
+            return _socket(family, type, proto)
+        else:
+            return _socket(family, type)
+    return TimeoutSocket( _socket(family, type), _DefaultTimeout )
+# end timeoutsocket
+
+#
+# The TimeoutSocket class definition
+#
+class TimeoutSocket:
+    """TimeoutSocket object
+    Implements a socket-like object that raises Timeout whenever
+    an operation takes too long.
+    The definition of 'too long' can be changed using the
+    set_timeout() method.
+    """
+
+    _copies = 0
+    _blocking = 1
+    
+    def __init__(self, sock, timeout):
+        self._sock     = sock
+        self._timeout  = timeout
+    # end __init__
+
+    def __getattr__(self, key):
+        return getattr(self._sock, key)
+    # end __getattr__
+
+    def get_timeout(self):
+        return self._timeout
+    # end set_timeout
+
+    def set_timeout(self, timeout=None):
+        self._timeout = timeout
+    # end set_timeout
+
+    def setblocking(self, blocking):
+        self._blocking = blocking
+        return self._sock.setblocking(blocking)
+    # end set_timeout
+
+    def connect_ex(self, addr):
+        errcode = 0
+        try:
+            self.connect(addr)
+        except Error, why:
+            errcode = why[0]
+        return errcode
+    # end connect_ex
+        
+    def connect(self, addr, port=None, dumbhack=None):
+        # In case we were called as connect(host, port)
+        if port != None:  addr = (addr, port)
+
+        # Shortcuts
+        sock    = self._sock
+        timeout = self._timeout
+        blocking = self._blocking
+
+        # First, make a non-blocking call to connect
+        try:
+            sock.setblocking(0)
+            sock.connect(addr)
+            sock.setblocking(blocking)
+            return
+        except Error, why:
+            # Set the socket's blocking mode back
+            sock.setblocking(blocking)
+            
+            # If we are not blocking, re-raise
+            if not blocking:
+                raise
+            
+            # If we are already connected, then return success.
+            # If we got a genuine error, re-raise it.
+            errcode = why[0]
+            if dumbhack and errcode in _IsConnected:
+                return
+            elif errcode not in _ConnectBusy:
+                raise
+            
+        # Now, wait for the connect to happen
+        # ONLY if dumbhack indicates this is pass number one.
+        #   If select raises an error, we pass it on.
+        #   Is this the right behavior?
+        if not dumbhack:
+            r,w,e = select.select([], [sock], [], timeout)
+            if w:
+                return self.connect(addr, dumbhack=1)
+
+        # If we get here, then we should raise Timeout
+        raise Timeout("Attempted connect to %s timed out." % str(addr) )
+    # end connect
+
+    def accept(self, dumbhack=None):
+        # Shortcuts
+        sock     = self._sock
+        timeout  = self._timeout
+        blocking = self._blocking
+
+        # First, make a non-blocking call to accept
+        #  If we get a valid result, then convert the
+        #  accept'ed socket into a TimeoutSocket.
+        # Be carefult about the blocking mode of ourselves.
+        try:
+            sock.setblocking(0)
+            newsock, addr = sock.accept()
+            sock.setblocking(blocking)
+            timeoutnewsock = self.__class__(newsock, timeout)
+            timeoutnewsock.setblocking(blocking)
+            return (timeoutnewsock, addr)
+        except Error, why:
+            # Set the socket's blocking mode back
+            sock.setblocking(blocking)
+
+            # If we are not supposed to block, then re-raise
+            if not blocking:
+                raise
+            
+            # If we got a genuine error, re-raise it.
+            errcode = why[0]
+            if errcode not in _AcceptBusy:
+                raise
+            
+        # Now, wait for the accept to happen
+        # ONLY if dumbhack indicates this is pass number one.
+        #   If select raises an error, we pass it on.
+        #   Is this the right behavior?
+        if not dumbhack:
+            r,w,e = select.select([sock], [], [], timeout)
+            if r:
+                return self.accept(dumbhack=1)
+
+        # If we get here, then we should raise Timeout
+        raise Timeout("Attempted accept timed out.")
+    # end accept
+
+    def send(self, data, flags=0):
+        sock = self._sock
+        if self._blocking:
+            r,w,e = select.select([],[sock],[], self._timeout)
+            if not w:
+                raise Timeout("Send timed out")
+        return sock.send(data, flags)
+    # end send
+
+    def recv(self, bufsize, flags=0):
+        sock = self._sock
+        if self._blocking:
+            r,w,e = select.select([sock], [], [], self._timeout)
+            if not r:
+                raise Timeout("Recv timed out")
+        return sock.recv(bufsize, flags)
+    # end recv
+
+    def makefile(self, flags="r", bufsize=-1):
+        self._copies = self._copies +1
+        return TimeoutFile(self, flags, bufsize)
+    # end makefile
+
+    def close(self):
+        if self._copies <= 0:
+            self._sock.close()
+        else:
+            self._copies = self._copies -1
+    # end close
+
+# end TimeoutSocket
+
+
+class TimeoutFile:
+    """TimeoutFile object
+    Implements a file-like object on top of TimeoutSocket.
+    """
+    
+    def __init__(self, sock, mode="r", bufsize=4096):
+        self._sock          = sock
+        self._bufsize       = 4096
+        if bufsize > 0: self._bufsize = bufsize
+        if not hasattr(sock, "_inqueue"): self._sock._inqueue = ""
+
+    # end __init__
+
+    def __getattr__(self, key):
+        return getattr(self._sock, key)
+    # end __getattr__
+
+    def close(self):
+        self._sock.close()
+        self._sock = None
+    # end close
+    
+    def write(self, data):
+        self.send(data)
+    # end write
+
+    def read(self, size=-1):
+        _sock = self._sock
+        _bufsize = self._bufsize
+        while 1:
+            datalen = len(_sock._inqueue)
+            if datalen >= size >= 0:
+                break
+            bufsize = _bufsize
+            if size > 0:
+                bufsize = min(bufsize, size - datalen )
+            buf = self.recv(bufsize)
+            if not buf:
+                break
+            _sock._inqueue = _sock._inqueue + buf
+        data = _sock._inqueue
+        _sock._inqueue = ""
+        if size > 0 and datalen > size:
+            _sock._inqueue = data[size:]
+            data = data[:size]
+        return data
+    # end read
+
+    def readline(self, size=-1):
+        _sock = self._sock
+        _bufsize = self._bufsize
+        while 1:
+            idx = string.find(_sock._inqueue, "\n")
+            if idx >= 0:
+                break
+            datalen = len(_sock._inqueue)
+            if datalen >= size >= 0:
+                break
+            bufsize = _bufsize
+            if size > 0:
+                bufsize = min(bufsize, size - datalen )
+            buf = self.recv(bufsize)
+            if not buf:
+                break
+            _sock._inqueue = _sock._inqueue + buf
+
+        data = _sock._inqueue
+        _sock._inqueue = ""
+        if idx >= 0:
+            idx = idx + 1
+            _sock._inqueue = data[idx:]
+            data = data[:idx]
+        elif size > 0 and datalen > size:
+            _sock._inqueue = data[size:]
+            data = data[:size]
+        return data
+    # end readline
+
+    def readlines(self, sizehint=-1):
+        result = []
+        data = self.read()
+        while data:
+            idx = string.find(data, "\n")
+            if idx >= 0:
+                idx = idx + 1
+                result.append( data[:idx] )
+                data = data[idx:]
+            else:
+                result.append( data )
+                data = ""
+        return result
+    # end readlines
+
+    def flush(self):  pass
+
+# end TimeoutFile
+
+
+#
+# Silently replace the socket() builtin function with
+# our timeoutsocket() definition.
+#
+if not hasattr(socket, "_no_timeoutsocket"):
+    socket._no_timeoutsocket = socket.socket
+    socket.socket = timeoutsocket
+del socket
+socket = timeoutsocket
+# Finis

server/activator

@@ -59,11 +59,13 @@ def print_help():
 
 options = sys.argv[1:]
 
-# no color parsing
+# preliminary options parsing
 _options = []
 for opt in options:
     if (opt == "--nocolor"):
 	entropyTools.nocolor()
+    elif (opt == "--debug"):
+	entropyTools.enableDebug()
     else:
 	_options.append(opt)
 options = _options

server/enzyme

@@ -86,11 +86,13 @@ def print_help():
 
 options = sys.argv[1:]
 
-# no color parsing
+# preliminary options parsing
 _options = []
 for opt in options:
     if (opt == "--nocolor"):
 	entropyTools.nocolor()
+    elif (opt == "--debug"):
+	entropyTools.enableDebug()
     else:
 	_options.append(opt)
 options = _options

server/reagent

@@ -65,11 +65,13 @@ def print_help():
 
 options = sys.argv[1:]
 
-# no color parsing
+# preliminary options parsing
 _options = []
 for opt in options:
     if (opt == "--nocolor"):
 	entropyTools.nocolor()
+    elif (opt == "--debug"):
+	entropyTools.enableDebug()
     else:
 	_options.append(opt)
 options = _options