diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index f9c387590..2462aec3e 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -98,7 +98,7 @@ def packages(options): print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) remotePackages = ftp.listFTPdir() remotePackagesInfo = ftp.getRoughList() @@ -246,7 +246,7 @@ def packages(options): # upload queue if (detailedUploadQueue != []): - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) uploadCounter = str(len(detailedUploadQueue)) currentCounter = 0 @@ -260,7 +260,7 @@ def packages(options): # download queue if (detailedDownloadQueue != []): - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) downloadCounter = str(len(detailedDownloadQueue)) currentCounter = 0 diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 2345f4f0c..7f22c85ed 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -5,7 +5,7 @@ Copyright (C) 2007 Fabio Erculiani - This program is free software; you can redistribute it and/or modify + This program is free software; you can entropyTools.redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. @@ -24,25 +24,27 @@ # EXIT STATUSES: 300-399 from entropyConstants import * -from entropyTools import * +import entropyTools from pysqlite2 import dbapi2 as sqlite #import commands #import re import os import sys +# TIP OF THE DAY: +# never nest closeDB() and re-init inside a loop !!!!!!!!!!!! NEVER ! def database(options): if len(options) == 0: - print_error(yellow(" * ")+red("Not enough parameters")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Not enough parameters")) sys.exit(301) if (options[0] == "--initialize"): # initialize the database - print_info(green(" * ")+red("Initializing Entropy database..."), back = True) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Initializing Entropy database..."), back = True) # database file: etpConst['etpdatabasefile'] if os.path.isfile(etpConst['etpdatabasefile']): - print_info(red(" * ")+bold("WARNING")+red(": database file already exists. Overwriting.")) + entropyTools.print_info(entropyTools.red(" * ")+entropyTools.bold("WARNING")+entropyTools.red(": database file already exists. Overwriting.")) rc = askquestion("\n Do you want to continue ?") if rc == "No": sys.exit(0) @@ -52,85 +54,85 @@ def database(options): dbconn = etpDatabase() dbconn.initializeDatabase() - print_info(green(" * ")+red("Reinitializing Entropy database using Portage database...")) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Reinitializing Entropy database using Portage database...")) # now run quickpkg for all the packages and then extract data installedAtoms, atomsnumber = getInstalledPackages() currCounter = 0 import reagentTools for atom in installedAtoms: currCounter += 1 - print_info(green(" (")+ blue(str(currCounter))+"/"+red(str(atomsnumber))+green(") ")+red("Analyzing ")+bold(atom)+red(" ...")) + entropyTools.print_info(entropyTools.green(" (")+ entropyTools.blue(str(currCounter))+"/"+entropyTools.red(str(atomsnumber))+entropyTools.green(") ")+entropyTools.red("Analyzing ")+entropyTools.bold(atom)+entropyTools.red(" ...")) quickpkg(atom,etpConst['packagestmpdir']) # file is etpConst['packagestmpdir']+"/atomscan/"+pkgnamever.tbz2 etpData = reagentTools.extractPkgData(etpConst['packagestmpdir']+"/"+atom.split("/")[1]+".tbz2") # fill the db entry dbconn.addPackage(etpData) os.system("rm -rf "+etpConst['packagestmpdir']+"/"+atom.split("/")[1]+"*") - dbconn.commitChanges() + dbconn.commitChanges() dbconn.closeDB() - print_info(green(" * ")+red("Entropy database has been reinitialized using Portage database entries")) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Entropy database has been reinitialized using Portage database entries")) elif (options[0] == "search"): mykeywords = options[1:] if (len(mykeywords) == 0): - print_error(yellow(" * ")+red("Not enough parameters")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Not enough parameters")) sys.exit(302) if (not os.path.isfile(etpConst['etpdatabasefile'])): - print_error(yellow(" * ")+red("Entropy Datbase does not exist")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Entropy Datbase does not exist")) sys.exit(303) # search tool - print_info(green(" * ")+red("Searching inside the Entropy database...")) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Searching inside the Entropy database...")) dbconn = etpDatabase() for mykeyword in mykeywords: results = dbconn.searchPackages(mykeyword) for result in results: print - print_info(green(" * ")+bold(result[0])) # package atom - print_info(red("\t Name: ")+blue(result[1])) - print_info(red("\t Installed version: ")+blue(result[2])) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.bold(result[0])) # package atom + entropyTools.print_info(entropyTools.red("\t Name: ")+entropyTools.blue(result[1])) + entropyTools.print_info(entropyTools.red("\t Installed version: ")+entropyTools.blue(result[2])) if (result[3]): - print_info(red("\t Description: ")+result[3]) - print_info(red("\t CHOST: ")+blue(result[5])) - print_info(red("\t CFLAGS: ")+darkred(result[6])) - print_info(red("\t CXXFLAGS: ")+darkred(result[7])) + entropyTools.print_info(entropyTools.red("\t Description: ")+result[3]) + entropyTools.print_info(entropyTools.red("\t CHOST: ")+entropyTools.blue(result[5])) + entropyTools.print_info(entropyTools.red("\t CFLAGS: ")+entropyTools.darkred(result[6])) + entropyTools.print_info(entropyTools.red("\t CXXFLAGS: ")+entropyTools.darkred(result[7])) if (result[8]): - print_info(red("\t Website: ")+result[8]) + entropyTools.print_info(entropyTools.red("\t Website: ")+result[8]) if (result[9]): - print_info(red("\t USE Flags: ")+blue(result[9])) - print_info(red("\t License: ")+bold(result[10])) - print_info(red("\t Source keywords: ")+darkblue(result[11])) - print_info(red("\t Binary keywords: ")+green(result[12])) - print_info(red("\t Package path: ")+result[13]) - print_info(red("\t Download relative URL: ")+result[14]) - print_info(red("\t Package Checksum: ")+green(result[15])) + entropyTools.print_info(entropyTools.red("\t USE Flags: ")+entropyTools.blue(result[9])) + entropyTools.print_info(entropyTools.red("\t License: ")+entropyTools.bold(result[10])) + entropyTools.print_info(entropyTools.red("\t Source keywords: ")+entropyTools.darkblue(result[11])) + entropyTools.print_info(entropyTools.red("\t Binary keywords: ")+entropyTools.green(result[12])) + entropyTools.print_info(entropyTools.red("\t Package path: ")+result[13]) + entropyTools.print_info(entropyTools.red("\t Download relative URL: ")+result[14]) + entropyTools.print_info(entropyTools.red("\t Package Checksum: ")+entropyTools.green(result[15])) if (result[16]): - print_info(red("\t Sources")) + entropyTools.print_info(entropyTools.red("\t Sources")) sources = result[16].split() for source in sources: - print_info(darkred("\t # Source package: ")+yellow(source)) + entropyTools.print_info(entropyTools.darkred("\t # Source package: ")+entropyTools.yellow(source)) if (result[17]): - print_info(red("\t Slot: ")+yellow(result[17])) - #print_info(red("\t Blah: ")+result[18]) # I don't need to print mirrorlinks + entropyTools.print_info(entropyTools.red("\t Slot: ")+entropyTools.yellow(result[17])) + #entropyTools.print_info(entropyTools.red("\t Blah: ")+result[18]) # I don't need to print mirrorlinks if (result[20]): deps = result[20].split() - print_info(red("\t Dependencies")) + entropyTools.print_info(entropyTools.red("\t Dependencies")) for dep in deps: - print_info(darkred("\t # Depends on: ")+dep) - #print_info(red("\t Blah: ")+result[20]) --> it's a dup of [21] + entropyTools.print_info(entropyTools.darkred("\t # Depends on: ")+dep) + #entropyTools.print_info(entropyTools.red("\t Blah: ")+result[20]) --> it's a dup of [21] if (result[22]): rundeps = result[22].split() - print_info(red("\t Built with runtime dependencies")) + entropyTools.print_info(entropyTools.red("\t Built with runtime dependencies")) for rundep in rundeps: - print_info(darkred("\t # Dependency: ")+rundep) + entropyTools.print_info(entropyTools.darkred("\t # Dependency: ")+rundep) if (result[23]): - print_info(red("\t Conflicts with")) + entropyTools.print_info(entropyTools.red("\t Conflicts with")) conflicts = result[23].split() for conflict in conflicts: - print_info(darkred("\t # Conflict: ")+conflict) - print_info(red("\t Entry API: ")+green(result[24])) - print_info(red("\t Entry revision: ")+str(result[25])) + entropyTools.print_info(entropyTools.darkred("\t # Conflict: ")+conflict) + entropyTools.print_info(entropyTools.red("\t Entry API: ")+entropyTools.green(result[24])) + entropyTools.print_info(entropyTools.red("\t Entry revision: ")+str(result[25])) #print result print dbconn.closeDB() @@ -138,28 +140,29 @@ def database(options): elif (options[0] == "dump-package-info"): mypackages = options[1:] if (len(mypackages) == 0): - print_error(yellow(" * ")+red("Not enough parameters")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Not enough parameters")) sys.exit(302) + + dbconn = etpDatabase() + for package in mypackages: - print_info(green(" * ")+red("Searching package ")+bold(package)+red(" ...")) - if isjustname(package) or (package.find("/") == -1): - print_warning(yellow(" * ")+red("Package ")+bold(package)+red(" is not a complete atom.")) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Searching package ")+entropyTools.bold(package)+entropyTools.red(" ...")) + if entropyTools.isjustname(package) or (package.find("/") == -1): + entropyTools.print_warning(entropyTools.yellow(" * ")+entropyTools.red("Package ")+entropyTools.bold(package)+entropyTools.red(" is not a complete atom.")) continue # open db connection - dbconn = etpDatabase() if (not dbconn.isPackageAvailable(package)): # package does not exist in the Entropy database - print_warning(yellow(" * ")+red("Package ")+bold(package)+red(" does not exist in Entropy database.")) - dbconn.closeDB() + entropyTools.print_warning(entropyTools.yellow(" * ")+entropyTools.red("Package ")+entropyTools.bold(package)+entropyTools.red(" does not exist in Entropy database.")) continue - myEtpData = tmpEtpData + myEtpData = entropyTools.etpData.copy() # reset for i in myEtpData: myEtpData[i] = "" - for i in tmpEtpData: + for i in myEtpData: myEtpData[i] = dbconn.retrievePackageVar(package,i) # sort and print @@ -169,43 +172,44 @@ def database(options): sortList = [] for i in myEtpData: sortList.append(i) - sortList = alphaSorter(sortList) + sortList = entropyTools.alphaSorter(sortList) for i in sortList: if (myEtpData[i]): f.write(i+": "+myEtpData[i]+"\n") f.flush() f.close() - print_info(green(" * ")+red("Dump generated in ")+bold(filepath)+red(" .")) - dbconn.closeDB() + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Dump generated in ")+entropyTools.bold(filepath)+entropyTools.red(" .")) + + dbconn.closeDB() elif (options[0] == "inject-package-info"): if (len(options[1:]) == 0): - print_error(yellow(" * ")+red("Not enough parameters")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Not enough parameters")) sys.exit(303) mypath = options[1:][0] if (not os.path.isfile(mypath)): - print_error(yellow(" * ")+red("File does not exist.")) + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("File does not exist.")) sys.exit(303) # revision is surely bumped - etpDataOut = parseEtpDump(mypath) + etpDataOut = entropyTools.parseEtpDump(mypath) dbconn = etpDatabase() updated, revision = dbconn.handlePackage(etpDataOut) dbconn.closeDB() if (updated) and (revision != 0): - print_info(green(" * ")+red("Package ")+bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+red(" entry has been updated. Revision: ")+bold(str(revision))) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Package ")+entropyTools.bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+entropyTools.red(" entry has been updated. Revision: ")+entropyTools.bold(str(revision))) elif (updated) and (revision == 0): - print_info(green(" * ")+red("Package ")+bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+red(" entry newly created.")) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Package ")+entropyTools.bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+entropyTools.red(" entry newly created.")) else: - print_info(green(" * ")+red("Package ")+bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+red(" does not need to be updated. Current revision: ")+bold(str(revision))) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Package ")+entropyTools.bold(etpDataOut['category']+"/"+etpDataOut['name']+"-"+etpDataOut['version'])+entropyTools.red(" does not need to be updated. Current revision: ")+entropyTools.bold(str(revision))) """ sortList = [] for i in etpDataOut: sortList.append(i) - sortList = alphaSorter(sortList) + sortList = entropyTools.alphaSorter(sortList) """ # print out the changes before doing them? @@ -213,21 +217,64 @@ def database(options): ############ -# Functions +# Functions and Classes ##################################################################################### +# this class simply describes the current database status +class databaseStatus: + + def __init__(self): + self.databaseBumped = False + self.databaseInfoCached = False + self.databaseLock = False + self.databaseDownloadLocl = False + self.databaseAlreadyTainted = False + + def isDatabaseAlreadyBumped(self): + return self.databaseBumped + + def isDatabaseAlreadyTainted(self): + return self.databaseAlreadyTainted + + def setDatabaseTaint(self,bool): + self.databaseAlreadyTainted = bool + + def setDatabaseBump(self,bool): + self.databaseBumped = bool + + def setDatabaseLock(self): + self.databaseLock = True + + def unsetDatabaseLock(self): + self.databaseLock = False + + def getDatabaseLock(self): + return self.databaseLock + + def setDatabaseDownloadLock(self): + self.databaseDownloadLock = True + + def unsetDatabaseDownloadLock(self): + self.databaseDownloadLock = False + + def getDatabaseDownloadLock(self): + return self.databaseDownloadLock class etpDatabase: def __init__(self): + # The first time you run this, sync the database and then lock + # FIXME: do this # initialization open the database connection self.connection = sqlite.connect(etpConst['etpdatabasefile']) self.cursor = self.connection.cursor() def closeDB(self): - if (self.isDatabaseTainted()): - # bump revision + # FIXME verify all this shit, for now it works... + if (entropyTools.dbStatus.isDatabaseAlreadyTainted()) and (not entropyTools.dbStatus.isDatabaseAlreadyBumped()): + # bump revision, setting DatabaseBump causes the session to just bump once + entropyTools.dbStatus.setDatabaseBump(True) self.revisionBump() self.cursor.close() self.connection.close() @@ -242,8 +289,10 @@ class etpDatabase: f.write(etpConst['currentarch']+" database tainted\n") f.flush() f.close() + entropyTools.dbStatus.setDatabaseTaint(True) def untaintDatabase(self): + entropyTools.dbStatus.setDatabaseTaint(False) # untaint the database status os.system("rm -f "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) @@ -280,7 +329,8 @@ class etpDatabase: if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])): update, revision = self.addPackage(etpData) else: - update, revision = self.updatePackage(etpData,forceBump) + print "update" + update, revision = self.updatePackage(etpData,forceBump) return update, revision def addPackage(self,etpData, revision = 0): @@ -342,7 +392,7 @@ class etpDatabase: # You must provide the full atom to this function def removePackage(self,key): - key = removePackageOperators(key) + key = entropyTools.removePackageOperators(key) self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'"') self.commitChanges() @@ -361,15 +411,20 @@ class etpDatabase: for i in myEtpData: myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i) + print etpData + print + print myEtpData + for i in etpData: if etpData[i] != myEtpData[i]: + print "differs" return False return True # You must provide the full atom to this function def retrievePackageInfo(self,pkgkey): - pkgkey = removePackageOperators(pkgkey) + pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "'+pkgkey+'"') for row in self.cursor: @@ -378,7 +433,7 @@ class etpDatabase: # You must provide the full atom to this function def retrievePackageVar(self,pkgkey,pkgvar): - pkgkey = removePackageOperators(pkgkey) + pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] self.cursor.execute('SELECT '+pkgvar+' FROM etpData WHERE atom LIKE "'+pkgkey+'"') for row in self.cursor: @@ -387,7 +442,7 @@ class etpDatabase: # You must provide the full atom to this function def isPackageAvailable(self,pkgkey): - pkgkey = removePackageOperators(pkgkey) + pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "'+pkgkey+'"') for row in self.cursor: @@ -401,4 +456,190 @@ class etpDatabase: self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "%'+keyword+'%"') for row in self.cursor: result.append(row) - return result \ No newline at end of file + return result + + +# ------ BEGIN: activator tools ------ + +class handlerFTP: + + # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org + # this must be run before calling the other functions + def __init__(self, ftpuri): + + from ftplib import FTP + + self.ftpuri = ftpuri + + self.ftphost = extractFTPHostFromUri(self.ftpuri) + + self.ftpuser = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1].split(":")[0] + if (self.ftpuser == ""): + self.ftpuser = "anonymous@" + self.ftppassword = "anonymous" + else: + self.ftppassword = ftpuri.split("@")[:len(ftpuri.split("@"))-1] + if len(self.ftppassword) > 1: + import string + self.ftppassword = string.join(self.ftppassword,"@") + self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] + if (self.ftppassword == ""): + self.ftppassword = "anonymous" + else: + self.ftppassword = self.ftppassword[0] + self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] + if (self.ftppassword == ""): + self.ftppassword = "anonymous" + + self.ftpport = ftpuri.split(":")[len(ftpuri.split(":"))-1] + try: + self.ftpport = int(self.ftpport) + except: + self.ftpport = 21 + + self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1] + self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1] + self.ftpdir = self.ftpdir.split(":")[0] + if self.ftpdir.endswith("/"): + self.ftpdir = self.ftpdir[:len(self.ftpdir)-1] + + self.ftpconn = FTP(self.ftphost) + self.ftpconn.login(self.ftpuser,self.ftppassword) + # change to our dir + self.ftpconn.cwd(self.ftpdir) + self.currentdir = self.ftpdir + + + # this can be used in case of exceptions + def reconnectHost(self): + self.ftpconn = FTP(self.ftphost) + self.ftpconn.login(self.ftpuser,self.ftppassword) + self.ftpconn.cwd(self.currentdir) + + def getFTPHost(self): + return self.ftphost + + def getFTPPort(self): + return self.ftpport + + def getFTPDir(self): + return self.ftpdir + + def getCWD(self): + return self.ftpconn.pwd() + + def setCWD(self,dir): + self.ftpconn.cwd(dir) + self.currentdir = dir + + def getFileMtime(self,path): + rc = self.ftpconn.sendcmd("mdtm "+path) + return rc.split()[len(rc.split())-1] + + def spawnFTPCommand(self,cmd): + rc = self.ftpconn.sendcmd(cmd) + return rc + + # list files and directory of a FTP + # @returns a list + def listFTPdir(self): + # directory is: self.ftpdir + try: + rc = self.ftpconn.nlst() + _rc = [] + for i in rc: + _rc.append(i.split("/")[len(i.split("/"))-1]) + rc = _rc + except: + return [] + return rc + + # list if the file is available + # @returns True or False + def isFileAvailable(self,filename): + # directory is: self.ftpdir + try: + rc = self.ftpconn.nlst() + _rc = [] + for i in rc: + _rc.append(i.split("/")[len(i.split("/"))-1]) + rc = _rc + for i in rc: + if i == filename: + return True + return False + except: + return False + + def deleteFile(self,file): + try: + rc = self.ftpconn.delete(file) + if rc.startswith("250"): + return True + else: + return False + except: + return False + + def uploadFile(self,file,ascii = False): + for i in range(10): # ten tries + f = open(file) + filename = file.split("/")[len(file.split("/"))-1] + try: + if (ascii): + rc = self.ftpconn.storlines("STOR "+filename+".tmp",f) + else: + rc = self.ftpconn.storbinary("STOR "+filename+".tmp",f) + # now we can rename the file with its original name + self.renameFile(filename+".tmp",filename) + return rc + except socket.error: # connection reset by peer + entropyTools.print_info(entropyTools.red("Upload issue, retrying...")) + self.reconnectHost() # reconnect + self.deleteFile(filename) + self.deleteFile(filename+".tmp") + f.close() + continue + + def downloadFile(self,filepath,downloaddir,ascii = False): + file = filepath.split("/")[len(filepath.split("/"))-1] + if (not ascii): + f = open(downloaddir+"/"+file,"wb") + rc = self.ftpconn.retrbinary('RETR '+file,f.write) + else: + f = open(downloaddir+"/"+file,"w") + rc = self.ftpconn.retrlines('RETR '+file,f.write) + f.flush() + f.close() + return rc + + # also used to move files + # FIXME: beautify ! + def renameFile(self,fromfile,tofile): + self.ftpconn.rename(fromfile,tofile) + + # not supported by dreamhost.com + def getFileSize(self,file): + return self.ftpconn.size(file) + + def getFileSizeCompat(self,file): + list = getRoughList() + for item in list: + if item.find(file) != -1: + # extact the size + return item.split()[4] + return "" + + def bufferizer(self,buf): + self.FTPbuffer.append(buf) + + def getRoughList(self): + self.FTPbuffer = [] + self.ftpconn.dir(self.bufferizer) + return self.FTPbuffer + + def closeFTPConnection(self): + self.ftpconn.quit() + +# ------ END: activator tools ------ + diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index adbfd91d9..e2134586c 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -176,6 +176,7 @@ if not os.path.isdir(ETP_DIR): print "you need to run this as root at least once." sys.exit(100) +# FIXME: IS THIS REALLY USED BY ANYTHING ???? etpSources = { 'packagesuri': "", # URIs where are stored binary packages 'databaseuri': "", # URIs where are stored entropy files diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 1e74bcc4c..acd07d1a5 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -38,7 +38,6 @@ import portage import portage_const from portage_dep import isvalidatom, isspecific, isjustname, dep_getkey, dep_getcpv from entropyConstants import * - initializePortageTree() # colours support @@ -49,6 +48,10 @@ import sys import random import commands +# Instantiate the databaseStatus: +import databaseTools +dbStatus = databaseTools.databaseStatus() + # EXIT STATUSES: 100-199 def getArchFromChost(chost): @@ -762,190 +765,6 @@ def getPortageAppDbPath(): return rc+"/" return rc -# ------ BEGIN: activator tools ------ - -class activatorFTP: - - # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org - # this must be run before calling the other functions - def __init__(self, ftpuri): - - from ftplib import FTP - - self.ftpuri = ftpuri - - self.ftphost = extractFTPHostFromUri(self.ftpuri) - - self.ftpuser = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1].split(":")[0] - if (self.ftpuser == ""): - self.ftpuser = "anonymous@" - self.ftppassword = "anonymous" - else: - self.ftppassword = ftpuri.split("@")[:len(ftpuri.split("@"))-1] - if len(self.ftppassword) > 1: - import string - self.ftppassword = string.join(self.ftppassword,"@") - self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] - if (self.ftppassword == ""): - self.ftppassword = "anonymous" - else: - self.ftppassword = self.ftppassword[0] - self.ftppassword = self.ftppassword.split(":")[len(self.ftppassword.split(":"))-1] - if (self.ftppassword == ""): - self.ftppassword = "anonymous" - - self.ftpport = ftpuri.split(":")[len(ftpuri.split(":"))-1] - try: - self.ftpport = int(self.ftpport) - except: - self.ftpport = 21 - - self.ftpdir = ftpuri.split("ftp://")[len(ftpuri.split("ftp://"))-1] - self.ftpdir = self.ftpdir.split("/")[len(self.ftpdir.split("/"))-1] - self.ftpdir = self.ftpdir.split(":")[0] - if self.ftpdir.endswith("/"): - self.ftpdir = self.ftpdir[:len(self.ftpdir)-1] - - self.ftpconn = FTP(self.ftphost) - self.ftpconn.login(self.ftpuser,self.ftppassword) - # change to our dir - self.ftpconn.cwd(self.ftpdir) - self.currentdir = self.ftpdir - - - # this can be used in case of exceptions - def reconnectHost(self): - self.ftpconn = FTP(self.ftphost) - self.ftpconn.login(self.ftpuser,self.ftppassword) - self.ftpconn.cwd(self.currentdir) - - def getFTPHost(self): - return self.ftphost - - def getFTPPort(self): - return self.ftpport - - def getFTPDir(self): - return self.ftpdir - - def getCWD(self): - return self.ftpconn.pwd() - - def setCWD(self,dir): - self.ftpconn.cwd(dir) - self.currentdir = dir - - def getFileMtime(self,path): - rc = self.ftpconn.sendcmd("mdtm "+path) - return rc.split()[len(rc.split())-1] - - def spawnFTPCommand(self,cmd): - rc = self.ftpconn.sendcmd(cmd) - return rc - - # list files and directory of a FTP - # @returns a list - def listFTPdir(self): - # directory is: self.ftpdir - try: - rc = self.ftpconn.nlst() - _rc = [] - for i in rc: - _rc.append(i.split("/")[len(i.split("/"))-1]) - rc = _rc - except: - return [] - return rc - - # list if the file is available - # @returns True or False - def isFileAvailable(self,filename): - # directory is: self.ftpdir - try: - rc = self.ftpconn.nlst() - _rc = [] - for i in rc: - _rc.append(i.split("/")[len(i.split("/"))-1]) - rc = _rc - for i in rc: - if i == filename: - return True - return False - except: - return False - - def deleteFile(self,file): - try: - rc = self.ftpconn.delete(file) - if rc.startswith("250"): - return True - else: - return False - except: - return False - - def uploadFile(self,file,ascii = False): - for i in range(10): # ten tries - f = open(file) - filename = file.split("/")[len(file.split("/"))-1] - try: - if (ascii): - rc = self.ftpconn.storlines("STOR "+filename+".tmp",f) - else: - rc = self.ftpconn.storbinary("STOR "+filename+".tmp",f) - # now we can rename the file with its original name - self.renameFile(filename+".tmp",filename) - return rc - except socket.error: # connection reset by peer - print_info(red("Upload issue, retrying...")) - self.reconnectHost() # reconnect - self.deleteFile(filename) - self.deleteFile(filename+".tmp") - f.close() - continue - - def downloadFile(self,filepath,downloaddir,ascii = False): - file = filepath.split("/")[len(filepath.split("/"))-1] - if (not ascii): - f = open(downloaddir+"/"+file,"wb") - rc = self.ftpconn.retrbinary('RETR '+file,f.write) - else: - f = open(downloaddir+"/"+file,"w") - rc = self.ftpconn.retrlines('RETR '+file,f.write) - f.flush() - f.close() - return rc - - # also used to move files - # FIXME: beautify ! - def renameFile(self,fromfile,tofile): - self.ftpconn.rename(fromfile,tofile) - - # not supported by dreamhost.com - def getFileSize(self,file): - return self.ftpconn.size(file) - - def getFileSizeCompat(self,file): - list = getRoughList() - for item in list: - if item.find(file) != -1: - # extact the size - return item.split()[4] - return "" - - def bufferizer(self,buf): - self.FTPbuffer.append(buf) - - def getRoughList(self): - self.FTPbuffer = [] - self.ftpconn.dir(self.bufferizer) - return self.FTPbuffer - - def closeFTPConnection(self): - self.ftpconn.quit() - -# ------ END: activator tools ------ - def extractFTPHostFromUri(uri): ftphost = uri.split("ftp://")[len(uri.split("ftp://"))-1] ftphost = ftphost.split("@")[len(ftphost.split("@"))-1] @@ -958,7 +777,7 @@ def getEtpRemoteDatabaseStatus(): uriDbInfo = [] for uri in etpConst['activatoruploaduris']: - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) ftp.setCWD(etpConst['etpurirelativepath']) rc = ftp.isFileAvailable(translateArchFromUname(ETP_ARCH_CONST)+etpConst['etpdatabasefile']) if (rc): @@ -1143,7 +962,7 @@ def uploadDatabase(uris,dbfile): downloadLockDatabases(True,[uri]) print_info(green(" * ")+red("Uploading database to ")+bold(extractFTPHostFromUri(uri))+red(" ...")) print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True) - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) print_info(green(" * ")+red("Changing directory to ")+bold(etpConst['etpurirelativepath'])+red(" ..."), back = True) ftp.setCWD(etpConst['etpurirelativepath']) print_info(green(" * ")+red("Uploading file ")+bold(dbfile)+red(" ..."), back = True) @@ -1173,7 +992,7 @@ def uploadDatabase(uris,dbfile): def downloadDatabase(uri,dbfile): print_info(green(" * ")+red("Downloading database from ")+bold(extractFTPHostFromUri(uri))+red(" ...")) print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True) - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) print_info(green(" * ")+red("Changing directory to ")+bold(etpConst['etpurirelativepath'])+red(" ..."), back = True) ftp.setCWD(etpConst['etpurirelativepath']) print_info(green(" * ")+red("Downloading file to ")+bold(dbfile)+red(" ..."), back = True) @@ -1217,7 +1036,7 @@ def getMirrorsLock(): dbstatus = [] for uri in etpConst['activatoruploaduris']: data = [ uri, False , False ] - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) ftp.setCWD(etpConst['etpurirelativepath']) if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])): # Upload is locked @@ -1292,7 +1111,7 @@ def lockDatabases(lock = True, mirrorList = []): print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror..."),back = True) else: print_info(yellow(" * ")+red("Unlocking ")+bold(extractFTPHostFromUri(uri))+red(" mirror..."),back = True) - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) # upload the lock file to database/%ARCH% directory ftp.setCWD(etpConst['etpurirelativepath']) # check if the lock is already there @@ -1338,7 +1157,7 @@ def downloadLockDatabases(lock = True, mirrorList = []): print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True) else: print_info(yellow(" * ")+red("Unlocking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True) - ftp = activatorFTP(uri) + ftp = databaseTools.handlerFTP(uri) # upload the lock file to database/%ARCH% directory ftp.setCWD(etpConst['etpurirelativepath']) # check if the lock is already there diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 03de352cc..47b56db8a 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -26,7 +26,6 @@ from entropyConstants import * from entropyTools import * -import databaseTools import commands import re import sys @@ -241,6 +240,7 @@ def extractPkgData(package): etpData['sources'] += "="+atom[:len(atom)-1]+"|" elif (not atom.startswith("(")) and (not atom.startswith(")")): etpData['sources'] += atom+" " + etpData['sources'] = removeSpaceAtTheEnd(etpData['sources']) except IOError: etpData['sources'] = ""