sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more work on logTools and logging implementation

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@308 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-06-06 20:14:59 +00:00
parent 7710d29721
commit 768f38abe2
10 changed files with 331 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO
+1 -4
View File
@@ -1,9 +1,6 @@
TODO list (for developers only):
- implement logTools inside handlers
- activatorTools
- reagentTools
- mirrorTools
- add option to cleanup logs
- enzymeTools
- reagent: complete smartapps section
- enzyme, reagent: test kernel dependent packages
- enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails
conf/mirrors.conf
+8
View File
@@ -0,0 +1,8 @@
# Project Entropy 1.0 Mirrors configuration file
# Log level
# 0: No Logging
# 1: Normal Logging
# 2: Verbose Logging
loglevel|1
libraries/activatorTools.py
+64 -10
View File
@@ -36,9 +36,12 @@ import time
# Logging initialization
import logTools
activatorLog = logTools.LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]")
# example: activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.")
def sync(options, justTidy = False):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: called with justTidy -> "+str(justTidy))
print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ..."))
if (not justTidy):
@@ -85,6 +88,7 @@ def sync(options, justTidy = False):
removeList.append(repoBin)
if (removeList == []):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: no packages to remove from the lirrors.")
print_info(green(" * ")+red("No packages to remove from the mirrors."))
print_info(green(" * ")+red("Syncronization across mirrors completed."))
return
@@ -99,11 +103,14 @@ def sync(options, justTidy = False):
sys.exit(0)
# remove them!
activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: starting to remove packages from mirrors.")
for uri in etpConst['activatoruploaduris']:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: connecting to mirror "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Connecting to: ")+bold(extractFTPHostFromUri(uri)))
ftp = mirrorTools.handlerFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
for file in removeList:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: removing (remote) file "+file)
print_info(green(" * ")+red("Removing file: ")+bold(file), back = True)
# remove remotely
if (ftp.isFileAvailable(file)):
@@ -114,6 +121,7 @@ def sync(options, justTidy = False):
print_warning(yellow(" * ")+red("ATTENTION: remote file ")+bold(file)+red(" cannot be removed."))
# remove locally
if os.path.isfile(etpConst['packagesbindir']+"/"+file):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: removing (local) file "+file)
print_info(green(" * ")+red("Package file: ")+bold(file)+red(" removed successfully from ")+bold(etpConst['packagesbindir']))
os.remove(etpConst['packagesbindir']+"/"+file)
ftp.closeFTPConnection()
@@ -123,6 +131,8 @@ def sync(options, justTidy = False):
def packages(options):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: called with options -> "+str(options))
# Options available for all the packages submodules
myopts = options[1:]
activatorRequestAsk = False
@@ -144,6 +154,8 @@ def packages(options):
currentUri = 0
totalSuccessfulUri = 0
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: called sync.")
for uri in etpConst['activatoruploaduris']:
currentUri += 1
@@ -159,6 +171,9 @@ def packages(options):
toBeUploaded.append(tbz2)
if tbz2.endswith(".tbz2"):
uploadCounter += 1
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: upload directory stats -> files: "+str(uploadCounter)+" | upload packages list: "+str(toBeUploaded))
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
localPackagesRepository = [] # parse etpConst['packagesbindir']
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
@@ -168,6 +183,9 @@ def packages(options):
localPackagesRepository.append(tbz2)
if tbz2.endswith(".tbz2"):
packageCounter += 1
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: packages directory stats -> files: "+str(packageCounter)+" | download packages list (including md5): "+str(localPackagesRepository))
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
@@ -182,13 +200,18 @@ def packages(options):
for tbz2 in remotePackages:
if tbz2.endswith(".tbz2"):
remoteCounter += 1
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: remote packages stats -> files: "+str(remoteCounter))
print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
print_info(green(" * ")+yellow("Calculating..."))
uploadQueue = []
downloadQueue = []
removalQueue = []
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: starting packages calculation...")
# if a package is in the packages directory but not online, we have to upload it
# we have localPackagesRepository and remotePackages
for localPackage in localPackagesRepository:
@@ -320,9 +343,9 @@ def packages(options):
# filter duplicates
removalQueue = list(set(removalQueue))
downloadQueue = list(set(downloadQueue))
uploadQueue = list(set(uploadQueue))
removalQueue = filterDuplicatedEntries(removalQueue)
downloadQueue = filterDuplicatedEntries(downloadQueue)
uploadQueue = filterDuplicatedEntries(uploadQueue)
# order alphabetically
if (removalQueue != []):
@@ -346,6 +369,10 @@ def packages(options):
_downloadQueue.append(p)
downloadQueue = _downloadQueue
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: removal queue -> "+str(removalQueue))
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: download queue -> "+str(downloadQueue))
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: upload queue -> "+str(uploadQueue))
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0):
print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty.")))
totalSuccessfulUri += 1
@@ -421,6 +448,12 @@ def packages(options):
uploadCounter = "0"
downloadCounter = "0"
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed removal queue -> "+str(detailedRemovalQueue))
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed simple copy queue -> "+str(simpleCopyQueue))
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed download queue -> "+str(detailedDownloadQueue))
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed upload queue -> "+str(detailedUploadQueue))
# removal queue
if (detailedRemovalQueue != []):
for item in detailedRemovalQueue:
@@ -501,12 +534,16 @@ def packages(options):
# trap exceptions, failed to upload/download someting?
except:
activatorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"packages: cannot properly syncronize "+extractFTPHostFromUri(uri)+". Trying to continue if possible.")
# print warning cannot sync uri
print_warning(yellow(" * ")+red("ATTENTION: cannot properly syncronize ")+bold(extractFTPHostFromUri(uri))+red(". Continuing if possible..."))
# decide what to do
if (totalSuccessfulUri > 0) or (activatorRequestPretend):
# we're safe
activatorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"packages: at least one mirror has been synced properly. I'm fine.")
print_info(green(" * ")+red("At least one mirror has been synced properly. I'm fine."))
continue
else:
@@ -516,13 +553,15 @@ def packages(options):
else:
# no mirrors were synced properly
# show error and return, do not move files from the upload dir
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"packages: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.")
print_error(yellow(" * ")+red("ERROR: no mirrors have been properly syncronized. Check network status and retry. Cannot continue."))
return False
# if at least one server has been synced successfully, move files
if (totalSuccessfulUri > 0) and (not activatorRequestPretend):
# now we can store the files in upload/%ARCH% in packages/%ARCH%
# now we can store the files in upload/%ARCH% in packages/%ARCH%
activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"packages: all done. Now it's time to move packages to "+etpConst['packagesbindir'])
os.system("mv -f "+etpConst['packagessuploaddir']+"/* "+etpConst['packagesbindir']+"/ &> /dev/null")
return True
else:
@@ -536,13 +575,17 @@ def packages(options):
def database(options):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: called with -> "+str(options))
# lock tool
if (options[0] == "lock"):
print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
rc = lockDatabases(lock = True)
if (rc):
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (lock) A problem occured on at least one mirror !")
print_info(green(" * ")+red("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases lock complete"))
# unlock tool
@@ -550,8 +593,10 @@ def database(options):
print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
rc = lockDatabases(lock = False)
if (rc):
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases unlock complete"))
# download lock tool
@@ -559,8 +604,10 @@ def database(options):
print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
rc = downloadLockDatabases(lock = True)
if (rc):
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (download-lock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Download mirrors lock complete.")
print_info(green(" * ")+green("Download mirrors lock complete"))
# download unlock tool
@@ -568,8 +615,10 @@ def database(options):
print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
rc = downloadLockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
else:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Download mirrors unlock complete.")
print_info(green(" * ")+green("Download mirrors unlock complete"))
# lock status tool
@@ -589,7 +638,9 @@ def database(options):
# database sync tool
elif (options[0] == "sync"):
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: database sync called.")
print_info(green(" * ")+red("Checking database status ..."), back = True)
dbLockFile = False
@@ -622,14 +673,17 @@ def database(options):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
else:
print
print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ..."))
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...")
print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later..."))
sys.exit(422)
else:
if (dbLockFile):
activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']))
sys.exit(423)
else:
activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...")
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
syncRemoteDatabases()
libraries/databaseTools.py
+73 -73
View File
@@ -704,7 +704,7 @@ class databaseStatus:
def __init__(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus.__init__ called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus.__init__ called.")
self.databaseBumped = False
self.databaseInfoCached = False
@@ -714,103 +714,103 @@ class databaseStatus:
self.databaseAlreadyTainted = False
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: database tainted.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: database tainted.")
self.databaseAlreadyTainted = True
def isDatabaseAlreadyBumped(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: already bumped? "+str(self.databaseBumped))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: already bumped? "+str(self.databaseBumped))
return self.databaseBumped
def isDatabaseAlreadyTainted(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: tainted? "+str(self.databaseAlreadyTainted))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: tainted? "+str(self.databaseAlreadyTainted))
return self.databaseAlreadyTainted
def setDatabaseTaint(self,bool):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: setting database taint to: "+str(bool))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: setting database taint to: "+str(bool))
self.databaseAlreadyTainted = bool
def setDatabaseBump(self,bool):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: setting database bump to: "+str(bool))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: setting database bump to: "+str(bool))
self.databaseBumped = bool
def setDatabaseLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (upload)")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (upload)")
self.databaseLock = True
def unsetDatabaseLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (upload)")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (upload)")
self.databaseLock = False
def getDatabaseLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (upload), status: "+str(self.databaseLock))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (upload), status: "+str(self.databaseLock))
return self.databaseLock
def setDatabaseDownloadLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (download)")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (download)")
self.databaseDownloadLock = True
def unsetDatabaseDownloadLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (download)")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (download)")
self.databaseDownloadLock = False
def getDatabaseDownloadLock(self):
dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (download), status: "+str(self.databaseDownloadLock))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (download), status: "+str(self.databaseDownloadLock))
return self.databaseDownloadLock
class etpDatabase:
def __init__(self, readOnly = False, noUpload = False):
dbLog.log(ETP_LOG_VERBOSE,"etpDatabase.__init__ called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase.__init__ called.")
self.readOnly = readOnly
self.noUpload = noUpload
if (self.readOnly):
dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: database opened readonly")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: database opened readonly")
# if the database is opened readonly, we don't need to lock the online status
self.connection = sqlite.connect(etpConst['etpdatabasefilepath'])
self.cursor = self.connection.cursor()
# set the table read only
return
dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: database opened in read/write mode")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: database opened in read/write mode")
# check if the database is locked locally
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
dbLog.log(ETP_LOG_NORMAL,"etpDatabase: database already locked")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"etpDatabase: database already locked")
print_info(red(" * ")+red(" Entropy database is already locked by you :-)"))
else:
# check if the database is locked REMOTELY
dbLog.log(ETP_LOG_NORMAL,"etpDatabase: starting to lock and sync database")
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"etpDatabase: starting to lock and sync database")
print_info(red(" * ")+red(" Locking and Syncing Entropy database ..."), back = True)
for uri in etpConst['activatoruploaduris']:
dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: connecting to "+uri)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: connecting to "+uri)
ftp = mirrorTools.handlerFTP(uri)
ftp.setCWD(etpConst['etpurirelativepath'])
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) and (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])):
import time
print_info(red(" * ")+bold("WARNING")+red(": online database is already locked. Waiting up to 2 minutes..."), back = True)
dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database already locked. Waiting 2 minutes")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"etpDatabase: online database already locked. Waiting 2 minutes")
unlocked = False
for x in range(120):
time.sleep(1)
if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database has been unlocked !")
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"etpDatabase: online database has been unlocked !")
print_info(red(" * ")+bold("HOORAY")+red(": online database has been unlocked. Locking back and syncing..."))
unlocked = True
break
if (unlocked):
break
dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database has not been unlocked in time. Giving up.")
dbLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"etpDatabase: online database has not been unlocked in time. Giving up.")
# time over
print_info(red(" * ")+bold("ERROR")+red(": online database has not been unlocked. Giving up. Who the hell is working on it? Damn, it's so frustrating for me. I'm a piece of python code with a soul dude!"))
# FIXME show the lock status
print_info(yellow(" * ")+green("Mirrors status table:"))
dbstatus = entropyTools.getMirrorsLock()
dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: showing mirrors status table:")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: showing mirrors status table:")
for db in dbstatus:
if (db[1]):
db[1] = red("Locked")
@@ -820,7 +820,7 @@ class etpDatabase:
db[2] = red("Locked")
else:
db[2] = green("Unlocked")
dbLog.log(ETP_LOG_VERBOSE," "+entropyTools.extractFTPHostFromUri(db[0])+": DATABASE: "+db[1]+" | DOWNLOAD: "+db[2])
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE," "+entropyTools.extractFTPHostFromUri(db[0])+": DATABASE: "+db[1]+" | DOWNLOAD: "+db[2])
print_info(bold("\t"+entropyTools.extractFTPHostFromUri(db[0])+": ")+red("[")+yellow("DATABASE: ")+db[1]+red("] [")+yellow("DOWNLOAD: ")+db[2]+red("]"))
ftp.closeFTPConnection()
@@ -840,13 +840,13 @@ class etpDatabase:
# if the class is opened readOnly, close and forget
if (self.readOnly):
dbLog.log(ETP_LOG_VERBOSE,"closeDB: closing database opened in readonly.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"closeDB: closing database opened in readonly.")
#self.connection.rollback()
self.cursor.close()
self.connection.close()
return
dbLog.log(ETP_LOG_VERBOSE,"closeDB: closing database opened in read/write.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"closeDB: closing database opened in read/write.")
# FIXME verify all this shit, for now it works...
if (entropyTools.dbStatus.isDatabaseAlreadyTainted()) and (not entropyTools.dbStatus.isDatabaseAlreadyBumped()):
@@ -865,15 +865,15 @@ class etpDatabase:
def commitChanges(self):
if (not self.readOnly):
dbLog.log(ETP_LOG_VERBOSE,"commitChanges: writing changes to database.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"commitChanges: writing changes to database.")
self.connection.commit()
self.taintDatabase()
else:
dbLog.log(ETP_LOG_VERBOSE,"commitChanges: discarding changes to database (opened readonly).")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"commitChanges: discarding changes to database (opened readonly).")
self.discardChanges() # is it ok?
def taintDatabase(self):
dbLog.log(ETP_LOG_VERBOSE,"taintDatabase: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"taintDatabase: called.")
# taint the database status
f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'],"w")
f.write(etpConst['currentarch']+" database tainted\n")
@@ -882,13 +882,13 @@ class etpDatabase:
entropyTools.dbStatus.setDatabaseTaint(True)
def untaintDatabase(self):
dbLog.log(ETP_LOG_VERBOSE,"untaintDatabase: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"untaintDatabase: called.")
entropyTools.dbStatus.setDatabaseTaint(False)
# untaint the database status
os.system("rm -f "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
def revisionBump(self):
dbLog.log(ETP_LOG_VERBOSE,"revisionBump: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"revisionBump: called.")
if (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'])):
revision = 0
else:
@@ -902,18 +902,18 @@ class etpDatabase:
f.close()
def isDatabaseTainted(self):
dbLog.log(ETP_LOG_VERBOSE,"isDatabaseTainted: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isDatabaseTainted: called.")
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
return True
return False
def discardChanges(self):
dbLog.log(ETP_LOG_VERBOSE,"discardChanges: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"discardChanges: called.")
self.connection.rollback()
# never use this unless you know what you're doing
def initializeDatabase(self):
dbLog.log(ETP_LOG_VERBOSE,"initializeDatabase: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"initializeDatabase: called.")
self.cursor.execute(etpSQLInitDestroyAll)
self.cursor.execute(etpSQLInit)
self.commitChanges()
@@ -922,7 +922,7 @@ class etpDatabase:
# if it does not exist, it fires up addPackage
# otherwise it fires up updatePackage
def handlePackage(self, etpData, forceBump = False):
dbLog.log(ETP_LOG_VERBOSE,"handlePackage: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlePackage: called.")
if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])):
update, revision, etpDataUpdated = self.addPackage(etpData)
else:
@@ -932,7 +932,7 @@ class etpDatabase:
# default add an unstable package
def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"):
dbLog.log(ETP_LOG_VERBOSE,"addPackage: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addPackage: called.")
# Handle package name
etpData['download'] = etpData['download'].split(".tbz2")[0]
@@ -941,7 +941,7 @@ class etpDatabase:
# if a similar package, in the same branch exists, mark for removal
searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch)
dbLog.log(ETP_LOG_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar))
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar))
removelist = []
for oldpkg in searchsimilar:
# get the package slot
@@ -953,9 +953,9 @@ class etpDatabase:
for pkg in removelist:
self.removePackage(pkg)
dbLog.log(ETP_LOG_VERBOSE,"addPackage: inserting: ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addPackage: inserting: ")
for ln in etpData:
dbLog.log(ETP_LOG_VERBOSE,"\t "+ln+": "+str(etpData[ln]))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"\t "+ln+": "+str(etpData[ln]))
# wantedBranch = etpData['branch']
self.cursor.execute(
'INSERT into etpData VALUES '
@@ -998,11 +998,11 @@ class etpDatabase:
# returns False,revision if not
def updatePackage(self, etpData, forceBump = False):
dbLog.log(ETP_LOG_VERBOSE,"updatePackage: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"updatePackage: called.")
# are there any stable packages?
searchsimilarStable = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = "stable")
dbLog.log(ETP_LOG_NORMAL,"updatePackage: here is the list of similar stable packages found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilarStable))
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: here is the list of similar stable packages found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilarStable))
# filter the one with the same version
stableFound = False
for pkg in searchsimilarStable:
@@ -1016,7 +1016,7 @@ class etpDatabase:
if (stableFound):
dbLog.log(ETP_LOG_NORMAL,"updatePackage: found an old stable package, if etpData['neededlibs'] is equal, mark the branch of this updated package, stable too")
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: found an old stable package, if etpData['neededlibs'] is equal, mark the branch of this updated package, stable too")
# in this case, we should compare etpData['neededlibs'] with the db entry to see if there has been a API breakage
dbStoredNeededLibs = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "neededlibs", "stable")
@@ -1026,7 +1026,7 @@ class etpDatabase:
# - same libraries requirements
# setup etpData['branch'] accordingly
etpData['branch'] = "stable"
dbLog.log(ETP_LOG_NORMAL,"updatePackage: yes, their etpData['neededlibs'] match, marking the new package stable.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: yes, their etpData['neededlibs'] match, marking the new package stable.")
# get selected package revision
@@ -1049,17 +1049,17 @@ class etpDatabase:
# bump revision nevertheless
curRevision += 1
dbLog.log(ETP_LOG_NORMAL,"updatePackage: current revision set to "+str(curRevision))
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: current revision set to "+str(curRevision))
# add the new one
dbLog.log(ETP_LOG_NORMAL,"updatePackage: complete. Now spawning addPackage.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: complete. Now spawning addPackage.")
self.addPackage(etpData,curRevision,etpData['branch'])
# You must provide the full atom to this function
# FIXME: this must be fixed to work with branches
def removePackage(self,key, branch = "unstable"):
dbLog.log(ETP_LOG_NORMAL,"removePackage: trying to remove (if exists) -> "+str(key)+" | branch: "+branch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"removePackage: trying to remove (if exists) -> "+str(key)+" | branch: "+branch)
key = entropyTools.removePackageOperators(key)
self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'" AND branch = "'+branch+'"')
self.commitChanges()
@@ -1080,20 +1080,20 @@ class etpDatabase:
for i in myEtpData:
myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i,dbPkgBranch)
dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: called for "+str(etpData['name'])+" and "+str(myEtpData['name'])+" | branch: "+dbPkgBranch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"comparePackagesData: called for "+str(etpData['name'])+" and "+str(myEtpData['name'])+" | branch: "+dbPkgBranch)
for i in etpData:
if etpData[i] != myEtpData[i]:
dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: they don't match")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"comparePackagesData: they don't match")
return False
dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: they match")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"comparePackagesData: they match")
return True
# You must provide the full atom to this function
def retrievePackageInfo(self,pkgkey, branch = "unstable"):
pkgkey = entropyTools.removePackageOperators(pkgkey)
dbLog.log(ETP_LOG_VERBOSE,"retrievePackageInfo: retrieving package info for "+pkgkey+" | branch: "+branch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageInfo: retrieving package info for "+pkgkey+" | branch: "+branch)
result = []
self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
@@ -1103,7 +1103,7 @@ class etpDatabase:
# You must provide the full atom to this function
def retrievePackageVar(self,pkgkey,pkgvar, branch = "unstable"):
pkgkey = entropyTools.removePackageOperators(pkgkey)
dbLog.log(ETP_LOG_VERBOSE,"retrievePackageVar: retrieving package variable "+pkgvar+" for "+pkgkey+" | branch: "+branch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageVar: retrieving package variable "+pkgvar+" for "+pkgkey+" | branch: "+branch)
result = []
self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
@@ -1117,7 +1117,7 @@ class etpDatabase:
# package associated to a certain binary package file (.tbz2)
def retrievePackageVarFromBinaryPackage(self,binaryPkgName,pkgvar):
# search binary package
dbLog.log(ETP_LOG_VERBOSE,"retrievePackageVarFromBinaryPackage: retrieving package variable "+pkgvar+" for "+binaryPkgName)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageVarFromBinaryPackage: retrieving package variable "+pkgvar+" for "+binaryPkgName)
result = []
self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE download = "'+etpConst['binaryurirelativepath']+binaryPkgName+'"')
for row in self.cursor:
@@ -1130,34 +1130,34 @@ class etpDatabase:
# You must provide the full atom to this function
# WARNING: this function does not support branches !!!
def isPackageAvailable(self,pkgkey):
dbLog.log(ETP_LOG_VERBOSE,"isPackageAvailable: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isPackageAvailable: called.")
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'"')
for row in self.cursor:
result.append(row)
if result == []:
dbLog.log(ETP_LOG_NORMAL,"isPackageAvailable: "+pkgkey+" not available.")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"isPackageAvailable: "+pkgkey+" not available.")
return False
dbLog.log(ETP_LOG_VERBOSE,"isPackageAvailable: "+pkgkey+" available.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isPackageAvailable: "+pkgkey+" available.")
return True
# This version is more specific and supports branches
def isSpecificPackageAvailable(self,pkgkey, branch):
dbLog.log(ETP_LOG_VERBOSE,"isSpecificPackageAvailable: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isSpecificPackageAvailable: called.")
pkgkey = entropyTools.removePackageOperators(pkgkey)
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "'+pkgkey+'" AND branch = "'+branch+'"')
for row in self.cursor:
result.append(row[0])
if result == []:
dbLog.log(ETP_LOG_NORMAL,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> not found.")
dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> not found.")
return False
dbLog.log(ETP_LOG_VERBOSE,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> found !")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> found !")
return True
def searchPackages(self,keyword):
dbLog.log(ETP_LOG_VERBOSE,"searchPackages: called for "+keyword)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchPackages: called for "+keyword)
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "%'+keyword+'%"')
for row in self.cursor:
@@ -1165,7 +1165,7 @@ class etpDatabase:
return result
def searchPackagesInBranch(self,keyword,branch):
dbLog.log(ETP_LOG_VERBOSE,"searchPackagesInBranch: called.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchPackagesInBranch: called.")
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "%'+keyword+'%" AND branch = "'+branch+'"')
for row in self.cursor:
@@ -1176,7 +1176,7 @@ class etpDatabase:
# you must provide something like: media-sound/amarok
# optionally, you can add version too.
def searchSimilarPackages(self,atom, branch = "unstable"):
dbLog.log(ETP_LOG_VERBOSE,"searchSimilarPackages: called for "+atom+" | branch: "+branch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchSimilarPackages: called for "+atom+" | branch: "+branch)
category = atom.split("/")[0]
name = atom.split("/")[1]
result = []
@@ -1188,7 +1188,7 @@ class etpDatabase:
# NOTE: unstable and stable packages are pulled in
# so, there might be duplicates! that's normal
def listAllPackages(self):
dbLog.log(ETP_LOG_VERBOSE,"listAllPackages: called. ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listAllPackages: called. ")
result = []
self.cursor.execute('SELECT atom FROM etpData')
for row in self.cursor:
@@ -1196,7 +1196,7 @@ class etpDatabase:
return result
def listAllPackagesTbz2(self):
dbLog.log(ETP_LOG_VERBOSE,"listAllPackagesTbz2: called. ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listAllPackagesTbz2: called. ")
result = []
pkglist = self.listAllPackages()
for pkg in pkglist:
@@ -1213,7 +1213,7 @@ class etpDatabase:
return result
def listStablePackages(self):
dbLog.log(ETP_LOG_VERBOSE,"listStablePackages: called. ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listStablePackages: called. ")
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE branch = "stable"')
for row in self.cursor:
@@ -1221,7 +1221,7 @@ class etpDatabase:
return result
def listUnstablePackages(self):
dbLog.log(ETP_LOG_VERBOSE,"listUnstablePackages: called. ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listUnstablePackages: called. ")
result = []
self.cursor.execute('SELECT atom FROM etpData WHERE branch = "unstable"')
for row in self.cursor:
@@ -1229,7 +1229,7 @@ class etpDatabase:
return result
def searchStablePackages(self,atom):
dbLog.log(ETP_LOG_VERBOSE,"searchStablePackages: called for "+atom)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchStablePackages: called for "+atom)
category = atom.split("/")[0]
name = atom.split("/")[1]
result = []
@@ -1239,7 +1239,7 @@ class etpDatabase:
return result
def searchUnstablePackages(self,atom):
dbLog.log(ETP_LOG_VERBOSE,"searchUnstablePackages: called for "+atom)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchUnstablePackages: called for "+atom)
category = atom.split("/")[0]
name = atom.split("/")[1]
result = []
@@ -1251,12 +1251,12 @@ class etpDatabase:
# useful to quickly retrieve (and trash) all the data
# and look for problems.
def noopCycle(self):
dbLog.log(ETP_LOG_VERBOSE,"noopCycle: called. ")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"noopCycle: called. ")
self.cursor.execute('SELECT * FROM etpData')
def stabilizePackage(self,atom,stable = True):
dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable))
action = "unstable"
removeaction = "stable"
@@ -1264,10 +1264,10 @@ class etpDatabase:
action = "stable"
removeaction = "unstable"
dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: add action: "+action+" | remove action: "+removeaction)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: add action: "+action+" | remove action: "+removeaction)
if (self.isSpecificPackageAvailable(atom, removeaction)):
dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: there's something old that needs to be removed.")
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: there's something old that needs to be removed.")
# ! Get rid of old entries with the same slot, pkgcat/name that
# were already marked "stable"
# get its pkgname
@@ -1285,10 +1285,10 @@ class etpDatabase:
myslot = self.retrievePackageVar(result,"slot", branch = action)
if (myslot == slot):
removelist.append(result)
dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: removelist: "+str(removelist))
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: removelist: "+str(removelist))
for pkg in removelist:
self.removePackage(pkg, branch = action)
dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: updating "+atom+" setting branch: "+action)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: updating "+atom+" setting branch: "+action)
self.cursor.execute('UPDATE etpData SET branch = "'+action+'" WHERE atom = "'+atom+'" AND branch = "'+removeaction+'"')
self.commitChanges()
@@ -1296,7 +1296,7 @@ class etpDatabase:
return False,action
def writePackageParameter(self,atom,field,what,branch):
dbLog.log(ETP_LOG_VERBOSE,"writePackageParameter: writing '"+what+"' into field '"+field+"' for '"+atom+"' | branch: "+branch)
dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"writePackageParameter: writing '"+what+"' into field '"+field+"' for '"+atom+"' | branch: "+branch)
self.cursor.execute('UPDATE etpData SET '+field+' = "'+what+'" WHERE atom = "'+atom+'" AND branch = "'+branch+'"')
self.commitChanges()
libraries/entropyConstants.py
+11 -2
View File
@@ -126,6 +126,9 @@ ETP_ROOT_DIR = "/"
ETP_LOG_DIR = ETP_DIR+"/"+"logs"
ETP_LOG_NORMAL = 1
ETP_LOG_VERBOSE = 2
ETP_LOG_INFO = "[ INFO ]"
ETP_LOG_WARNING = "[ WARNING ]"
ETP_LOG_ERROR = "[ ERROR ]"
# NEVER APPEND another \n to this file because it will break the md5 check of reagent
ETP_HEADER_TEXT = "# Sabayon Linux (C - 2007) - Entropy Package Specifications (GPLv2)\n"
MAX_ETP_REVISION_COUNT = 99999
@@ -152,6 +155,7 @@ etpConst = {
'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file
'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file
'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # Source Package Manager backend configuration (Portage now)
'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file
'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf)
'activatordownloaduris': [], # list of URIs that activator can use to fetch data
'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository.
@@ -166,19 +170,24 @@ etpConst = {
'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped)
'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file
'databaseloglevel': 1, # Database log level (default: 1 - see enzyme.conf for more info)
'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info)
'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info)
'enzymeloglevel': 1 , # Enzyme log level (default: 1 - see enzyme.conf for more info)
'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info)
'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info)
'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info)
'spmbackendloglevel': 1, # # Source Package Manager backend log level (default: 1 - see entropy.conf for more info)
'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit
'databaselogfile': ETP_LOG_DIR+"/database.log", # database operations log file
'mirrorslogfile': ETP_LOG_DIR+"/mirrors.log", # Mirrors operations log file
'spmbackendlogfile': ETP_LOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file
'databaselogfile': ETP_LOG_DIR+"/database.log", # Database operations log file
'enzymelogfile': ETP_LOG_DIR+"/enzyme.log", # Enzyme operations log file
'reagentlogfile': ETP_LOG_DIR+"/reagent.log", # Reagent operations log file
'activatorlogfile': ETP_LOG_DIR+"/activator.log", # Activator operations log file
'entropylogfile': ETP_LOG_DIR+"/entropy.log", # Activator operations log file
'distcc-status': False, # used by Enzyme, if True distcc is enabled
'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file
'etpdatabasedir': ETP_DIR+ETP_DBDIR, #
libraries/entropyTools.py
+61 -60
View File
@@ -37,7 +37,7 @@ dbStatus = databaseTools.databaseStatus()
# Logging initialization
import logTools
entropyLog = logTools.LogFile(level=etpConst['entropyloglevel'],filename = etpConst['entropylogfile'], header = "[Entropy]")
# example: entropyLog.log(ETP_LOG_VERBOSE,"testFuncton: called.")
# example: entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.")
# EXIT STATUSES: 100-199
@@ -87,7 +87,7 @@ def md5sum(filepath):
# @returns the complete hash file path
# FIXME: add more hashes, SHA1 for example
def createHashFile(tbz2filepath):
entropyLog.log(ETP_LOG_VERBOSE,"createHashFile: for "+tbz2filepath)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"createHashFile: for "+tbz2filepath)
md5hash = md5sum(tbz2filepath)
hashfile = tbz2filepath+etpConst['packageshashfileext']
f = open(hashfile,"w")
@@ -98,14 +98,14 @@ def createHashFile(tbz2filepath):
return hashfile
def compareMd5(filepath,checksum):
entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareMd5: called. ")
checksum = str(checksum)
result = md5sum(filepath)
result = str(result)
if checksum == result:
entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: match. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareMd5: match. ")
return True
entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: no match. ")
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"compareMd5: no match. ")
return False
def md5string(string):
@@ -278,7 +278,7 @@ def filterDuplicatedEntries(nameslist):
# Tool to run commands
def spawnCommand(command, redirect = None):
entropyLog.log(ETP_LOG_VERBOSE,"spawnCommand: called for: "+command)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"spawnCommand: called for: "+command)
if redirect is not None:
command += " "+redirect
rc = os.system(command)
@@ -294,7 +294,7 @@ def extractFTPHostFromUri(uri):
# This function check the Entropy online database status
def getEtpRemoteDatabaseStatus():
entropyLog.log(ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: called.")
uriDbInfo = []
for uri in etpConst['activatoruploaduris']:
@@ -320,13 +320,13 @@ def getEtpRemoteDatabaseStatus():
uriDbInfo.append(info)
ftp.closeFTPConnection()
entropyLog.log(ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: dump -> "+str(uriDbInfo))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: dump -> "+str(uriDbInfo))
return uriDbInfo
def syncRemoteDatabases(noUpload = False, justStats = False):
entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: called.")
remoteDbsStatus = getEtpRemoteDatabaseStatus()
print_info(green(" * ")+red("Remote Entropy Database Repository Status:"))
@@ -439,13 +439,13 @@ def syncRemoteDatabases(noUpload = False, justStats = False):
uploadList.append(dbstat)
if (downloadLatest == []) and (not uploadLatest):
entropyLog.log(ETP_LOG_NORMAL,"syncRemoteDatabases: online database does not need to be updated.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"syncRemoteDatabases: online database does not need to be updated.")
print_info(green(" * ")+red("Online database does not need to be updated."))
return
# now run the selected task!
if (downloadLatest != []):
entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: download latest database needed.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: download latest database needed.")
# match the proper URI
for uri in etpConst['activatoruploaduris']:
if downloadLatest[0].startswith(uri):
@@ -453,7 +453,7 @@ def syncRemoteDatabases(noUpload = False, justStats = False):
downloadDatabase(downloadLatest[0])
if (uploadLatest) and (not noUpload):
entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: some mirrors don't have the latest database.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: some mirrors don't have the latest database.")
print_info(green(" * ")+red("Starting to update the needed mirrors ..."))
_uploadList = []
for uri in etpConst['activatoruploaduris']:
@@ -474,7 +474,7 @@ def syncRemoteDatabases(noUpload = False, justStats = False):
def uploadDatabase(uris):
entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: called.")
# our fancy compressor :-)
import gzip
@@ -482,7 +482,7 @@ def uploadDatabase(uris):
for uri in uris:
downloadLockDatabases(True,[uri])
entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading data to: "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: uploading data to: "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Uploading database to ")+bold(extractFTPHostFromUri(uri))+red(" ..."))
print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True)
@@ -531,10 +531,10 @@ def uploadDatabase(uris):
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" ..."), back = True)
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],True)
if (rc == True):
entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" successfull.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" successfull.")
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" completed. Disconnecting."))
else:
entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.")
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.")
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# close connection
@@ -545,11 +545,11 @@ def uploadDatabase(uris):
def downloadDatabase(uri):
entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: called.")
import gzip
entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading from -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading from -> "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Downloading database from ")+bold(extractFTPHostFromUri(uri))+red(" ..."))
print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True)
@@ -579,7 +579,7 @@ def downloadDatabase(uri):
print_info(green(" * ")+red("Decompression of ")+bold(etpConst['etpdatabasefilegzip'])+red(" completed."))
# downloading revision file
entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading revision file for "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading revision file for "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
rc = ftp.downloadFile(etpConst['etpdatabaserevisionfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
if (rc == True):
@@ -587,16 +587,17 @@ def downloadDatabase(uri):
else:
print_warning(yellow(" * ")+red("Cannot properly download to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading digest file for "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading digest file for "+extractFTPHostFromUri(uri))
# downlading digest
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasehashfile'])+red(" ..."), back = True)
rc = ftp.downloadFile(etpConst['etpdatabasehashfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
if (rc == True):
print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabasehashfile'])+red(" completed. Disconnecting."))
else:
print_warning(yellow(" * ")+red("Cannot properly download to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"downloadDatabase: Cannot properly download from "+extractFTPHostFromUri(uri)+". Please check.")
print_warning(yellow(" * ")+red("Cannot properly download from ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: do some tidy.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: do some tidy.")
os.system("rm -f " + etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasefilegzip']+" &> /dev/null")
# close connection
ftp.closeFTPConnection()
@@ -606,7 +607,7 @@ def downloadDatabase(uri):
# @ the second parameter is referred to upload locks, while the second to download ones
def getMirrorsLock():
entropyLog.log(ETP_LOG_VERBOSE,"getMirrorsLock: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getMirrorsLock: called.")
# parse etpConst['activatoruploaduris']
dbstatus = []
@@ -627,13 +628,13 @@ def getMirrorsLock():
def downloadPackageFromMirror(uri,pkgfile):
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+pkgfile)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+pkgfile)
tries = 0
maxtries = 5
for i in range(maxtries):
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") downloading -> "+pkgfile)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") downloading -> "+pkgfile)
pkgfilename = pkgfile.split("/")[len(pkgfile.split("/"))-1]
print_info(red(" * Connecting to ")+bold(extractFTPHostFromUri(uri)), back = True)
@@ -644,12 +645,12 @@ def downloadPackageFromMirror(uri,pkgfile):
print_info(red(" * Downloading ")+yellow(pkgfilename)+red(" from ")+bold(extractFTPHostFromUri(uri)))
rc = ftp.downloadFile(pkgfilename,etpConst['packagesbindir'])
if (rc is None):
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") --- ERROR --- FILE NOT FOUND -> "+pkgfile)
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") Error. File not found. -> "+pkgfile)
# file does not exist
print_warning(red(" * File ")+yellow(pkgfilename)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
return None
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") checking md5 for -> "+pkgfile)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") checking md5 for -> "+pkgfile)
# check md5
dbconn = databaseTools.etpDatabase(readOnly = True)
storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkgfilename,"digest")
@@ -661,11 +662,11 @@ def downloadPackageFromMirror(uri,pkgfile):
return True
else:
if (tries == maxtries):
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.")
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.")
print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Please consider to download or repackage again. Giving up."))
return False
else:
entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...")
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...")
print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Trying to download it again..."))
tries += 1
if os.path.isfile(etpConst['packagesbindir']+"/"+pkgfilename):
@@ -684,7 +685,7 @@ def compressTarBz2(storepath,pathtocompress):
# tar.bz2 uncompress function...
def uncompressTarBz2(filepath, extractPath = None):
entropyLog.log(ETP_LOG_VERBOSE,"uncompressTarBz2: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uncompressTarBz2: called. ")
if extractPath is None:
extractPath = os.path.dirname(filepath)
cmd = "tar xjf "+filepath+" -C "+extractPath
@@ -704,7 +705,7 @@ def bytesIntoHuman(bytes):
# hide password from full ftp URI
def hideFTPpassword(uri):
entropyLog.log(ETP_LOG_VERBOSE,"hideFTPpassword: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"hideFTPpassword: called. ")
ftppassword = uri.split("@")[:len(uri.split("@"))-1]
if len(ftppassword) > 1:
import string
@@ -723,14 +724,14 @@ def hideFTPpassword(uri):
def lockDatabases(lock = True, mirrorList = []):
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: called. ")
outstat = False
if (mirrorList == []):
mirrorList = etpConst['activatoruploaduris']
for uri in mirrorList:
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: locking? "+str(lock)+" for: "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: locking? "+str(lock)+" for: "+extractFTPHostFromUri(uri))
if (lock):
print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror..."),back = True)
@@ -742,13 +743,13 @@ def lockDatabases(lock = True, mirrorList = []):
# check if the lock is already there
if (lock):
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already locked.")
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already locked.")
print_info(green(" * ")+red("Mirror database at ")+bold(extractFTPHostFromUri(uri))+red(" already locked."))
ftp.closeFTPConnection()
continue
else:
if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already unlocked.")
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already unlocked.")
print_info(green(" * ")+red("Mirror database at ")+bold(extractFTPHostFromUri(uri))+red(" already unlocked."))
ftp.closeFTPConnection()
continue
@@ -759,24 +760,24 @@ def lockDatabases(lock = True, mirrorList = []):
f.close()
rc = ftp.uploadFile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],ascii= True)
if (rc == True):
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully locked.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully locked.")
print_info(green(" * ")+red("Succesfully locked ")+bold(extractFTPHostFromUri(uri))+red(" mirror."))
else:
outstat = True
print "\n"
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while locking.")
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while locking.")
print_warning(red(" * ")+red("A problem occured while locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror. Please have a look."))
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
else:
rc = ftp.deleteFile(etpConst['etpdatabaselockfile'])
if (rc):
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully unlocked.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully unlocked.")
print_info(green(" * ")+red("Succesfully unlocked ")+bold(extractFTPHostFromUri(uri))+red(" mirror."))
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
else:
entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while unlocking.")
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while unlocking.")
outstat = True
print "\n"
print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(extractFTPHostFromUri(uri))+red(" mirror. Please have a look."))
@@ -785,17 +786,17 @@ def lockDatabases(lock = True, mirrorList = []):
def downloadLockDatabases(lock = True, mirrorList = []):
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: called. ")
outstat = False
if (mirrorList == []):
mirrorList = etpConst['activatoruploaduris']
for uri in mirrorList:
if (lock):
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: download locking -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: download locking -> "+extractFTPHostFromUri(uri))
print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True)
else:
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: download unlocking -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: download unlocking -> "+extractFTPHostFromUri(uri))
print_info(yellow(" * ")+red("Unlocking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True)
ftp = mirrorTools.handlerFTP(uri)
# upload the lock file to database/%ARCH% directory
@@ -808,7 +809,7 @@ def downloadLockDatabases(lock = True, mirrorList = []):
continue
else:
if (not ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: already unlocked -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"downloadLockDatabases: already unlocked -> "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Download mirror at ")+bold(extractFTPHostFromUri(uri))+red(" already unlocked."))
ftp.closeFTPConnection()
continue
@@ -819,20 +820,20 @@ def downloadLockDatabases(lock = True, mirrorList = []):
f.close()
rc = ftp.uploadFile(etpConst['packagestmpdir']+"/"+etpConst['etpdatabasedownloadlockfile'],ascii= True)
if (rc == True):
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: successfully locked -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: successfully locked -> "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Succesfully locked ")+bold(extractFTPHostFromUri(uri))+red(" download mirror."))
else:
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download lock -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download lock -> "+extractFTPHostFromUri(uri))
outstat = True
print "\n"
print_warning(red(" * ")+red("A problem occured while locking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror. Please have a look."))
else:
rc = ftp.deleteFile(etpConst['etpdatabasedownloadlockfile'])
if (rc):
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: successfully unlocked -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: successfully unlocked -> "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Succesfully unlocked ")+bold(extractFTPHostFromUri(uri))+red(" download mirror."))
else:
entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download unlock -> "+extractFTPHostFromUri(uri))
entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download unlock -> "+extractFTPHostFromUri(uri))
outstat = True
print "\n"
print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror. Please have a look."))
@@ -840,7 +841,7 @@ def downloadLockDatabases(lock = True, mirrorList = []):
return outstat
def getLocalDatabaseRevision():
entropyLog.log(ETP_LOG_VERBOSE,"getLocalDatabaseRevision: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getLocalDatabaseRevision: called. ")
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile']):
f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'])
rev = f.readline().strip()
@@ -853,7 +854,7 @@ def getLocalDatabaseRevision():
# parse a dumped .etp file and returns etpData
def parseEtpDump(file):
entropyLog.log(ETP_LOG_VERBOSE,"parseEtpDump: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"parseEtpDump: called. ")
myEtpData = etpData.copy()
# reset
@@ -872,7 +873,7 @@ def parseEtpDump(file):
# Distcc check status function
def setDistCC(status = True):
entropyLog.log(ETP_LOG_VERBOSE,"setDistCC: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"setDistCC: called. ")
f = open(etpConst['enzymeconf'],"r")
enzymeconf = f.readlines()
@@ -893,7 +894,7 @@ def setDistCC(status = True):
def getDistCCHosts():
entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: called.")
f = open(etpConst['enzymeconf'],"r")
enzymeconf = f.readlines()
@@ -904,15 +905,15 @@ def getDistCCHosts():
line = line.strip().split("|")[1].split()
for host in line:
hostslist.append(host)
entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: hosts list dump -> "+str(hostslist))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: hosts list dump -> "+str(hostslist))
return hostslist
entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: hosts list EMPTY.")
entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getDistCCHosts: hosts list EMPTY.")
return []
# @returns True if validIP (type: string) is a valid IP
# @param validIP: IP string
def isValidIP(validIP):
entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: called. ")
validIPExpr = re.compile('(([0-9]|[01]?[0-9]{2}|2([0-4][0-9]|5[0-5]))\.){3}([0-9]|[01]?[0-9]{2}|2([0-4][0-9]|5[0-5]))$')
result = validIPExpr.match(validIP)
@@ -923,7 +924,7 @@ def isValidIP(validIP):
# you must provide a list
def addDistCCHosts(hosts):
entropyLog.log(ETP_LOG_VERBOSE,"addDistCCHosts: called.")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addDistCCHosts: called.")
hostslist = getDistCCHosts()
for host in hosts:
@@ -932,7 +933,7 @@ def addDistCCHosts(hosts):
# filter dupies
hostslist = list(set(hostslist))
entropyLog.log(ETP_LOG_VERBOSE,"addDistCCHosts: hostslist dump -> "+str(hostslist))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addDistCCHosts: hostslist dump -> "+str(hostslist))
# write back to file
f = open(etpConst['enzymeconf'],"r")
@@ -968,7 +969,7 @@ def addDistCCHosts(hosts):
# you must provide a list
def removeDistCCHosts(hosts):
entropyLog.log(ETP_LOG_VERBOSE,"removeDistCCHosts: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"removeDistCCHosts: called. ")
hostslist = getDistCCHosts()
cleanedhosts = []
@@ -985,7 +986,7 @@ def removeDistCCHosts(hosts):
# filter dupies
cleanedhosts = list(set(cleanedhosts))
entropyLog.log(ETP_LOG_VERBOSE,"removeDistCCHosts: cleanedhosts dump: "+cleanedhosts)
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"removeDistCCHosts: cleanedhosts dump: "+cleanedhosts)
# write back to file
f = open(etpConst['enzymeconf'],"r")
@@ -1023,7 +1024,7 @@ def getDistCCStatus():
def isIPAvailable(ip):
entropyLog.log(ETP_LOG_VERBOSE,"isIPAvailable: called. ")
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isIPAvailable: called. ")
rc = os.system("ping -c 1 "+ip+" &> /dev/null")
if (rc):
@@ -1087,7 +1088,7 @@ def alphaSorter(seq):
# Temporary files cleaner
def cleanup(options):
entropyLog.log(ETP_LOG_VERBOSE,"cleanup: with options: "+str(options))
entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"cleanup: with options: "+str(options))
toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ]
counter = 0
libraries/logTools.py
+2 -2
View File
@@ -66,9 +66,9 @@ class LogFile:
def set_loglevel(self, level):
self.level = level
def log(self, level, message):
def log(self, messagetype, level, message):
if self.level >= level:
self.handler(self.getTimeDateHeader()+self.header+' '+message)
self.handler(self.getTimeDateHeader()+messagetype+' '+self.header+' '+message)
def getTimeDateHeader(self):
return time.strftime('[%X %x %Z] ')
libraries/mirrorTools.py
+51 -3
View File
@@ -29,12 +29,20 @@ import entropyTools
import string
import os
# Logging initialization
import logTools
mirrorLog = logTools.LogFile(level=etpConst['mirrorsloglevel'],filename = etpConst['mirrorslogfile'], header = "[Mirrors]")
# example: mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.")
class handlerFTP:
# ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org
# this must be run before calling the other functions
def __init__(self, ftpuri):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.__init__: called.")
from ftplib import FTP
self.ftpuri = ftpuri
@@ -81,40 +89,52 @@ class handlerFTP:
# this can be used in case of exceptions
def reconnectHost(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.reconnectHost: called.")
self.ftpconn = FTP(self.ftphost)
self.ftpconn.login(self.ftpuser,self.ftppassword)
self.ftpconn.cwd(self.currentdir)
def getFTPHost(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPHost: called -> "+self.ftphost)
return self.ftphost
def getFTPPort(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpport)
return self.ftpport
def getFTPDir(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpdir)
return self.ftpdir
def getCWD(self):
return self.ftpconn.pwd()
pwd = self.ftpconn.pwd()
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getCWD: called -> "+pwd)
return pwd
def setCWD(self,dir):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.setCWD: called -> "+dir)
self.ftpconn.cwd(dir)
self.currentdir = dir
def setPASV(self,bool):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.setPASV: called -> "+str(bool))
self.ftpconn.set_pasv(bool)
def getFileMtime(self,path):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileMtime: called for -> "+path)
rc = self.ftpconn.sendcmd("mdtm "+path)
return rc.split()[len(rc.split())-1]
def spawnFTPCommand(self,cmd):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.spawnFTPCommand: called, command -> "+cmd)
rc = self.ftpconn.sendcmd(cmd)
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.spawnFTPCommand: called, rc -> "+str(rc))
return rc
# list files and directory of a FTP
# @returns a list
def listFTPdir(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.listFTPdir: called.")
# directory is: self.ftpdir
try:
rc = self.ftpconn.nlst()
@@ -129,6 +149,7 @@ class handlerFTP:
# list if the file is available
# @returns True or False
def isFileAvailable(self,filename):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.isFileAvailable: called for -> "+filename)
# directory is: self.ftpdir
try:
rc = self.ftpconn.nlst()
@@ -138,23 +159,31 @@ class handlerFTP:
rc = _rc
for i in rc:
if i == filename:
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.isFileAvailable: result -> True")
return True
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_WARNING,"handlerFTP.isFileAvailable: result -> False (no exception)")
return False
except:
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_ERROR,"handlerFTP.isFileAvailable: result -> False (exception occured!!!)")
return False
def deleteFile(self,file):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: called for -> "+str(file))
try:
rc = self.ftpconn.delete(file)
if rc.startswith("250"):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> True")
return True
else:
mirrorLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> False (no exception)")
return False
except:
mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> False (exception occured!!!)")
return False
# this function also supports callback, because storbinary doesn't
def advancedStorBinary(self, cmd, fp, callback=None, blocksize=8192):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.advancedStorBinary: called with -> "+str(cmd))
''' Store a file in binary mode. Our version supports a callback function'''
self.ftpconn.voidcmd('TYPE I')
conn = self.ftpconn.transfercmd(cmd)
@@ -178,7 +207,12 @@ class handlerFTP:
# print !
print_info(currentText,back = True)
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"handlerFTP.uploadFile: called for -> "+str(file)+" mode, ascii?: "+str(ascii))
for i in range(10): # ten tries
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.uploadFile: try #"+str(i))
f = open(file)
filename = file.split("/")[len(file.split("/"))-1]
try:
@@ -197,10 +231,13 @@ class handlerFTP:
self.renameFile(filename+".tmp",filename)
f.close()
if rc.find("226") != -1: # upload complete
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.uploadFile: upload complete.")
return True
else:
mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.uploadFile: upload failed !!.")
return False
except: # connection reset by peer
mirrorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"handlerFTP.uploadFile: upload issues, retrying...")
print_info(red("Upload issue, retrying..."))
self.reconnectHost() # reconnect
if self.isFileAvailable(filename):
@@ -222,6 +259,8 @@ class handlerFTP:
# print !
print_info(currentText,back = True)
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"handlerFTP.downloadFile: called for -> "+str(filepath)+" | download directory: "+str(downloaddir)+" | ascii? "+str(ascii))
file = filepath.split("/")[len(filepath.split("/"))-1]
# look if the file exist
if self.isFileAvailable(file):
@@ -243,21 +282,28 @@ class handlerFTP:
f.flush()
f.close()
if rc.find("226") != -1: # upload complete
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.downloadFile: download success.")
return True
else:
mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.downloadFile: download issues !!.")
return False
else:
mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.downloadFile: file '"+file+"' not available !!.")
return None
# also used to move files
def renameFile(self,fromfile,tofile):
self.ftpconn.rename(fromfile,tofile)
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.renameFile: rename file from '"+fromfile+"' to '"+tofile+"'.")
rc = self.ftpconn.rename(fromfile,tofile)
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.renameFile: return output: '"+rc+"'")
# not supported by dreamhost.com
def getFileSize(self,file):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileSize: called for -> "+file)
return self.ftpconn.size(file)
def getFileSizeCompat(self,file):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileSizeCompat: called for -> "+file)
list = self.getRoughList()
for item in list:
if item.find(file) != -1:
@@ -269,9 +315,11 @@ class handlerFTP:
self.FTPbuffer.append(buf)
def getRoughList(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getRoughList: called.")
self.FTPbuffer = []
self.ftpconn.dir(self.bufferizer)
return self.FTPbuffer
def closeFTPConnection(self):
mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.closeFTPConnection: called.")
self.ftpconn.quit()
libraries/portageTools.py
+45 -45
View File
@@ -61,7 +61,7 @@ import entropyTools
# Logging initialization
import logTools
portageLog = logTools.LogFile(level=etpConst['spmbackendloglevel'],filename = etpConst['spmbackendlogfile'], header = "[Portage]")
# portageLog.log(ETP_LOG_VERBOSE,"testFunction: example. ")
# portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFunction: example. ")
############
# Functions and Classes
@@ -71,41 +71,41 @@ def getThirdPartyMirrors(mirrorname):
return portage.thirdpartymirrors[mirrorname]
def getPortageEnv(var):
portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPortageEnv: called. ")
try:
rc = portage.config(clone=portage.settings).environ()[var]
portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: variable available -> "+str(var))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPortageEnv: variable available -> "+str(var))
return rc
except KeyError:
portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: variable not available -> "+str(var))
portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getPortageEnv: variable not available -> "+str(var))
return None
# resolve atoms automagically (best, not current!)
# sys-libs/application --> sys-libs/application-1.2.3-r1
def getBestAtom(atom):
portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: called -> "+str(atom))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: called -> "+str(atom))
try:
rc = portage.portdb.xmatch("bestmatch-visible",str(atom))
portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc))
return rc
except ValueError:
portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: conflict found. ")
portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getBestAtom: conflict found. ")
return "!!conflicts"
# same as above but includes masked ebuilds
def getBestMaskedAtom(atom):
portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: called. ")
atoms = portage.portdb.xmatch("match-all",str(atom))
# find the best
from portage_versions import best
rc = best(atoms)
portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc))
return rc
# I need a valid complete atom...
def calculateFullAtomsDependencies(atoms, deep = False, extraopts = ""):
portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: called -> "+str(atoms)+" | extra options: "+extraopts)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: called -> "+str(atoms)+" | extra options: "+extraopts)
# in order... thanks emerge :-)
deepOpt = ""
@@ -141,17 +141,17 @@ def calculateFullAtomsDependencies(atoms, deep = False, extraopts = ""):
blocklist = _blocklist
if deplist != []:
portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist -> "+len(deplist)+" | blocklist -> "+len(blocklist))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist -> "+len(deplist)+" | blocklist -> "+len(blocklist))
return deplist, blocklist
else:
portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist empty. Giving up.")
portageLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist empty. Giving up.")
rc = os.system(cmd)
sys.exit(100)
def calculateAtomUSEFlags(atom):
portageLog.log(ETP_LOG_VERBOSE,"calculateAtomUSEFlags: called -> "+str(atom))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateAtomUSEFlags: called -> "+str(atom))
try:
useflags = "USE='"+os.environ['USE']+"' "
@@ -166,7 +166,7 @@ def calculateAtomUSEFlags(atom):
useparm = useparm.split()
_useparm = []
portageLog.log(ETP_LOG_VERBOSE,"calculateAtomUSEFlags: USE flags -> "+str(useparm))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateAtomUSEFlags: USE flags -> "+str(useparm))
for use in useparm:
# -cups
@@ -190,12 +190,12 @@ def calculateAtomUSEFlags(atom):
# should be only used when a pkgcat/pkgname <-- is not specified (example: db, amarok, AND NOT media-sound/amarok)
def getAtomCategory(atom):
portageLog.log(ETP_LOG_VERBOSE,"getAtomCategory: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getAtomCategory: called. ")
try:
rc = portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0]
return rc
except:
portageLog.log(ETP_LOG_VERBOSE,"getAtomCategory: error, can't extract category !")
portageLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"getAtomCategory: error, can't extract category !")
return None
# This function compare the version number of two atoms
@@ -204,7 +204,7 @@ def getAtomCategory(atom):
# if atom1 > atom2 --> returns a POSITIVE number
# if atom1 == atom2 --> returns 0
def compareAtoms(atom1,atom2):
portageLog.log(ETP_LOG_VERBOSE,"compareAtoms: called -> "+atom1+" && "+atom2)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareAtoms: called -> "+atom1+" && "+atom2)
# filter pkgver
x, atom1 = extractPkgNameVer(atom1)
x, atom2 = extractPkgNameVer(atom2)
@@ -213,7 +213,7 @@ def compareAtoms(atom1,atom2):
# please always force =pkgcat/pkgname-ver if possible
def getInstalledAtom(atom):
portageLog.log(ETP_LOG_VERBOSE,"getInstalledAtom: called -> "+str(atom))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledAtom: called -> "+str(atom))
rc = portage.db['/']['vartree'].dep_match(str(atom))
if (rc != []):
if (len(rc) == 1):
@@ -224,20 +224,20 @@ def getInstalledAtom(atom):
return None
def getPackageSlot(atom):
portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageSlot: called. ")
if atom.startswith("="):
atom = atom[1:]
rc = portage.db['/']['vartree'].getslot(atom)
if rc != "":
portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: slot found -> "+str(atom)+" -> "+str(rc))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageSlot: slot found -> "+str(atom)+" -> "+str(rc))
return rc
else:
portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: slot not found -> "+str(atom))
portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getPackageSlot: slot not found -> "+str(atom))
return None
# you must provide a complete atom
def collectBinaryFilesForInstalledPackage(atom):
portageLog.log(ETP_LOG_VERBOSE,"collectBinaryFilesForInstalledPackage: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"collectBinaryFilesForInstalledPackage: called. ")
if atom.startswith("="):
atom = atom[1:]
pkgcat = atom.split("/")[0]
@@ -258,11 +258,11 @@ def collectBinaryFilesForInstalledPackage(atom):
return binarylibs
def getEbuildDbPath(atom):
portageLog.log(ETP_LOG_VERBOSE,"getEbuildDbPath: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEbuildDbPath: called -> "+atom)
return portage.db['/']['vartree'].getebuildpath(atom)
def getEbuildTreePath(atom):
portageLog.log(ETP_LOG_VERBOSE,"getEbuildTreePath: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEbuildTreePath: called -> "+atom)
if atom.startswith("="):
atom = atom[1:]
rc = portage.portdb.findname(atom)
@@ -272,7 +272,7 @@ def getEbuildTreePath(atom):
return None
def getPackageDownloadSize(atom):
portageLog.log(ETP_LOG_VERBOSE,"getPackageDownloadSize: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageDownloadSize: called -> "+atom)
if atom.startswith("="):
atom = atom[1:]
@@ -298,7 +298,7 @@ def getPackageDownloadSize(atom):
return "N/A"
def getInstalledAtoms(atom):
portageLog.log(ETP_LOG_VERBOSE,"getInstalledAtoms: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledAtoms: called -> "+atom)
rc = portage.db['/']['vartree'].dep_match(str(atom))
if (rc != []):
return rc
@@ -309,7 +309,7 @@ def getInstalledAtoms(atom):
# YOU MUST PROVIDE A COMPLETE ATOM with a pkgcat !
def unmerge(atom):
portageLog.log(ETP_LOG_VERBOSE,"umerge: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"umerge: called -> "+atom)
if isjustname(atom) or (not isvalidatom(atom)) or (atom.find("/") == -1):
return 1
else:
@@ -323,7 +323,7 @@ def unmerge(atom):
# TO THIS FUNCTION:
# must be provided a valid and complete atom
def extractPkgNameVer(atom):
portageLog.log(ETP_LOG_VERBOSE,"extractPkgNameVer: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"extractPkgNameVer: called -> "+atom)
package = dep_getcpv(atom)
package = atom.split("/")[len(atom.split("/"))-1]
package = package.split("-")
@@ -343,7 +343,7 @@ def extractPkgNameVer(atom):
return pkgname,pkgver
def emerge(atom, options, outfile = None, redirect = "&>", simulate = False):
portageLog.log(ETP_LOG_VERBOSE,"emerge: called -> "+atom+" | options: "+str(options)+" | redirect: "+str(redirect)+" | outfile: "+str(outfile)+" | simulate: "+str(simulate))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"emerge: called -> "+atom+" | options: "+str(options)+" | redirect: "+str(redirect)+" | outfile: "+str(outfile)+" | simulate: "+str(simulate))
if (simulate):
return 0,"" # simulation enabled
if (outfile is None) and (redirect == "&>"):
@@ -381,7 +381,7 @@ def emerge(atom, options, outfile = None, redirect = "&>", simulate = False):
except:
ldflags = " "
portageLog.log(ETP_LOG_VERBOSE,"emerge: USE: "+useflags+" | CFLAGS: "+cflags+" | MAKEOPTS: "+makeopts+" | LDFLAGS: "+ldflags)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"emerge: USE: "+useflags+" | CFLAGS: "+cflags+" | MAKEOPTS: "+makeopts+" | LDFLAGS: "+ldflags)
# elog configuration
elogopts = dbPORTAGE_ELOG_OPTS+" "
@@ -405,7 +405,7 @@ def emerge(atom, options, outfile = None, redirect = "&>", simulate = False):
def parseElogFile(atom):
portageLog.log(ETP_LOG_VERBOSE,"parseElogFile: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"parseElogFile: called. ")
if atom.startswith("="):
atom = atom[1:]
@@ -445,7 +445,7 @@ def parseElogFile(atom):
def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles):
portageLog.log(ETP_LOG_VERBOSE,"compareLibraryLists: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareLibraryLists: called. ")
brokenBinariesList = []
# check if there has been a API breakage
@@ -488,7 +488,7 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles):
# create a .tbz2 file in the specified path
def quickpkg(atom,dirpath):
portageLog.log(ETP_LOG_VERBOSE,"quickpkg: called -> "+atom+" | dirpath: "+dirpath)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"quickpkg: called -> "+atom+" | dirpath: "+dirpath)
# getting package info
pkgname = atom.split("/")[1]
@@ -534,7 +534,7 @@ def quickpkg(atom,dirpath):
def unpackTbz2(tbz2File,tmpdir = None):
portageLog.log(ETP_LOG_VERBOSE,"unpackTbz2: called -> "+tbz2File)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"unpackTbz2: called -> "+tbz2File)
import xpak
if tmpdir is None:
@@ -550,7 +550,7 @@ def unpackTbz2(tbz2File,tmpdir = None):
# NOTE: atom must be a COMPLETE atom, with version!
def isTbz2PackageAvailable(atom, verbose = False):
portageLog.log(ETP_LOG_VERBOSE,"isTbz2PackageAvailable: called -> "+atom)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isTbz2PackageAvailable: called -> "+atom)
# check if the package have been already merged
atomName = atom.split("/")[len(atom.split("/"))-1]
@@ -571,12 +571,12 @@ def isTbz2PackageAvailable(atom, verbose = False):
tbz2Available = uploadPath
if (verbose): print_info("found here: "+str(tbz2Available))
portageLog.log(ETP_LOG_VERBOSE,"isTbz2PackageAvailable: result -> "+str(tbz2Available))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isTbz2PackageAvailable: result -> "+str(tbz2Available))
return tbz2Available
def checkAtom(atom):
portageLog.log(ETP_LOG_VERBOSE,"checkAtom: called -> "+str(atom))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"checkAtom: called -> "+str(atom))
bestAtom = getBestAtom(atom)
if bestAtom == "!!conflicts":
bestAtom = ""
@@ -586,7 +586,7 @@ def checkAtom(atom):
def getPackageDependencyList(atom):
portageLog.log(ETP_LOG_VERBOSE,"getPackageDependencyList: called -> "+str(atom))
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageDependencyList: called -> "+str(atom))
pkgSplittedDeps = []
tmp = portage.portdb.aux_get(atom, ["DEPEND"])[0].split()
for i in tmp:
@@ -603,7 +603,7 @@ def getPackageDependencyList(atom):
# this file is contained in the .tbz2->.xpak file
def getPackageRuntimeDependencies(NEEDED):
portageLog.log(ETP_LOG_VERBOSE,"getPackageRuntimeDependencies: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageRuntimeDependencies: called. ")
if not os.path.isfile(NEEDED):
return [],[],[] # all empty
@@ -653,16 +653,16 @@ def getPackageRuntimeDependencies(NEEDED):
return runtimeNeededPackages, runtimeNeededPackagesXT, neededLibraries
def getUSEFlags():
portageLog.log(ETP_LOG_VERBOSE,"getUSEFlags: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getUSEFlags: called. ")
return getPortageEnv('USE')
# you must provide a complete atom
def getPackageIUSE(atom):
portageLog.log(ETP_LOG_VERBOSE,"getPackageIUSE: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageIUSE: called. ")
return getPackageVar(atom,"IUSE")
def getPackageVar(atom,var):
portageLog.log(ETP_LOG_VERBOSE,"getPackageVar: called -> "+atom+" | var: "+var)
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageVar: called -> "+atom+" | var: "+var)
if atom.startswith("="):
atom = atom[1:]
# can't check - return error
@@ -671,7 +671,7 @@ def getPackageVar(atom,var):
return portage.portdb.aux_get(atom,[var])[0]
def synthetizeRoughDependencies(roughDependencies, useflags = None):
portageLog.log(ETP_LOG_VERBOSE,"synthetizeRoughDependencies: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"synthetizeRoughDependencies: called. ")
if useflags is None:
useflags = getUSEFlags()
# returns dependencies, conflicts
@@ -761,7 +761,7 @@ def getPortageAppDbPath():
# Collect installed packages
def getInstalledPackages():
portageLog.log(ETP_LOG_VERBOSE,"getInstalledPackages: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledPackages: called. ")
import os
appDbDir = getPortageAppDbPath()
dbDirs = os.listdir(appDbDir)
@@ -777,7 +777,7 @@ def getInstalledPackages():
def packageSearch(keyword):
portageLog.log(ETP_LOG_VERBOSE,"packageSearch: called. ")
portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packageSearch: called. ")
SearchDirs = []
# search in etpConst['portagetreedir']
libraries/reagentTools.py
+15
View File
@@ -37,8 +37,12 @@ from portageTools import unpackTbz2, synthetizeRoughDependencies, getPackageRunt
import logTools
reagentLog = logTools.LogFile(level=etpConst['reagentloglevel'],filename = etpConst['reagentlogfile'], header = "[Reagent]")
# reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFunction: example. ")
def generator(package, enzymeRequestBump = False, dbconnection = None):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: called -> Package: "+str(package)+" | enzymeRequestBump: "+str(enzymeRequestBump)+" | dbconnection: "+str(dbconnection))
# check if the package provided is valid
validFile = False
if os.path.isfile(package) and package.endswith(".tbz2"):
@@ -66,12 +70,15 @@ def generator(package, enzymeRequestBump = False, dbconnection = None):
dbconn.closeDB()
if (updated) and (revision != 0):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" has been updated to revision: "+str(revision))
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision)))
return True, newFileName
elif (updated) and (revision == 0):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" newly created.")
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created."))
return True, newFileName
else:
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" kept intact, no updates needed.")
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" does not need to be updated. Current revision: ")+bold(str(revision)))
return False, newFileName
@@ -79,6 +86,8 @@ def generator(package, enzymeRequestBump = False, dbconnection = None):
# This tool is used by Entropy after enzyme, it simply parses the content of etpConst['packagesstoredir']
def enzyme(options):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"enzyme: called -> options: "+str(options))
enzymeRequestBump = False
#_atoms = []
for i in options:
@@ -126,6 +135,8 @@ def enzyme(options):
# This function extracts all the info from a .tbz2 file and returns them
def extractPkgData(package):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"extractPkgData: called -> package: "+str(package))
# Clean the variables
for i in etpData:
etpData[i] = u""
@@ -436,6 +447,8 @@ def extractPkgData(package):
def smartapps(options):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"smartapps: called -> options: "+str(options))
if (len(options) == 0):
print_error(yellow(" * ")+red("No valid tool specified."))
sys.exit(501)
@@ -484,6 +497,8 @@ def smartapps(options):
# NOTE: this section is highly portage dependent
def smartgenerator(atom):
reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"smartgenerator: called -> package: "+str(atom))
dbconn = databaseTools.etpDatabase(readOnly = True)
# handle branch management: