diff --git a/TODO b/TODO index f5ccab12c..bf204a88d 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ TODO list (for developers only): - implement logTools inside handlers - - activatorTools - - reagentTools - - mirrorTools - - add option to cleanup logs + - enzymeTools - reagent: complete smartapps section - enzyme, reagent: test kernel dependent packages - enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails diff --git a/conf/mirrors.conf b/conf/mirrors.conf new file mode 100644 index 000000000..4199020c2 --- /dev/null +++ b/conf/mirrors.conf @@ -0,0 +1,8 @@ +# Project Entropy 1.0 Mirrors configuration file + +# Log level +# 0: No Logging +# 1: Normal Logging +# 2: Verbose Logging +loglevel|1 + diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 834e3d559..0a65a2459 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -36,9 +36,12 @@ import time # Logging initialization import logTools activatorLog = logTools.LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]") +# example: activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.") def sync(options, justTidy = False): + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: called with justTidy -> "+str(justTidy)) + print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ...")) if (not justTidy): @@ -85,6 +88,7 @@ def sync(options, justTidy = False): removeList.append(repoBin) if (removeList == []): + activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: no packages to remove from the lirrors.") print_info(green(" * ")+red("No packages to remove from the mirrors.")) print_info(green(" * ")+red("Syncronization across mirrors completed.")) return @@ -99,11 +103,14 @@ def sync(options, justTidy = False): sys.exit(0) # remove them! + activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: starting to remove packages from mirrors.") for uri in etpConst['activatoruploaduris']: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"sync: connecting to mirror "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Connecting to: ")+bold(extractFTPHostFromUri(uri))) ftp = mirrorTools.handlerFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) for file in removeList: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: removing (remote) file "+file) print_info(green(" * ")+red("Removing file: ")+bold(file), back = True) # remove remotely if (ftp.isFileAvailable(file)): @@ -114,6 +121,7 @@ def sync(options, justTidy = False): print_warning(yellow(" * ")+red("ATTENTION: remote file ")+bold(file)+red(" cannot be removed.")) # remove locally if os.path.isfile(etpConst['packagesbindir']+"/"+file): + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"sync: removing (local) file "+file) print_info(green(" * ")+red("Package file: ")+bold(file)+red(" removed successfully from ")+bold(etpConst['packagesbindir'])) os.remove(etpConst['packagesbindir']+"/"+file) ftp.closeFTPConnection() @@ -123,6 +131,8 @@ def sync(options, justTidy = False): def packages(options): + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: called with options -> "+str(options)) + # Options available for all the packages submodules myopts = options[1:] activatorRequestAsk = False @@ -144,6 +154,8 @@ def packages(options): currentUri = 0 totalSuccessfulUri = 0 + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: called sync.") + for uri in etpConst['activatoruploaduris']: currentUri += 1 @@ -159,6 +171,9 @@ def packages(options): toBeUploaded.append(tbz2) if tbz2.endswith(".tbz2"): uploadCounter += 1 + + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: upload directory stats -> files: "+str(uploadCounter)+" | upload packages list: "+str(toBeUploaded)) + print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) localPackagesRepository = [] # parse etpConst['packagesbindir'] print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True) @@ -168,6 +183,9 @@ def packages(options): localPackagesRepository.append(tbz2) if tbz2.endswith(".tbz2"): packageCounter += 1 + + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: packages directory stats -> files: "+str(packageCounter)+" | download packages list (including md5): "+str(localPackagesRepository)) + print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) @@ -182,13 +200,18 @@ def packages(options): for tbz2 in remotePackages: if tbz2.endswith(".tbz2"): remoteCounter += 1 + + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: remote packages stats -> files: "+str(remoteCounter)) + print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored.")) print_info(green(" * ")+yellow("Calculating...")) uploadQueue = [] downloadQueue = [] removalQueue = [] - + + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: starting packages calculation...") + # if a package is in the packages directory but not online, we have to upload it # we have localPackagesRepository and remotePackages for localPackage in localPackagesRepository: @@ -320,9 +343,9 @@ def packages(options): # filter duplicates - removalQueue = list(set(removalQueue)) - downloadQueue = list(set(downloadQueue)) - uploadQueue = list(set(uploadQueue)) + removalQueue = filterDuplicatedEntries(removalQueue) + downloadQueue = filterDuplicatedEntries(downloadQueue) + uploadQueue = filterDuplicatedEntries(uploadQueue) # order alphabetically if (removalQueue != []): @@ -346,6 +369,10 @@ def packages(options): _downloadQueue.append(p) downloadQueue = _downloadQueue + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: removal queue -> "+str(removalQueue)) + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: download queue -> "+str(downloadQueue)) + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: upload queue -> "+str(uploadQueue)) + if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0): print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty."))) totalSuccessfulUri += 1 @@ -421,6 +448,12 @@ def packages(options): uploadCounter = "0" downloadCounter = "0" + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed removal queue -> "+str(detailedRemovalQueue)) + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed simple copy queue -> "+str(simpleCopyQueue)) + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed download queue -> "+str(detailedDownloadQueue)) + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packages: detailed upload queue -> "+str(detailedUploadQueue)) + + # removal queue if (detailedRemovalQueue != []): for item in detailedRemovalQueue: @@ -501,12 +534,16 @@ def packages(options): # trap exceptions, failed to upload/download someting? except: + + activatorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"packages: cannot properly syncronize "+extractFTPHostFromUri(uri)+". Trying to continue if possible.") + # print warning cannot sync uri print_warning(yellow(" * ")+red("ATTENTION: cannot properly syncronize ")+bold(extractFTPHostFromUri(uri))+red(". Continuing if possible...")) # decide what to do if (totalSuccessfulUri > 0) or (activatorRequestPretend): # we're safe + activatorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"packages: at least one mirror has been synced properly. I'm fine.") print_info(green(" * ")+red("At least one mirror has been synced properly. I'm fine.")) continue else: @@ -516,13 +553,15 @@ def packages(options): else: # no mirrors were synced properly # show error and return, do not move files from the upload dir + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"packages: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.") print_error(yellow(" * ")+red("ERROR: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.")) return False # if at least one server has been synced successfully, move files if (totalSuccessfulUri > 0) and (not activatorRequestPretend): - # now we can store the files in upload/%ARCH% in packages/%ARCH% + # now we can store the files in upload/%ARCH% in packages/%ARCH% + activatorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"packages: all done. Now it's time to move packages to "+etpConst['packagesbindir']) os.system("mv -f "+etpConst['packagessuploaddir']+"/* "+etpConst['packagesbindir']+"/ &> /dev/null") return True else: @@ -536,13 +575,17 @@ def packages(options): def database(options): + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: called with -> "+str(options)) + # lock tool if (options[0] == "lock"): print_info(green(" * ")+green("Starting to lock mirrors' databases...")) rc = lockDatabases(lock = True) if (rc): - print_info(green(" * ")+green("A problem occured on at least one mirror...")) + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (lock) A problem occured on at least one mirror !") + print_info(green(" * ")+red("A problem occured on at least one mirror !")) else: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Databases lock complete.") print_info(green(" * ")+green("Databases lock complete")) # unlock tool @@ -550,8 +593,10 @@ def database(options): print_info(green(" * ")+green("Starting to unlock mirrors' databases...")) rc = lockDatabases(lock = False) if (rc): - print_info(green(" * ")+green("A problem occured on at least one mirror...")) + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (unlock) A problem occured on at least one mirror !") + print_info(green(" * ")+green("A problem occured on at least one mirror !")) else: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Databases lock complete.") print_info(green(" * ")+green("Databases unlock complete")) # download lock tool @@ -559,8 +604,10 @@ def database(options): print_info(green(" * ")+green("Starting to lock download mirrors' databases...")) rc = downloadLockDatabases(lock = True) if (rc): - print_info(green(" * ")+green("A problem occured on at least one mirror...")) + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (download-lock) A problem occured on at least one mirror !") + print_info(green(" * ")+green("A problem occured on at least one mirror !")) else: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Download mirrors lock complete.") print_info(green(" * ")+green("Download mirrors lock complete")) # download unlock tool @@ -568,8 +615,10 @@ def database(options): print_info(green(" * ")+green("Starting to unlock download mirrors' databases...")) rc = downloadLockDatabases(lock = False) if (rc): + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !") print_info(green(" * ")+green("A problem occured on at least one mirror...")) else: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: Download mirrors unlock complete.") print_info(green(" * ")+green("Download mirrors unlock complete")) # lock status tool @@ -589,7 +638,9 @@ def database(options): # database sync tool elif (options[0] == "sync"): - + + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database: database sync called.") + print_info(green(" * ")+red("Checking database status ..."), back = True) dbLockFile = False @@ -622,14 +673,17 @@ def database(options): os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) else: print - print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ...")) + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...") + print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later...")) sys.exit(422) else: if (dbLockFile): + activatorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])) sys.exit(423) else: + activatorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...") print_info(green(" * ")+red("Mirrors are not locked. Fetching data...")) syncRemoteDatabases() diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index c8c66f998..312dec5cc 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -704,7 +704,7 @@ class databaseStatus: def __init__(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus.__init__ called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus.__init__ called.") self.databaseBumped = False self.databaseInfoCached = False @@ -714,103 +714,103 @@ class databaseStatus: self.databaseAlreadyTainted = False if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: database tainted.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: database tainted.") self.databaseAlreadyTainted = True def isDatabaseAlreadyBumped(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: already bumped? "+str(self.databaseBumped)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: already bumped? "+str(self.databaseBumped)) return self.databaseBumped def isDatabaseAlreadyTainted(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: tainted? "+str(self.databaseAlreadyTainted)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: tainted? "+str(self.databaseAlreadyTainted)) return self.databaseAlreadyTainted def setDatabaseTaint(self,bool): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: setting database taint to: "+str(bool)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: setting database taint to: "+str(bool)) self.databaseAlreadyTainted = bool def setDatabaseBump(self,bool): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: setting database bump to: "+str(bool)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: setting database bump to: "+str(bool)) self.databaseBumped = bool def setDatabaseLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (upload)") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (upload)") self.databaseLock = True def unsetDatabaseLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (upload)") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (upload)") self.databaseLock = False def getDatabaseLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (upload), status: "+str(self.databaseLock)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (upload), status: "+str(self.databaseLock)) return self.databaseLock def setDatabaseDownloadLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (download)") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Locking database (download)") self.databaseDownloadLock = True def unsetDatabaseDownloadLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (download)") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: Unlocking database (download)") self.databaseDownloadLock = False def getDatabaseDownloadLock(self): - dbLog.log(ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (download), status: "+str(self.databaseDownloadLock)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"DatabaseStatus: getting database lock info (download), status: "+str(self.databaseDownloadLock)) return self.databaseDownloadLock class etpDatabase: def __init__(self, readOnly = False, noUpload = False): - dbLog.log(ETP_LOG_VERBOSE,"etpDatabase.__init__ called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase.__init__ called.") self.readOnly = readOnly self.noUpload = noUpload if (self.readOnly): - dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: database opened readonly") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: database opened readonly") # if the database is opened readonly, we don't need to lock the online status self.connection = sqlite.connect(etpConst['etpdatabasefilepath']) self.cursor = self.connection.cursor() # set the table read only return - dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: database opened in read/write mode") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: database opened in read/write mode") # check if the database is locked locally if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): - dbLog.log(ETP_LOG_NORMAL,"etpDatabase: database already locked") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"etpDatabase: database already locked") print_info(red(" * ")+red(" Entropy database is already locked by you :-)")) else: # check if the database is locked REMOTELY - dbLog.log(ETP_LOG_NORMAL,"etpDatabase: starting to lock and sync database") + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"etpDatabase: starting to lock and sync database") print_info(red(" * ")+red(" Locking and Syncing Entropy database ..."), back = True) for uri in etpConst['activatoruploaduris']: - dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: connecting to "+uri) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: connecting to "+uri) ftp = mirrorTools.handlerFTP(uri) ftp.setCWD(etpConst['etpurirelativepath']) if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) and (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])): import time print_info(red(" * ")+bold("WARNING")+red(": online database is already locked. Waiting up to 2 minutes..."), back = True) - dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database already locked. Waiting 2 minutes") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"etpDatabase: online database already locked. Waiting 2 minutes") unlocked = False for x in range(120): time.sleep(1) if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])): - dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database has been unlocked !") + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"etpDatabase: online database has been unlocked !") print_info(red(" * ")+bold("HOORAY")+red(": online database has been unlocked. Locking back and syncing...")) unlocked = True break if (unlocked): break - dbLog.log(ETP_LOG_NORMAL,"etpDatabase: online database has not been unlocked in time. Giving up.") + dbLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"etpDatabase: online database has not been unlocked in time. Giving up.") # time over print_info(red(" * ")+bold("ERROR")+red(": online database has not been unlocked. Giving up. Who the hell is working on it? Damn, it's so frustrating for me. I'm a piece of python code with a soul dude!")) # FIXME show the lock status print_info(yellow(" * ")+green("Mirrors status table:")) dbstatus = entropyTools.getMirrorsLock() - dbLog.log(ETP_LOG_VERBOSE,"etpDatabase: showing mirrors status table:") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"etpDatabase: showing mirrors status table:") for db in dbstatus: if (db[1]): db[1] = red("Locked") @@ -820,7 +820,7 @@ class etpDatabase: db[2] = red("Locked") else: db[2] = green("Unlocked") - dbLog.log(ETP_LOG_VERBOSE," "+entropyTools.extractFTPHostFromUri(db[0])+": DATABASE: "+db[1]+" | DOWNLOAD: "+db[2]) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE," "+entropyTools.extractFTPHostFromUri(db[0])+": DATABASE: "+db[1]+" | DOWNLOAD: "+db[2]) print_info(bold("\t"+entropyTools.extractFTPHostFromUri(db[0])+": ")+red("[")+yellow("DATABASE: ")+db[1]+red("] [")+yellow("DOWNLOAD: ")+db[2]+red("]")) ftp.closeFTPConnection() @@ -840,13 +840,13 @@ class etpDatabase: # if the class is opened readOnly, close and forget if (self.readOnly): - dbLog.log(ETP_LOG_VERBOSE,"closeDB: closing database opened in readonly.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"closeDB: closing database opened in readonly.") #self.connection.rollback() self.cursor.close() self.connection.close() return - dbLog.log(ETP_LOG_VERBOSE,"closeDB: closing database opened in read/write.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"closeDB: closing database opened in read/write.") # FIXME verify all this shit, for now it works... if (entropyTools.dbStatus.isDatabaseAlreadyTainted()) and (not entropyTools.dbStatus.isDatabaseAlreadyBumped()): @@ -865,15 +865,15 @@ class etpDatabase: def commitChanges(self): if (not self.readOnly): - dbLog.log(ETP_LOG_VERBOSE,"commitChanges: writing changes to database.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"commitChanges: writing changes to database.") self.connection.commit() self.taintDatabase() else: - dbLog.log(ETP_LOG_VERBOSE,"commitChanges: discarding changes to database (opened readonly).") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"commitChanges: discarding changes to database (opened readonly).") self.discardChanges() # is it ok? def taintDatabase(self): - dbLog.log(ETP_LOG_VERBOSE,"taintDatabase: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"taintDatabase: called.") # taint the database status f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'],"w") f.write(etpConst['currentarch']+" database tainted\n") @@ -882,13 +882,13 @@ class etpDatabase: entropyTools.dbStatus.setDatabaseTaint(True) def untaintDatabase(self): - dbLog.log(ETP_LOG_VERBOSE,"untaintDatabase: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"untaintDatabase: called.") entropyTools.dbStatus.setDatabaseTaint(False) # untaint the database status os.system("rm -f "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) def revisionBump(self): - dbLog.log(ETP_LOG_VERBOSE,"revisionBump: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"revisionBump: called.") if (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'])): revision = 0 else: @@ -902,18 +902,18 @@ class etpDatabase: f.close() def isDatabaseTainted(self): - dbLog.log(ETP_LOG_VERBOSE,"isDatabaseTainted: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isDatabaseTainted: called.") if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): return True return False def discardChanges(self): - dbLog.log(ETP_LOG_VERBOSE,"discardChanges: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"discardChanges: called.") self.connection.rollback() # never use this unless you know what you're doing def initializeDatabase(self): - dbLog.log(ETP_LOG_VERBOSE,"initializeDatabase: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"initializeDatabase: called.") self.cursor.execute(etpSQLInitDestroyAll) self.cursor.execute(etpSQLInit) self.commitChanges() @@ -922,7 +922,7 @@ class etpDatabase: # if it does not exist, it fires up addPackage # otherwise it fires up updatePackage def handlePackage(self, etpData, forceBump = False): - dbLog.log(ETP_LOG_VERBOSE,"handlePackage: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlePackage: called.") if (not self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version'])): update, revision, etpDataUpdated = self.addPackage(etpData) else: @@ -932,7 +932,7 @@ class etpDatabase: # default add an unstable package def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"): - dbLog.log(ETP_LOG_VERBOSE,"addPackage: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addPackage: called.") # Handle package name etpData['download'] = etpData['download'].split(".tbz2")[0] @@ -941,7 +941,7 @@ class etpDatabase: # if a similar package, in the same branch exists, mark for removal searchsimilar = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = wantedBranch) - dbLog.log(ETP_LOG_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"addPackage: here is the list of similar packages (that will be removed) found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilar)) removelist = [] for oldpkg in searchsimilar: # get the package slot @@ -953,9 +953,9 @@ class etpDatabase: for pkg in removelist: self.removePackage(pkg) - dbLog.log(ETP_LOG_VERBOSE,"addPackage: inserting: ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addPackage: inserting: ") for ln in etpData: - dbLog.log(ETP_LOG_VERBOSE,"\t "+ln+": "+str(etpData[ln])) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"\t "+ln+": "+str(etpData[ln])) # wantedBranch = etpData['branch'] self.cursor.execute( 'INSERT into etpData VALUES ' @@ -998,11 +998,11 @@ class etpDatabase: # returns False,revision if not def updatePackage(self, etpData, forceBump = False): - dbLog.log(ETP_LOG_VERBOSE,"updatePackage: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"updatePackage: called.") # are there any stable packages? searchsimilarStable = self.searchSimilarPackages(etpData['category']+"/"+etpData['name'], branch = "stable") - dbLog.log(ETP_LOG_NORMAL,"updatePackage: here is the list of similar stable packages found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilarStable)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: here is the list of similar stable packages found for "+etpData['category']+"/"+etpData['name']+": "+str(searchsimilarStable)) # filter the one with the same version stableFound = False for pkg in searchsimilarStable: @@ -1016,7 +1016,7 @@ class etpDatabase: if (stableFound): - dbLog.log(ETP_LOG_NORMAL,"updatePackage: found an old stable package, if etpData['neededlibs'] is equal, mark the branch of this updated package, stable too") + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: found an old stable package, if etpData['neededlibs'] is equal, mark the branch of this updated package, stable too") # in this case, we should compare etpData['neededlibs'] with the db entry to see if there has been a API breakage dbStoredNeededLibs = self.retrievePackageVar(etpData['category'] + "/" + etpData['name'] + "-" + etpData['version'], "neededlibs", "stable") @@ -1026,7 +1026,7 @@ class etpDatabase: # - same libraries requirements # setup etpData['branch'] accordingly etpData['branch'] = "stable" - dbLog.log(ETP_LOG_NORMAL,"updatePackage: yes, their etpData['neededlibs'] match, marking the new package stable.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: yes, their etpData['neededlibs'] match, marking the new package stable.") # get selected package revision @@ -1049,17 +1049,17 @@ class etpDatabase: # bump revision nevertheless curRevision += 1 - dbLog.log(ETP_LOG_NORMAL,"updatePackage: current revision set to "+str(curRevision)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: current revision set to "+str(curRevision)) # add the new one - dbLog.log(ETP_LOG_NORMAL,"updatePackage: complete. Now spawning addPackage.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"updatePackage: complete. Now spawning addPackage.") self.addPackage(etpData,curRevision,etpData['branch']) # You must provide the full atom to this function # FIXME: this must be fixed to work with branches def removePackage(self,key, branch = "unstable"): - dbLog.log(ETP_LOG_NORMAL,"removePackage: trying to remove (if exists) -> "+str(key)+" | branch: "+branch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"removePackage: trying to remove (if exists) -> "+str(key)+" | branch: "+branch) key = entropyTools.removePackageOperators(key) self.cursor.execute('DELETE FROM etpData WHERE atom = "'+key+'" AND branch = "'+branch+'"') self.commitChanges() @@ -1080,20 +1080,20 @@ class etpDatabase: for i in myEtpData: myEtpData[i] = self.retrievePackageVar(dbPkgInfo,i,dbPkgBranch) - dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: called for "+str(etpData['name'])+" and "+str(myEtpData['name'])+" | branch: "+dbPkgBranch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"comparePackagesData: called for "+str(etpData['name'])+" and "+str(myEtpData['name'])+" | branch: "+dbPkgBranch) for i in etpData: if etpData[i] != myEtpData[i]: - dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: they don't match") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"comparePackagesData: they don't match") return False - dbLog.log(ETP_LOG_VERBOSE,"comparePackagesData: they match") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"comparePackagesData: they match") return True # You must provide the full atom to this function def retrievePackageInfo(self,pkgkey, branch = "unstable"): pkgkey = entropyTools.removePackageOperators(pkgkey) - dbLog.log(ETP_LOG_VERBOSE,"retrievePackageInfo: retrieving package info for "+pkgkey+" | branch: "+branch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageInfo: retrieving package info for "+pkgkey+" | branch: "+branch) result = [] self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"') for row in self.cursor: @@ -1103,7 +1103,7 @@ class etpDatabase: # You must provide the full atom to this function def retrievePackageVar(self,pkgkey,pkgvar, branch = "unstable"): pkgkey = entropyTools.removePackageOperators(pkgkey) - dbLog.log(ETP_LOG_VERBOSE,"retrievePackageVar: retrieving package variable "+pkgvar+" for "+pkgkey+" | branch: "+branch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageVar: retrieving package variable "+pkgvar+" for "+pkgkey+" | branch: "+branch) result = [] self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"') for row in self.cursor: @@ -1117,7 +1117,7 @@ class etpDatabase: # package associated to a certain binary package file (.tbz2) def retrievePackageVarFromBinaryPackage(self,binaryPkgName,pkgvar): # search binary package - dbLog.log(ETP_LOG_VERBOSE,"retrievePackageVarFromBinaryPackage: retrieving package variable "+pkgvar+" for "+binaryPkgName) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"retrievePackageVarFromBinaryPackage: retrieving package variable "+pkgvar+" for "+binaryPkgName) result = [] self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE download = "'+etpConst['binaryurirelativepath']+binaryPkgName+'"') for row in self.cursor: @@ -1130,34 +1130,34 @@ class etpDatabase: # You must provide the full atom to this function # WARNING: this function does not support branches !!! def isPackageAvailable(self,pkgkey): - dbLog.log(ETP_LOG_VERBOSE,"isPackageAvailable: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isPackageAvailable: called.") pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] self.cursor.execute('SELECT * FROM etpData WHERE atom = "'+pkgkey+'"') for row in self.cursor: result.append(row) if result == []: - dbLog.log(ETP_LOG_NORMAL,"isPackageAvailable: "+pkgkey+" not available.") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"isPackageAvailable: "+pkgkey+" not available.") return False - dbLog.log(ETP_LOG_VERBOSE,"isPackageAvailable: "+pkgkey+" available.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isPackageAvailable: "+pkgkey+" available.") return True # This version is more specific and supports branches def isSpecificPackageAvailable(self,pkgkey, branch): - dbLog.log(ETP_LOG_VERBOSE,"isSpecificPackageAvailable: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isSpecificPackageAvailable: called.") pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "'+pkgkey+'" AND branch = "'+branch+'"') for row in self.cursor: result.append(row[0]) if result == []: - dbLog.log(ETP_LOG_NORMAL,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> not found.") + dbLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> not found.") return False - dbLog.log(ETP_LOG_VERBOSE,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> found !") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> found !") return True def searchPackages(self,keyword): - dbLog.log(ETP_LOG_VERBOSE,"searchPackages: called for "+keyword) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchPackages: called for "+keyword) result = [] self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "%'+keyword+'%"') for row in self.cursor: @@ -1165,7 +1165,7 @@ class etpDatabase: return result def searchPackagesInBranch(self,keyword,branch): - dbLog.log(ETP_LOG_VERBOSE,"searchPackagesInBranch: called.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchPackagesInBranch: called.") result = [] self.cursor.execute('SELECT atom FROM etpData WHERE atom LIKE "%'+keyword+'%" AND branch = "'+branch+'"') for row in self.cursor: @@ -1176,7 +1176,7 @@ class etpDatabase: # you must provide something like: media-sound/amarok # optionally, you can add version too. def searchSimilarPackages(self,atom, branch = "unstable"): - dbLog.log(ETP_LOG_VERBOSE,"searchSimilarPackages: called for "+atom+" | branch: "+branch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchSimilarPackages: called for "+atom+" | branch: "+branch) category = atom.split("/")[0] name = atom.split("/")[1] result = [] @@ -1188,7 +1188,7 @@ class etpDatabase: # NOTE: unstable and stable packages are pulled in # so, there might be duplicates! that's normal def listAllPackages(self): - dbLog.log(ETP_LOG_VERBOSE,"listAllPackages: called. ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listAllPackages: called. ") result = [] self.cursor.execute('SELECT atom FROM etpData') for row in self.cursor: @@ -1196,7 +1196,7 @@ class etpDatabase: return result def listAllPackagesTbz2(self): - dbLog.log(ETP_LOG_VERBOSE,"listAllPackagesTbz2: called. ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listAllPackagesTbz2: called. ") result = [] pkglist = self.listAllPackages() for pkg in pkglist: @@ -1213,7 +1213,7 @@ class etpDatabase: return result def listStablePackages(self): - dbLog.log(ETP_LOG_VERBOSE,"listStablePackages: called. ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listStablePackages: called. ") result = [] self.cursor.execute('SELECT atom FROM etpData WHERE branch = "stable"') for row in self.cursor: @@ -1221,7 +1221,7 @@ class etpDatabase: return result def listUnstablePackages(self): - dbLog.log(ETP_LOG_VERBOSE,"listUnstablePackages: called. ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"listUnstablePackages: called. ") result = [] self.cursor.execute('SELECT atom FROM etpData WHERE branch = "unstable"') for row in self.cursor: @@ -1229,7 +1229,7 @@ class etpDatabase: return result def searchStablePackages(self,atom): - dbLog.log(ETP_LOG_VERBOSE,"searchStablePackages: called for "+atom) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchStablePackages: called for "+atom) category = atom.split("/")[0] name = atom.split("/")[1] result = [] @@ -1239,7 +1239,7 @@ class etpDatabase: return result def searchUnstablePackages(self,atom): - dbLog.log(ETP_LOG_VERBOSE,"searchUnstablePackages: called for "+atom) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"searchUnstablePackages: called for "+atom) category = atom.split("/")[0] name = atom.split("/")[1] result = [] @@ -1251,12 +1251,12 @@ class etpDatabase: # useful to quickly retrieve (and trash) all the data # and look for problems. def noopCycle(self): - dbLog.log(ETP_LOG_VERBOSE,"noopCycle: called. ") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"noopCycle: called. ") self.cursor.execute('SELECT * FROM etpData') def stabilizePackage(self,atom,stable = True): - dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable)) action = "unstable" removeaction = "stable" @@ -1264,10 +1264,10 @@ class etpDatabase: action = "stable" removeaction = "unstable" - dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: add action: "+action+" | remove action: "+removeaction) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: add action: "+action+" | remove action: "+removeaction) if (self.isSpecificPackageAvailable(atom, removeaction)): - dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: there's something old that needs to be removed.") + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: there's something old that needs to be removed.") # ! Get rid of old entries with the same slot, pkgcat/name that # were already marked "stable" # get its pkgname @@ -1285,10 +1285,10 @@ class etpDatabase: myslot = self.retrievePackageVar(result,"slot", branch = action) if (myslot == slot): removelist.append(result) - dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: removelist: "+str(removelist)) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: removelist: "+str(removelist)) for pkg in removelist: self.removePackage(pkg, branch = action) - dbLog.log(ETP_LOG_VERBOSE,"stabilizePackage: updating "+atom+" setting branch: "+action) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"stabilizePackage: updating "+atom+" setting branch: "+action) self.cursor.execute('UPDATE etpData SET branch = "'+action+'" WHERE atom = "'+atom+'" AND branch = "'+removeaction+'"') self.commitChanges() @@ -1296,7 +1296,7 @@ class etpDatabase: return False,action def writePackageParameter(self,atom,field,what,branch): - dbLog.log(ETP_LOG_VERBOSE,"writePackageParameter: writing '"+what+"' into field '"+field+"' for '"+atom+"' | branch: "+branch) + dbLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"writePackageParameter: writing '"+what+"' into field '"+field+"' for '"+atom+"' | branch: "+branch) self.cursor.execute('UPDATE etpData SET '+field+' = "'+what+'" WHERE atom = "'+atom+'" AND branch = "'+branch+'"') self.commitChanges() diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 9723bf5b3..64a9a8c25 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -126,6 +126,9 @@ ETP_ROOT_DIR = "/" ETP_LOG_DIR = ETP_DIR+"/"+"logs" ETP_LOG_NORMAL = 1 ETP_LOG_VERBOSE = 2 +ETP_LOG_INFO = "[ INFO ]" +ETP_LOG_WARNING = "[ WARNING ]" +ETP_LOG_ERROR = "[ ERROR ]" # NEVER APPEND another \n to this file because it will break the md5 check of reagent ETP_HEADER_TEXT = "# Sabayon Linux (C - 2007) - Entropy Package Specifications (GPLv2)\n" MAX_ETP_REVISION_COUNT = 99999 @@ -152,6 +155,7 @@ etpConst = { 'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file 'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file 'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # Source Package Manager backend configuration (Portage now) + 'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file 'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf) 'activatordownloaduris': [], # list of URIs that activator can use to fetch data 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. @@ -166,19 +170,24 @@ etpConst = { 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) 'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file - 'databaseloglevel': 1, # Database log level (default: 1 - see enzyme.conf for more info) + 'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info) + 'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info) 'enzymeloglevel': 1 , # Enzyme log level (default: 1 - see enzyme.conf for more info) 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) 'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info) + 'spmbackendloglevel': 1, # # Source Package Manager backend log level (default: 1 - see entropy.conf for more info) 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit - 'databaselogfile': ETP_LOG_DIR+"/database.log", # database operations log file + 'mirrorslogfile': ETP_LOG_DIR+"/mirrors.log", # Mirrors operations log file + 'spmbackendlogfile': ETP_LOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file + 'databaselogfile': ETP_LOG_DIR+"/database.log", # Database operations log file 'enzymelogfile': ETP_LOG_DIR+"/enzyme.log", # Enzyme operations log file 'reagentlogfile': ETP_LOG_DIR+"/reagent.log", # Reagent operations log file 'activatorlogfile': ETP_LOG_DIR+"/activator.log", # Activator operations log file 'entropylogfile': ETP_LOG_DIR+"/entropy.log", # Activator operations log file + 'distcc-status': False, # used by Enzyme, if True distcc is enabled 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file 'etpdatabasedir': ETP_DIR+ETP_DBDIR, # diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 7f36a3dcd..e6fe529e4 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -37,7 +37,7 @@ dbStatus = databaseTools.databaseStatus() # Logging initialization import logTools entropyLog = logTools.LogFile(level=etpConst['entropyloglevel'],filename = etpConst['entropylogfile'], header = "[Entropy]") -# example: entropyLog.log(ETP_LOG_VERBOSE,"testFuncton: called.") +# example: entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.") # EXIT STATUSES: 100-199 @@ -87,7 +87,7 @@ def md5sum(filepath): # @returns the complete hash file path # FIXME: add more hashes, SHA1 for example def createHashFile(tbz2filepath): - entropyLog.log(ETP_LOG_VERBOSE,"createHashFile: for "+tbz2filepath) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"createHashFile: for "+tbz2filepath) md5hash = md5sum(tbz2filepath) hashfile = tbz2filepath+etpConst['packageshashfileext'] f = open(hashfile,"w") @@ -98,14 +98,14 @@ def createHashFile(tbz2filepath): return hashfile def compareMd5(filepath,checksum): - entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareMd5: called. ") checksum = str(checksum) result = md5sum(filepath) result = str(result) if checksum == result: - entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: match. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareMd5: match. ") return True - entropyLog.log(ETP_LOG_VERBOSE,"compareMd5: no match. ") + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"compareMd5: no match. ") return False def md5string(string): @@ -278,7 +278,7 @@ def filterDuplicatedEntries(nameslist): # Tool to run commands def spawnCommand(command, redirect = None): - entropyLog.log(ETP_LOG_VERBOSE,"spawnCommand: called for: "+command) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"spawnCommand: called for: "+command) if redirect is not None: command += " "+redirect rc = os.system(command) @@ -294,7 +294,7 @@ def extractFTPHostFromUri(uri): # This function check the Entropy online database status def getEtpRemoteDatabaseStatus(): - entropyLog.log(ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: called.") uriDbInfo = [] for uri in etpConst['activatoruploaduris']: @@ -320,13 +320,13 @@ def getEtpRemoteDatabaseStatus(): uriDbInfo.append(info) ftp.closeFTPConnection() - entropyLog.log(ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: dump -> "+str(uriDbInfo)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEtpRemoteDatabaseStatus: dump -> "+str(uriDbInfo)) return uriDbInfo def syncRemoteDatabases(noUpload = False, justStats = False): - entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: called.") remoteDbsStatus = getEtpRemoteDatabaseStatus() print_info(green(" * ")+red("Remote Entropy Database Repository Status:")) @@ -439,13 +439,13 @@ def syncRemoteDatabases(noUpload = False, justStats = False): uploadList.append(dbstat) if (downloadLatest == []) and (not uploadLatest): - entropyLog.log(ETP_LOG_NORMAL,"syncRemoteDatabases: online database does not need to be updated.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"syncRemoteDatabases: online database does not need to be updated.") print_info(green(" * ")+red("Online database does not need to be updated.")) return # now run the selected task! if (downloadLatest != []): - entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: download latest database needed.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: download latest database needed.") # match the proper URI for uri in etpConst['activatoruploaduris']: if downloadLatest[0].startswith(uri): @@ -453,7 +453,7 @@ def syncRemoteDatabases(noUpload = False, justStats = False): downloadDatabase(downloadLatest[0]) if (uploadLatest) and (not noUpload): - entropyLog.log(ETP_LOG_VERBOSE,"syncRemoteDatabases: some mirrors don't have the latest database.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"syncRemoteDatabases: some mirrors don't have the latest database.") print_info(green(" * ")+red("Starting to update the needed mirrors ...")) _uploadList = [] for uri in etpConst['activatoruploaduris']: @@ -474,7 +474,7 @@ def syncRemoteDatabases(noUpload = False, justStats = False): def uploadDatabase(uris): - entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: called.") # our fancy compressor :-) import gzip @@ -482,7 +482,7 @@ def uploadDatabase(uris): for uri in uris: downloadLockDatabases(True,[uri]) - entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading data to: "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: uploading data to: "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Uploading database to ")+bold(extractFTPHostFromUri(uri))+red(" ...")) print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True) @@ -531,10 +531,10 @@ def uploadDatabase(uris): print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" ..."), back = True) rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],True) if (rc == True): - entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" successfull.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" successfull.") print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" completed. Disconnecting.")) else: - entropyLog.log(ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.") + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.") print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) # close connection @@ -545,11 +545,11 @@ def uploadDatabase(uris): def downloadDatabase(uri): - entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: called.") import gzip - entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading from -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading from -> "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Downloading database from ")+bold(extractFTPHostFromUri(uri))+red(" ...")) print_info(green(" * ")+red("Connecting to ")+bold(extractFTPHostFromUri(uri))+red(" ..."), back = True) @@ -579,7 +579,7 @@ def downloadDatabase(uri): print_info(green(" * ")+red("Decompression of ")+bold(etpConst['etpdatabasefilegzip'])+red(" completed.")) # downloading revision file - entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading revision file for "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading revision file for "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True) rc = ftp.downloadFile(etpConst['etpdatabaserevisionfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True) if (rc == True): @@ -587,16 +587,17 @@ def downloadDatabase(uri): else: print_warning(yellow(" * ")+red("Cannot properly download to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) - entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: downloading digest file for "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: downloading digest file for "+extractFTPHostFromUri(uri)) # downlading digest print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasehashfile'])+red(" ..."), back = True) rc = ftp.downloadFile(etpConst['etpdatabasehashfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True) if (rc == True): print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabasehashfile'])+red(" completed. Disconnecting.")) else: - print_warning(yellow(" * ")+red("Cannot properly download to ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"downloadDatabase: Cannot properly download from "+extractFTPHostFromUri(uri)+". Please check.") + print_warning(yellow(" * ")+red("Cannot properly download from ")+bold(extractFTPHostFromUri(uri))+red(". Please check.")) - entropyLog.log(ETP_LOG_VERBOSE,"downloadDatabase: do some tidy.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadDatabase: do some tidy.") os.system("rm -f " + etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasefilegzip']+" &> /dev/null") # close connection ftp.closeFTPConnection() @@ -606,7 +607,7 @@ def downloadDatabase(uri): # @ the second parameter is referred to upload locks, while the second to download ones def getMirrorsLock(): - entropyLog.log(ETP_LOG_VERBOSE,"getMirrorsLock: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getMirrorsLock: called.") # parse etpConst['activatoruploaduris'] dbstatus = [] @@ -627,13 +628,13 @@ def getMirrorsLock(): def downloadPackageFromMirror(uri,pkgfile): - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+pkgfile) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+pkgfile) tries = 0 maxtries = 5 for i in range(maxtries): - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") downloading -> "+pkgfile) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") downloading -> "+pkgfile) pkgfilename = pkgfile.split("/")[len(pkgfile.split("/"))-1] print_info(red(" * Connecting to ")+bold(extractFTPHostFromUri(uri)), back = True) @@ -644,12 +645,12 @@ def downloadPackageFromMirror(uri,pkgfile): print_info(red(" * Downloading ")+yellow(pkgfilename)+red(" from ")+bold(extractFTPHostFromUri(uri))) rc = ftp.downloadFile(pkgfilename,etpConst['packagesbindir']) if (rc is None): - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") --- ERROR --- FILE NOT FOUND -> "+pkgfile) + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") Error. File not found. -> "+pkgfile) # file does not exist print_warning(red(" * File ")+yellow(pkgfilename)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri))) ftp.closeFTPConnection() return None - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") checking md5 for -> "+pkgfile) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadPackageFromMirror: ("+tries+"/"+maxtries+") checking md5 for -> "+pkgfile) # check md5 dbconn = databaseTools.etpDatabase(readOnly = True) storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkgfilename,"digest") @@ -661,11 +662,11 @@ def downloadPackageFromMirror(uri,pkgfile): return True else: if (tries == maxtries): - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.") + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.") print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Please consider to download or repackage again. Giving up.")) return False else: - entropyLog.log(ETP_LOG_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...") + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...") print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Trying to download it again...")) tries += 1 if os.path.isfile(etpConst['packagesbindir']+"/"+pkgfilename): @@ -684,7 +685,7 @@ def compressTarBz2(storepath,pathtocompress): # tar.bz2 uncompress function... def uncompressTarBz2(filepath, extractPath = None): - entropyLog.log(ETP_LOG_VERBOSE,"uncompressTarBz2: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"uncompressTarBz2: called. ") if extractPath is None: extractPath = os.path.dirname(filepath) cmd = "tar xjf "+filepath+" -C "+extractPath @@ -704,7 +705,7 @@ def bytesIntoHuman(bytes): # hide password from full ftp URI def hideFTPpassword(uri): - entropyLog.log(ETP_LOG_VERBOSE,"hideFTPpassword: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"hideFTPpassword: called. ") ftppassword = uri.split("@")[:len(uri.split("@"))-1] if len(ftppassword) > 1: import string @@ -723,14 +724,14 @@ def hideFTPpassword(uri): def lockDatabases(lock = True, mirrorList = []): - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: called. ") outstat = False if (mirrorList == []): mirrorList = etpConst['activatoruploaduris'] for uri in mirrorList: - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: locking? "+str(lock)+" for: "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: locking? "+str(lock)+" for: "+extractFTPHostFromUri(uri)) if (lock): print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror..."),back = True) @@ -742,13 +743,13 @@ def lockDatabases(lock = True, mirrorList = []): # check if the lock is already there if (lock): if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])): - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already locked.") + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already locked.") print_info(green(" * ")+red("Mirror database at ")+bold(extractFTPHostFromUri(uri))+red(" already locked.")) ftp.closeFTPConnection() continue else: if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])): - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already unlocked.") + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" already unlocked.") print_info(green(" * ")+red("Mirror database at ")+bold(extractFTPHostFromUri(uri))+red(" already unlocked.")) ftp.closeFTPConnection() continue @@ -759,24 +760,24 @@ def lockDatabases(lock = True, mirrorList = []): f.close() rc = ftp.uploadFile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],ascii= True) if (rc == True): - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully locked.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully locked.") print_info(green(" * ")+red("Succesfully locked ")+bold(extractFTPHostFromUri(uri))+red(" mirror.")) else: outstat = True print "\n" - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while locking.") + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while locking.") print_warning(red(" * ")+red("A problem occured while locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror. Please have a look.")) if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: rc = ftp.deleteFile(etpConst['etpdatabaselockfile']) if (rc): - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully unlocked.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" successfully unlocked.") print_info(green(" * ")+red("Succesfully unlocked ")+bold(extractFTPHostFromUri(uri))+red(" mirror.")) if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: - entropyLog.log(ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while unlocking.") + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"lockDatabases: mirror "+extractFTPHostFromUri(uri)+" had an unknown issue while unlocking.") outstat = True print "\n" print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(extractFTPHostFromUri(uri))+red(" mirror. Please have a look.")) @@ -785,17 +786,17 @@ def lockDatabases(lock = True, mirrorList = []): def downloadLockDatabases(lock = True, mirrorList = []): - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: called. ") outstat = False if (mirrorList == []): mirrorList = etpConst['activatoruploaduris'] for uri in mirrorList: if (lock): - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: download locking -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: download locking -> "+extractFTPHostFromUri(uri)) print_info(yellow(" * ")+red("Locking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True) else: - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: download unlocking -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: download unlocking -> "+extractFTPHostFromUri(uri)) print_info(yellow(" * ")+red("Unlocking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror..."),back = True) ftp = mirrorTools.handlerFTP(uri) # upload the lock file to database/%ARCH% directory @@ -808,7 +809,7 @@ def downloadLockDatabases(lock = True, mirrorList = []): continue else: if (not ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])): - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: already unlocked -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"downloadLockDatabases: already unlocked -> "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Download mirror at ")+bold(extractFTPHostFromUri(uri))+red(" already unlocked.")) ftp.closeFTPConnection() continue @@ -819,20 +820,20 @@ def downloadLockDatabases(lock = True, mirrorList = []): f.close() rc = ftp.uploadFile(etpConst['packagestmpdir']+"/"+etpConst['etpdatabasedownloadlockfile'],ascii= True) if (rc == True): - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: successfully locked -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: successfully locked -> "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Succesfully locked ")+bold(extractFTPHostFromUri(uri))+red(" download mirror.")) else: - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download lock -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download lock -> "+extractFTPHostFromUri(uri)) outstat = True print "\n" print_warning(red(" * ")+red("A problem occured while locking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror. Please have a look.")) else: rc = ftp.deleteFile(etpConst['etpdatabasedownloadlockfile']) if (rc): - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: successfully unlocked -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"downloadLockDatabases: successfully unlocked -> "+extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Succesfully unlocked ")+bold(extractFTPHostFromUri(uri))+red(" download mirror.")) else: - entropyLog.log(ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download unlock -> "+extractFTPHostFromUri(uri)) + entropyLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"downloadLockDatabases: a problem occured while trying to download unlock -> "+extractFTPHostFromUri(uri)) outstat = True print "\n" print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(extractFTPHostFromUri(uri))+red(" download mirror. Please have a look.")) @@ -840,7 +841,7 @@ def downloadLockDatabases(lock = True, mirrorList = []): return outstat def getLocalDatabaseRevision(): - entropyLog.log(ETP_LOG_VERBOSE,"getLocalDatabaseRevision: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getLocalDatabaseRevision: called. ") if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile']): f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile']) rev = f.readline().strip() @@ -853,7 +854,7 @@ def getLocalDatabaseRevision(): # parse a dumped .etp file and returns etpData def parseEtpDump(file): - entropyLog.log(ETP_LOG_VERBOSE,"parseEtpDump: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"parseEtpDump: called. ") myEtpData = etpData.copy() # reset @@ -872,7 +873,7 @@ def parseEtpDump(file): # Distcc check status function def setDistCC(status = True): - entropyLog.log(ETP_LOG_VERBOSE,"setDistCC: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"setDistCC: called. ") f = open(etpConst['enzymeconf'],"r") enzymeconf = f.readlines() @@ -893,7 +894,7 @@ def setDistCC(status = True): def getDistCCHosts(): - entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: called.") f = open(etpConst['enzymeconf'],"r") enzymeconf = f.readlines() @@ -904,15 +905,15 @@ def getDistCCHosts(): line = line.strip().split("|")[1].split() for host in line: hostslist.append(host) - entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: hosts list dump -> "+str(hostslist)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: hosts list dump -> "+str(hostslist)) return hostslist - entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: hosts list EMPTY.") + entropyLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getDistCCHosts: hosts list EMPTY.") return [] # @returns True if validIP (type: string) is a valid IP # @param validIP: IP string def isValidIP(validIP): - entropyLog.log(ETP_LOG_VERBOSE,"getDistCCHosts: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getDistCCHosts: called. ") validIPExpr = re.compile('(([0-9]|[01]?[0-9]{2}|2([0-4][0-9]|5[0-5]))\.){3}([0-9]|[01]?[0-9]{2}|2([0-4][0-9]|5[0-5]))$') result = validIPExpr.match(validIP) @@ -923,7 +924,7 @@ def isValidIP(validIP): # you must provide a list def addDistCCHosts(hosts): - entropyLog.log(ETP_LOG_VERBOSE,"addDistCCHosts: called.") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addDistCCHosts: called.") hostslist = getDistCCHosts() for host in hosts: @@ -932,7 +933,7 @@ def addDistCCHosts(hosts): # filter dupies hostslist = list(set(hostslist)) - entropyLog.log(ETP_LOG_VERBOSE,"addDistCCHosts: hostslist dump -> "+str(hostslist)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"addDistCCHosts: hostslist dump -> "+str(hostslist)) # write back to file f = open(etpConst['enzymeconf'],"r") @@ -968,7 +969,7 @@ def addDistCCHosts(hosts): # you must provide a list def removeDistCCHosts(hosts): - entropyLog.log(ETP_LOG_VERBOSE,"removeDistCCHosts: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"removeDistCCHosts: called. ") hostslist = getDistCCHosts() cleanedhosts = [] @@ -985,7 +986,7 @@ def removeDistCCHosts(hosts): # filter dupies cleanedhosts = list(set(cleanedhosts)) - entropyLog.log(ETP_LOG_VERBOSE,"removeDistCCHosts: cleanedhosts dump: "+cleanedhosts) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"removeDistCCHosts: cleanedhosts dump: "+cleanedhosts) # write back to file f = open(etpConst['enzymeconf'],"r") @@ -1023,7 +1024,7 @@ def getDistCCStatus(): def isIPAvailable(ip): - entropyLog.log(ETP_LOG_VERBOSE,"isIPAvailable: called. ") + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isIPAvailable: called. ") rc = os.system("ping -c 1 "+ip+" &> /dev/null") if (rc): @@ -1087,7 +1088,7 @@ def alphaSorter(seq): # Temporary files cleaner def cleanup(options): - entropyLog.log(ETP_LOG_VERBOSE,"cleanup: with options: "+str(options)) + entropyLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"cleanup: with options: "+str(options)) toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ] counter = 0 diff --git a/libraries/logTools.py b/libraries/logTools.py index 86dba7146..fda2950c8 100644 --- a/libraries/logTools.py +++ b/libraries/logTools.py @@ -66,9 +66,9 @@ class LogFile: def set_loglevel(self, level): self.level = level - def log(self, level, message): + def log(self, messagetype, level, message): if self.level >= level: - self.handler(self.getTimeDateHeader()+self.header+' '+message) + self.handler(self.getTimeDateHeader()+messagetype+' '+self.header+' '+message) def getTimeDateHeader(self): return time.strftime('[%X %x %Z] ') diff --git a/libraries/mirrorTools.py b/libraries/mirrorTools.py index ff8144e94..0b2b27afd 100644 --- a/libraries/mirrorTools.py +++ b/libraries/mirrorTools.py @@ -29,12 +29,20 @@ import entropyTools import string import os +# Logging initialization +import logTools +mirrorLog = logTools.LogFile(level=etpConst['mirrorsloglevel'],filename = etpConst['mirrorslogfile'], header = "[Mirrors]") +# example: mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFuncton: called.") + + class handlerFTP: # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org # this must be run before calling the other functions def __init__(self, ftpuri): - + + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.__init__: called.") + from ftplib import FTP self.ftpuri = ftpuri @@ -81,40 +89,52 @@ class handlerFTP: # this can be used in case of exceptions def reconnectHost(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.reconnectHost: called.") self.ftpconn = FTP(self.ftphost) self.ftpconn.login(self.ftpuser,self.ftppassword) self.ftpconn.cwd(self.currentdir) def getFTPHost(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPHost: called -> "+self.ftphost) return self.ftphost def getFTPPort(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpport) return self.ftpport def getFTPDir(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFTPPort: called -> "+self.ftpdir) return self.ftpdir def getCWD(self): - return self.ftpconn.pwd() + pwd = self.ftpconn.pwd() + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getCWD: called -> "+pwd) + return pwd def setCWD(self,dir): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.setCWD: called -> "+dir) self.ftpconn.cwd(dir) self.currentdir = dir def setPASV(self,bool): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.setPASV: called -> "+str(bool)) self.ftpconn.set_pasv(bool) def getFileMtime(self,path): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileMtime: called for -> "+path) rc = self.ftpconn.sendcmd("mdtm "+path) return rc.split()[len(rc.split())-1] def spawnFTPCommand(self,cmd): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.spawnFTPCommand: called, command -> "+cmd) rc = self.ftpconn.sendcmd(cmd) + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.spawnFTPCommand: called, rc -> "+str(rc)) return rc # list files and directory of a FTP # @returns a list def listFTPdir(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.listFTPdir: called.") # directory is: self.ftpdir try: rc = self.ftpconn.nlst() @@ -129,6 +149,7 @@ class handlerFTP: # list if the file is available # @returns True or False def isFileAvailable(self,filename): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.isFileAvailable: called for -> "+filename) # directory is: self.ftpdir try: rc = self.ftpconn.nlst() @@ -138,23 +159,31 @@ class handlerFTP: rc = _rc for i in rc: if i == filename: + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.isFileAvailable: result -> True") return True + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_WARNING,"handlerFTP.isFileAvailable: result -> False (no exception)") return False except: + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_ERROR,"handlerFTP.isFileAvailable: result -> False (exception occured!!!)") return False def deleteFile(self,file): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: called for -> "+str(file)) try: rc = self.ftpconn.delete(file) if rc.startswith("250"): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> True") return True else: + mirrorLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> False (no exception)") return False except: + mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"handlerFTP.deleteFile: result -> False (exception occured!!!)") return False # this function also supports callback, because storbinary doesn't def advancedStorBinary(self, cmd, fp, callback=None, blocksize=8192): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.advancedStorBinary: called with -> "+str(cmd)) ''' Store a file in binary mode. Our version supports a callback function''' self.ftpconn.voidcmd('TYPE I') conn = self.ftpconn.transfercmd(cmd) @@ -178,7 +207,12 @@ class handlerFTP: # print ! print_info(currentText,back = True) + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"handlerFTP.uploadFile: called for -> "+str(file)+" mode, ascii?: "+str(ascii)) + for i in range(10): # ten tries + + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.uploadFile: try #"+str(i)) + f = open(file) filename = file.split("/")[len(file.split("/"))-1] try: @@ -197,10 +231,13 @@ class handlerFTP: self.renameFile(filename+".tmp",filename) f.close() if rc.find("226") != -1: # upload complete + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.uploadFile: upload complete.") return True else: + mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.uploadFile: upload failed !!.") return False except: # connection reset by peer + mirrorLog.log(ETP_LOG_WARNING,ETP_LOG_NORMAL,"handlerFTP.uploadFile: upload issues, retrying...") print_info(red("Upload issue, retrying...")) self.reconnectHost() # reconnect if self.isFileAvailable(filename): @@ -222,6 +259,8 @@ class handlerFTP: # print ! print_info(currentText,back = True) + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_NORMAL,"handlerFTP.downloadFile: called for -> "+str(filepath)+" | download directory: "+str(downloaddir)+" | ascii? "+str(ascii)) + file = filepath.split("/")[len(filepath.split("/"))-1] # look if the file exist if self.isFileAvailable(file): @@ -243,21 +282,28 @@ class handlerFTP: f.flush() f.close() if rc.find("226") != -1: # upload complete + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.downloadFile: download success.") return True else: + mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.downloadFile: download issues !!.") return False else: + mirrorLog.log(ETP_LOG_ERROR,ETP_LOG_NORMAL,"handlerFTP.downloadFile: file '"+file+"' not available !!.") return None # also used to move files def renameFile(self,fromfile,tofile): - self.ftpconn.rename(fromfile,tofile) + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.renameFile: rename file from '"+fromfile+"' to '"+tofile+"'.") + rc = self.ftpconn.rename(fromfile,tofile) + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.renameFile: return output: '"+rc+"'") # not supported by dreamhost.com def getFileSize(self,file): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileSize: called for -> "+file) return self.ftpconn.size(file) def getFileSizeCompat(self,file): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getFileSizeCompat: called for -> "+file) list = self.getRoughList() for item in list: if item.find(file) != -1: @@ -269,9 +315,11 @@ class handlerFTP: self.FTPbuffer.append(buf) def getRoughList(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.getRoughList: called.") self.FTPbuffer = [] self.ftpconn.dir(self.bufferizer) return self.FTPbuffer def closeFTPConnection(self): + mirrorLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"handlerFTP.closeFTPConnection: called.") self.ftpconn.quit() diff --git a/libraries/portageTools.py b/libraries/portageTools.py index e95f98708..858e37631 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -61,7 +61,7 @@ import entropyTools # Logging initialization import logTools portageLog = logTools.LogFile(level=etpConst['spmbackendloglevel'],filename = etpConst['spmbackendlogfile'], header = "[Portage]") -# portageLog.log(ETP_LOG_VERBOSE,"testFunction: example. ") +# portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFunction: example. ") ############ # Functions and Classes @@ -71,41 +71,41 @@ def getThirdPartyMirrors(mirrorname): return portage.thirdpartymirrors[mirrorname] def getPortageEnv(var): - portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPortageEnv: called. ") try: rc = portage.config(clone=portage.settings).environ()[var] - portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: variable available -> "+str(var)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPortageEnv: variable available -> "+str(var)) return rc except KeyError: - portageLog.log(ETP_LOG_VERBOSE,"getPortageEnv: variable not available -> "+str(var)) + portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getPortageEnv: variable not available -> "+str(var)) return None # resolve atoms automagically (best, not current!) # sys-libs/application --> sys-libs/application-1.2.3-r1 def getBestAtom(atom): - portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: called -> "+str(atom)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: called -> "+str(atom)) try: rc = portage.portdb.xmatch("bestmatch-visible",str(atom)) - portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc)) return rc except ValueError: - portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: conflict found. ") + portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getBestAtom: conflict found. ") return "!!conflicts" # same as above but includes masked ebuilds def getBestMaskedAtom(atom): - portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: called. ") atoms = portage.portdb.xmatch("match-all",str(atom)) # find the best from portage_versions import best rc = best(atoms) - portageLog.log(ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getBestAtom: result -> "+str(rc)) return rc # I need a valid complete atom... def calculateFullAtomsDependencies(atoms, deep = False, extraopts = ""): - portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: called -> "+str(atoms)+" | extra options: "+extraopts) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: called -> "+str(atoms)+" | extra options: "+extraopts) # in order... thanks emerge :-) deepOpt = "" @@ -141,17 +141,17 @@ def calculateFullAtomsDependencies(atoms, deep = False, extraopts = ""): blocklist = _blocklist if deplist != []: - portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist -> "+len(deplist)+" | blocklist -> "+len(blocklist)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist -> "+len(deplist)+" | blocklist -> "+len(blocklist)) return deplist, blocklist else: - portageLog.log(ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist empty. Giving up.") + portageLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"calculateFullAtomsDependencies: deplist empty. Giving up.") rc = os.system(cmd) sys.exit(100) def calculateAtomUSEFlags(atom): - portageLog.log(ETP_LOG_VERBOSE,"calculateAtomUSEFlags: called -> "+str(atom)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateAtomUSEFlags: called -> "+str(atom)) try: useflags = "USE='"+os.environ['USE']+"' " @@ -166,7 +166,7 @@ def calculateAtomUSEFlags(atom): useparm = useparm.split() _useparm = [] - portageLog.log(ETP_LOG_VERBOSE,"calculateAtomUSEFlags: USE flags -> "+str(useparm)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"calculateAtomUSEFlags: USE flags -> "+str(useparm)) for use in useparm: # -cups @@ -190,12 +190,12 @@ def calculateAtomUSEFlags(atom): # should be only used when a pkgcat/pkgname <-- is not specified (example: db, amarok, AND NOT media-sound/amarok) def getAtomCategory(atom): - portageLog.log(ETP_LOG_VERBOSE,"getAtomCategory: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getAtomCategory: called. ") try: rc = portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0] return rc except: - portageLog.log(ETP_LOG_VERBOSE,"getAtomCategory: error, can't extract category !") + portageLog.log(ETP_LOG_ERROR,ETP_LOG_VERBOSE,"getAtomCategory: error, can't extract category !") return None # This function compare the version number of two atoms @@ -204,7 +204,7 @@ def getAtomCategory(atom): # if atom1 > atom2 --> returns a POSITIVE number # if atom1 == atom2 --> returns 0 def compareAtoms(atom1,atom2): - portageLog.log(ETP_LOG_VERBOSE,"compareAtoms: called -> "+atom1+" && "+atom2) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareAtoms: called -> "+atom1+" && "+atom2) # filter pkgver x, atom1 = extractPkgNameVer(atom1) x, atom2 = extractPkgNameVer(atom2) @@ -213,7 +213,7 @@ def compareAtoms(atom1,atom2): # please always force =pkgcat/pkgname-ver if possible def getInstalledAtom(atom): - portageLog.log(ETP_LOG_VERBOSE,"getInstalledAtom: called -> "+str(atom)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledAtom: called -> "+str(atom)) rc = portage.db['/']['vartree'].dep_match(str(atom)) if (rc != []): if (len(rc) == 1): @@ -224,20 +224,20 @@ def getInstalledAtom(atom): return None def getPackageSlot(atom): - portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageSlot: called. ") if atom.startswith("="): atom = atom[1:] rc = portage.db['/']['vartree'].getslot(atom) if rc != "": - portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: slot found -> "+str(atom)+" -> "+str(rc)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageSlot: slot found -> "+str(atom)+" -> "+str(rc)) return rc else: - portageLog.log(ETP_LOG_VERBOSE,"getPackageSlot: slot not found -> "+str(atom)) + portageLog.log(ETP_LOG_WARNING,ETP_LOG_VERBOSE,"getPackageSlot: slot not found -> "+str(atom)) return None # you must provide a complete atom def collectBinaryFilesForInstalledPackage(atom): - portageLog.log(ETP_LOG_VERBOSE,"collectBinaryFilesForInstalledPackage: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"collectBinaryFilesForInstalledPackage: called. ") if atom.startswith("="): atom = atom[1:] pkgcat = atom.split("/")[0] @@ -258,11 +258,11 @@ def collectBinaryFilesForInstalledPackage(atom): return binarylibs def getEbuildDbPath(atom): - portageLog.log(ETP_LOG_VERBOSE,"getEbuildDbPath: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEbuildDbPath: called -> "+atom) return portage.db['/']['vartree'].getebuildpath(atom) def getEbuildTreePath(atom): - portageLog.log(ETP_LOG_VERBOSE,"getEbuildTreePath: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getEbuildTreePath: called -> "+atom) if atom.startswith("="): atom = atom[1:] rc = portage.portdb.findname(atom) @@ -272,7 +272,7 @@ def getEbuildTreePath(atom): return None def getPackageDownloadSize(atom): - portageLog.log(ETP_LOG_VERBOSE,"getPackageDownloadSize: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageDownloadSize: called -> "+atom) if atom.startswith("="): atom = atom[1:] @@ -298,7 +298,7 @@ def getPackageDownloadSize(atom): return "N/A" def getInstalledAtoms(atom): - portageLog.log(ETP_LOG_VERBOSE,"getInstalledAtoms: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledAtoms: called -> "+atom) rc = portage.db['/']['vartree'].dep_match(str(atom)) if (rc != []): return rc @@ -309,7 +309,7 @@ def getInstalledAtoms(atom): # YOU MUST PROVIDE A COMPLETE ATOM with a pkgcat ! def unmerge(atom): - portageLog.log(ETP_LOG_VERBOSE,"umerge: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"umerge: called -> "+atom) if isjustname(atom) or (not isvalidatom(atom)) or (atom.find("/") == -1): return 1 else: @@ -323,7 +323,7 @@ def unmerge(atom): # TO THIS FUNCTION: # must be provided a valid and complete atom def extractPkgNameVer(atom): - portageLog.log(ETP_LOG_VERBOSE,"extractPkgNameVer: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"extractPkgNameVer: called -> "+atom) package = dep_getcpv(atom) package = atom.split("/")[len(atom.split("/"))-1] package = package.split("-") @@ -343,7 +343,7 @@ def extractPkgNameVer(atom): return pkgname,pkgver def emerge(atom, options, outfile = None, redirect = "&>", simulate = False): - portageLog.log(ETP_LOG_VERBOSE,"emerge: called -> "+atom+" | options: "+str(options)+" | redirect: "+str(redirect)+" | outfile: "+str(outfile)+" | simulate: "+str(simulate)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"emerge: called -> "+atom+" | options: "+str(options)+" | redirect: "+str(redirect)+" | outfile: "+str(outfile)+" | simulate: "+str(simulate)) if (simulate): return 0,"" # simulation enabled if (outfile is None) and (redirect == "&>"): @@ -381,7 +381,7 @@ def emerge(atom, options, outfile = None, redirect = "&>", simulate = False): except: ldflags = " " - portageLog.log(ETP_LOG_VERBOSE,"emerge: USE: "+useflags+" | CFLAGS: "+cflags+" | MAKEOPTS: "+makeopts+" | LDFLAGS: "+ldflags) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"emerge: USE: "+useflags+" | CFLAGS: "+cflags+" | MAKEOPTS: "+makeopts+" | LDFLAGS: "+ldflags) # elog configuration elogopts = dbPORTAGE_ELOG_OPTS+" " @@ -405,7 +405,7 @@ def emerge(atom, options, outfile = None, redirect = "&>", simulate = False): def parseElogFile(atom): - portageLog.log(ETP_LOG_VERBOSE,"parseElogFile: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"parseElogFile: called. ") if atom.startswith("="): atom = atom[1:] @@ -445,7 +445,7 @@ def parseElogFile(atom): def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles): - portageLog.log(ETP_LOG_VERBOSE,"compareLibraryLists: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"compareLibraryLists: called. ") brokenBinariesList = [] # check if there has been a API breakage @@ -488,7 +488,7 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles): # create a .tbz2 file in the specified path def quickpkg(atom,dirpath): - portageLog.log(ETP_LOG_VERBOSE,"quickpkg: called -> "+atom+" | dirpath: "+dirpath) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"quickpkg: called -> "+atom+" | dirpath: "+dirpath) # getting package info pkgname = atom.split("/")[1] @@ -534,7 +534,7 @@ def quickpkg(atom,dirpath): def unpackTbz2(tbz2File,tmpdir = None): - portageLog.log(ETP_LOG_VERBOSE,"unpackTbz2: called -> "+tbz2File) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"unpackTbz2: called -> "+tbz2File) import xpak if tmpdir is None: @@ -550,7 +550,7 @@ def unpackTbz2(tbz2File,tmpdir = None): # NOTE: atom must be a COMPLETE atom, with version! def isTbz2PackageAvailable(atom, verbose = False): - portageLog.log(ETP_LOG_VERBOSE,"isTbz2PackageAvailable: called -> "+atom) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isTbz2PackageAvailable: called -> "+atom) # check if the package have been already merged atomName = atom.split("/")[len(atom.split("/"))-1] @@ -571,12 +571,12 @@ def isTbz2PackageAvailable(atom, verbose = False): tbz2Available = uploadPath if (verbose): print_info("found here: "+str(tbz2Available)) - portageLog.log(ETP_LOG_VERBOSE,"isTbz2PackageAvailable: result -> "+str(tbz2Available)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"isTbz2PackageAvailable: result -> "+str(tbz2Available)) return tbz2Available def checkAtom(atom): - portageLog.log(ETP_LOG_VERBOSE,"checkAtom: called -> "+str(atom)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"checkAtom: called -> "+str(atom)) bestAtom = getBestAtom(atom) if bestAtom == "!!conflicts": bestAtom = "" @@ -586,7 +586,7 @@ def checkAtom(atom): def getPackageDependencyList(atom): - portageLog.log(ETP_LOG_VERBOSE,"getPackageDependencyList: called -> "+str(atom)) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageDependencyList: called -> "+str(atom)) pkgSplittedDeps = [] tmp = portage.portdb.aux_get(atom, ["DEPEND"])[0].split() for i in tmp: @@ -603,7 +603,7 @@ def getPackageDependencyList(atom): # this file is contained in the .tbz2->.xpak file def getPackageRuntimeDependencies(NEEDED): - portageLog.log(ETP_LOG_VERBOSE,"getPackageRuntimeDependencies: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageRuntimeDependencies: called. ") if not os.path.isfile(NEEDED): return [],[],[] # all empty @@ -653,16 +653,16 @@ def getPackageRuntimeDependencies(NEEDED): return runtimeNeededPackages, runtimeNeededPackagesXT, neededLibraries def getUSEFlags(): - portageLog.log(ETP_LOG_VERBOSE,"getUSEFlags: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getUSEFlags: called. ") return getPortageEnv('USE') # you must provide a complete atom def getPackageIUSE(atom): - portageLog.log(ETP_LOG_VERBOSE,"getPackageIUSE: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageIUSE: called. ") return getPackageVar(atom,"IUSE") def getPackageVar(atom,var): - portageLog.log(ETP_LOG_VERBOSE,"getPackageVar: called -> "+atom+" | var: "+var) + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getPackageVar: called -> "+atom+" | var: "+var) if atom.startswith("="): atom = atom[1:] # can't check - return error @@ -671,7 +671,7 @@ def getPackageVar(atom,var): return portage.portdb.aux_get(atom,[var])[0] def synthetizeRoughDependencies(roughDependencies, useflags = None): - portageLog.log(ETP_LOG_VERBOSE,"synthetizeRoughDependencies: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"synthetizeRoughDependencies: called. ") if useflags is None: useflags = getUSEFlags() # returns dependencies, conflicts @@ -761,7 +761,7 @@ def getPortageAppDbPath(): # Collect installed packages def getInstalledPackages(): - portageLog.log(ETP_LOG_VERBOSE,"getInstalledPackages: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"getInstalledPackages: called. ") import os appDbDir = getPortageAppDbPath() dbDirs = os.listdir(appDbDir) @@ -777,7 +777,7 @@ def getInstalledPackages(): def packageSearch(keyword): - portageLog.log(ETP_LOG_VERBOSE,"packageSearch: called. ") + portageLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"packageSearch: called. ") SearchDirs = [] # search in etpConst['portagetreedir'] diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index a74b3a9fe..99b941c79 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -37,8 +37,12 @@ from portageTools import unpackTbz2, synthetizeRoughDependencies, getPackageRunt import logTools reagentLog = logTools.LogFile(level=etpConst['reagentloglevel'],filename = etpConst['reagentlogfile'], header = "[Reagent]") +# reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"testFunction: example. ") + def generator(package, enzymeRequestBump = False, dbconnection = None): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: called -> Package: "+str(package)+" | enzymeRequestBump: "+str(enzymeRequestBump)+" | dbconnection: "+str(dbconnection)) + # check if the package provided is valid validFile = False if os.path.isfile(package) and package.endswith(".tbz2"): @@ -66,12 +70,15 @@ def generator(package, enzymeRequestBump = False, dbconnection = None): dbconn.closeDB() if (updated) and (revision != 0): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" has been updated to revision: "+str(revision)) print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision))) return True, newFileName elif (updated) and (revision == 0): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" newly created.") print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created.")) return True, newFileName else: + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"generator: entry for "+str(packagename)+" kept intact, no updates needed.") print_info(green(" * ")+red("Package ")+bold(packagename)+red(" does not need to be updated. Current revision: ")+bold(str(revision))) return False, newFileName @@ -79,6 +86,8 @@ def generator(package, enzymeRequestBump = False, dbconnection = None): # This tool is used by Entropy after enzyme, it simply parses the content of etpConst['packagesstoredir'] def enzyme(options): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"enzyme: called -> options: "+str(options)) + enzymeRequestBump = False #_atoms = [] for i in options: @@ -126,6 +135,8 @@ def enzyme(options): # This function extracts all the info from a .tbz2 file and returns them def extractPkgData(package): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"extractPkgData: called -> package: "+str(package)) + # Clean the variables for i in etpData: etpData[i] = u"" @@ -436,6 +447,8 @@ def extractPkgData(package): def smartapps(options): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"smartapps: called -> options: "+str(options)) + if (len(options) == 0): print_error(yellow(" * ")+red("No valid tool specified.")) sys.exit(501) @@ -484,6 +497,8 @@ def smartapps(options): # NOTE: this section is highly portage dependent def smartgenerator(atom): + reagentLog.log(ETP_LOG_INFO,ETP_LOG_VERBOSE,"smartgenerator: called -> package: "+str(atom)) + dbconn = databaseTools.etpDatabase(readOnly = True) # handle branch management: