diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index c96494f03..a18197554 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -24,12 +24,11 @@ import time import shutil from entropyConstants import * from outputTools import * -from entropy import FtpInterface, EquoInterface, rssFeed, LogFile +from entropy import FtpInterface, EquoInterface, rssFeed import exceptionTools Entropy = EquoInterface(noclientdb = 2) # Logging initialization -activatorLog = LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]") def sync(options, justTidy = False): @@ -42,7 +41,6 @@ def sync(options, justTidy = False): myopts.append(i) options = myopts - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: called with justTidy -> "+str(justTidy)) print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ...")) if (not justTidy): @@ -105,7 +103,6 @@ def sync(options, justTidy = False): # not expired. if (not removeList): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: no packages to remove from the lirrors.") print_info(green(" * ")+red("No packages to remove from the mirrors.")) print_info(green(" * ")+red("Syncronization across mirrors completed.")) continue @@ -121,15 +118,12 @@ def sync(options, justTidy = False): sys.exit(0) # remove them! - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: starting to remove packages from mirrors.") for uri in etpConst['activatoruploaduris']: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: connecting to mirror "+Entropy.entropyTools.extractFTPHostFromUri(uri)) print_info(green(" * ")+red("Connecting to: ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))) ftp = FtpInterface(uri, Entropy) ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch) for xfile in removeList: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (remote) file "+xfile) print_info(green(" * ")+red("Removing file: ")+bold(xfile), back = True) # remove remotely if (ftp.isFileAvailable(xfile)): @@ -147,7 +141,6 @@ def sync(options, justTidy = False): print_warning(brown(" * ")+red("ATTENTION: remote checksum file ")+bold(xfile)+red(" cannot be removed.")) # remove locally if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile) print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch)) os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile) try: @@ -156,7 +149,6 @@ def sync(options, justTidy = False): pass # checksum if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext']): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile+etpConst['packageshashfileext']) print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch)) os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext']) ftp.closeConnection() @@ -166,8 +158,6 @@ def sync(options, justTidy = False): def packages(options): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called with options -> "+str(options)) - # Options available for all the packages submodules myopts = options[1:] activatorRequestPackagesCheck = False @@ -187,8 +177,6 @@ def packages(options): totalSuccessfulUri = 0 mirrorsTainted = False - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called sync.") - for uri in etpConst['activatoruploaduris']: uriSuccessfulSync = 0 @@ -213,7 +201,6 @@ def packages(options): if tbz2.endswith(".tbz2"): uploadCounter += 1 - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: upload directory stats -> files: "+str(uploadCounter)+" | upload packages list: "+str(toBeUploaded)) print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) localPackagesRepository = set() # parse etpConst['packagesbindir'] @@ -225,7 +212,6 @@ def packages(options): if tbz2.endswith(".tbz2"): packageCounter += 1 - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: packages directory stats -> files: "+str(packageCounter)+" | download packages list (including md5): "+str(localPackagesRepository)) print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) @@ -270,7 +256,6 @@ def packages(options): downloadQueue = set() removalQueue = set() - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: starting packages calculation...") finePackages = set() # Fill uploadQueue and if something weird is found, add the packages to downloadQueue @@ -645,8 +630,6 @@ def packages(options): if (str(e) == "100"): sys.exit(0) - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.") - activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: cannot properly syncronize "+Entropy.entropyTools.extractFTPHostFromUri(uri)+". Trying to continue if possible.") # print warning cannot sync uri print_warning(brown(" * ")+red("ATTENTION: cannot properly syncronize ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Continuing if possible...")) @@ -654,7 +637,6 @@ def packages(options): # decide what to do if (totalSuccessfulUri > 0) or (etpUi['pretend']): # we're safe - activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: at least one mirror has been synced properly. I'm fine.") print_info(green(" * ")+red("At least one mirror has been synced properly. I'm fine.")) continue else: @@ -664,7 +646,6 @@ def packages(options): else: # no mirrors were synced properly # show error and return, do not move files from the upload dir - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.") print_error(brown(" * ")+red("ERROR: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.")) return False @@ -673,7 +654,6 @@ def packages(options): if (totalSuccessfulUri > 0): if etpUi['pretend']: return False - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"packages: all done. Now it's time to move packages to "+etpConst['packagesbindir']) pkgbranches = os.listdir(etpConst['packagessuploaddir']) pkgbranches = [x for x in pkgbranches if os.path.isdir(etpConst['packagessuploaddir']+"/"+x)] for branch in pkgbranches: @@ -701,17 +681,14 @@ def packages(options): def database(options): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: called with -> "+str(options)) # lock tool if (options[0] == "lock"): print_info(green(" * ")+green("Starting to lock mirrors' databases...")) rc = lockDatabases(lock = True) if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !") print_info(green(" * ")+red("A problem occured on at least one mirror !")) else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") print_info(green(" * ")+green("Databases lock complete")) # unlock tool @@ -719,10 +696,8 @@ def database(options): print_info(green(" * ")+green("Starting to unlock mirrors' databases...")) rc = lockDatabases(lock = False) if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !") print_info(green(" * ")+green("A problem occured on at least one mirror !")) else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") print_info(green(" * ")+green("Databases unlock complete")) # download lock tool @@ -730,10 +705,8 @@ def database(options): print_info(green(" * ")+green("Starting to lock download mirrors' databases...")) rc = downloadLockDatabases(lock = True) if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !") print_info(green(" * ")+green("A problem occured on at least one mirror !")) else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.") print_info(green(" * ")+green("Download mirrors lock complete")) # download unlock tool @@ -741,10 +714,8 @@ def database(options): print_info(green(" * ")+green("Starting to unlock download mirrors' databases...")) rc = downloadLockDatabases(lock = False) if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !") print_info(green(" * ")+green("A problem occured on at least one mirror...")) else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.") print_info(green(" * ")+green("Download mirrors unlock complete")) # lock status tool @@ -765,7 +736,6 @@ def database(options): # database sync tool elif (options[0] == "sync"): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.") print_info(green(" * ")+red("Checking database status ..."), back = True) @@ -813,15 +783,12 @@ def database(options): os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) else: print - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...") raise exceptionTools.OnlineMirrorError("OnlineMirrorError: At the moment, mirrors are locked, someone is working on their databases, try again later...") else: if (dbLockFile): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) raise exceptionTools.OnlineMirrorError("OnlineMirrorError: Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...") print_info(green(" * ")+red("Mirrors are not locked. Fetching data...")) syncRemoteDatabases() diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index e8c825127..fe57dbc86 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -3072,16 +3072,10 @@ class etpDatabase: columns.append(row[1]) return columns - def tablesChecksum(self): - # NOTE: if you will add dependstable to the validation - # please have a look at EquoInterface.__install_package_into_database world calculation cache stuff - self.cursor.execute('select * from baseinfo') - c_hash = 0 - for row in self.cursor: - c_hash += hash(str(row)) - self.cursor.execute('select * from dependenciesreference') - for row in self.cursor: - c_hash += hash(str(row)) + def database_checksum(self): + # primary keys are now autoincrement + self.cursor.execute('select idpackage from baseinfo') + c_hash = hash(tuple(self.cursor.fetchall())) return str(c_hash) @@ -3217,6 +3211,7 @@ class etpDatabase: def createNeededIndex(self): if (self.dbname != etpConst['serverdbid']) and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library )') + self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded )') self.commitChanges() def createUseflagsIndex(self): diff --git a/libraries/entropy.py b/libraries/entropy.py index f64c4633b..3d1f6dceb 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -25,7 +25,7 @@ import commands import urllib2 import time from entropyConstants import * -from outputTools import * +from outputTools import TextInterface, print_info, print_warning, print_error, red, brown, blue, green, darkgreen, darkred, bold import exceptionTools class matchContainer: @@ -50,9 +50,7 @@ class EquoInterface(TextInterface): def __init__(self, indexing = True, noclientdb = 0, xcache = True, user_xcache = False, repo_validation = True): - # Logging initialization - self.equoLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[Equo]") - + self.clientLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[client]") import dumpTools self.dumpTools = dumpTools import databaseTools @@ -1323,7 +1321,8 @@ class EquoInterface(TextInterface): if self.xcache: c_data = list(dependencies) c_data.sort() - c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps)) + client_checksum = self.clientDbconn.database_checksum() + c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps))+client_checksum c_hash = str(hash(c_hash)) del c_data cached = self.dumpTools.loadobj(etpCache['filter_satisfied_deps']+c_hash) @@ -1610,7 +1609,8 @@ class EquoInterface(TextInterface): if self.xcache: c_data = list(matched_atoms) c_data.sort() - c_hash = str(hash(tuple(c_data)))+str(hash(empty_deps))+str(hash(deep_deps)) + client_checksum = self.clientDbconn.database_checksum() + c_hash = str(hash(tuple(c_data)))+str(hash(empty_deps))+str(hash(deep_deps))+client_checksum c_hash = str(hash(c_hash)) del c_data cached = self.dumpTools.loadobj(etpCache['dep_tree']+c_hash) @@ -1805,7 +1805,7 @@ class EquoInterface(TextInterface): dbconn = self.openRepositoryDatabase(repo) except exceptionTools.RepositoryError: continue # repo not available - sum_hashes += dbconn.tablesChecksum() + sum_hashes += dbconn.database_checksum() return sum_hashes def get_available_packages_chash(self, branch): @@ -2114,6 +2114,9 @@ class EquoInterface(TextInterface): if install and removal: myremmatch = {} for x in removal: + # XXX check if stupid users removed idpackage while this whole instance is running + if not self.clientDbconn.isIDPackageAvailable(x): + continue myremmatch.update({(self.entropyTools.dep_getkey(self.clientDbconn.retrieveAtom(x)),self.clientDbconn.retrieveSlot(x)): x}) for packageInfo in install: dbconn = self.openRepositoryDatabase(packageInfo[1]) @@ -2411,7 +2414,7 @@ class EquoInterface(TextInterface): path = shiftpath+path try: exist = os.lstat(path) - except OSError, e: + except OSError: continue # skip file arcname = path[len(shiftpath):] # remove shiftpath if arcname.startswith("/"): @@ -2455,8 +2458,7 @@ class EquoInterface(TextInterface): tbz2.recompose(dbdir) if edb: - # appending entropy metadata - dbpath = self.inject_entropy_database_into_package(dirpath, pkgdata) + self.inject_entropy_database_into_package(dirpath, pkgdata) if os.path.isfile(dirpath): return dirpath @@ -2510,7 +2512,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package name/version..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) tbz2File = package @@ -2546,7 +2548,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package md5..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # .tbz2 md5 @@ -2557,7 +2559,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package mtime..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # .tbz2 md5 @@ -2568,7 +2570,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package size..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # .tbz2 byte size @@ -2579,7 +2581,7 @@ class EquoInterface(TextInterface): red(info_package+"Unpacking package data..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # unpack file @@ -2595,7 +2597,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package CHOST..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill chost @@ -2608,7 +2610,7 @@ class EquoInterface(TextInterface): red(info_package+"Setting package branch..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) data['branch'] = etpBranch @@ -2618,7 +2620,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package description..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill description @@ -2635,7 +2637,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package homepage..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill homepage @@ -2652,7 +2654,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package slot information..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # fill slot, if it is @@ -2669,7 +2671,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package injection information..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # fill slot, if it is @@ -2683,7 +2685,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package eclasses information..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # fill eclasses list @@ -2700,7 +2702,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package needed libraries information..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # fill needed list @@ -2727,7 +2729,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package content..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) data['content'] = {} @@ -2833,7 +2835,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package category..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill category @@ -2846,7 +2848,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package download URL..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill download relative URI @@ -2866,7 +2868,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package counter..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill counter @@ -2883,7 +2885,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package external trigger availability..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']): @@ -2896,7 +2898,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package CFLAGS..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill CFLAGS @@ -2913,7 +2915,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package CXXFLAGS..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # Fill CXXFLAGS @@ -2930,7 +2932,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting source package supported ARCHs..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # fill KEYWORDS @@ -2953,7 +2955,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package dependencies..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) @@ -3007,7 +3009,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package metadata information..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) portage_metadata = Spm.calculate_dependencies(iuse, use, lics, depend, rdepend, pdepend, provide, sources) @@ -3052,7 +3054,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting package mirrors list..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # manage data['sources'] to create data['mirrorlinks'] @@ -3070,7 +3072,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting System Packages List..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # write only if it's a systempackage @@ -3089,7 +3091,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting CONFIG_PROTECT/CONFIG_PROTECT_MASK List..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # write only if it's a systempackage @@ -3128,7 +3130,7 @@ class EquoInterface(TextInterface): red(etpConst['logdir']+"/elog")+" not set, have you configured make.conf properly?", importance = 1, type = "warning", - header = yellow(" * ") + header = brown(" * ") ) if not silent: @@ -3136,7 +3138,7 @@ class EquoInterface(TextInterface): red(info_package+"Getting Entropy API version..."), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) # write API info @@ -3155,7 +3157,7 @@ class EquoInterface(TextInterface): red(info_package+"Done"), importance = 0, type = "info", - header = yellow(" * "), + header = brown(" * "), back = True ) return data @@ -3911,8 +3913,7 @@ class PackageInterface: try: exist = os.lstat(path) - except OSError, e: - #print e + except OSError: continue # skip file ftype = package_content[encoded_path] if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype @@ -5490,7 +5491,7 @@ class RepoInterface: header = darkred(" @@ ") ) self.syncErrors = True - self._resources_run_remove_lock() + self.Entropy._resources_run_remove_lock() return 128 rc = False @@ -5552,7 +5553,7 @@ class FtpInterface: # this must be run before calling the other functions def __init__(self, ftpuri, EntropyInterface): - if not isinstance(EntropyInterface, (EquoInterface, TextInterface)): + if not isinstance(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)): raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based instance is needed") self.Entropy = EntropyInterface @@ -5745,7 +5746,7 @@ class FtpInterface: myUploadPercentage = str(myUploadPercentage)+"%" # create text - currentText = brown(" <-> Upload status: ")+green(str(myUploadSize))+"/"+red(str(self.myFileSize))+" kB "+yellow("[")+str(myUploadPercentage)+yellow("]") + currentText = brown(" <-> Upload status: ")+green(str(myUploadSize))+"/"+red(str(self.myFileSize))+" kB "+brown("[")+str(myUploadPercentage)+brown("]") # print ! print_info(currentText, back = True) # XXX too slow, reimplement self.updateProgress and do whatever you want @@ -5871,9 +5872,6 @@ class FtpInterface: self.ftpconn.quit() -''' - Entropy FTP/HTTP download interface -''' class urlFetcher: def __init__(self, url, pathToSave, checksum = True, showSpeed = True, resume = True): @@ -6043,9 +6041,10 @@ class urlFetcher: curbarsize = 1 if self.average > self.oldaverage+self.updatestep: averagesize = (self.average*barsize)/100 - for y in range(averagesize): + while averagesize > 0: curbarsize += 1 bartext += "=" + averagesize -= 1 bartext += ">" diffbarsize = barsize-curbarsize for y in range(diffbarsize): @@ -6058,7 +6057,6 @@ class urlFetcher: if len(average) < 2: average = " "+average currentText += " <-> "+average+"% "+bartext - # print ! print_info(currentText,back = True) @@ -6158,7 +6156,7 @@ class rssFeed: self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data.strip() self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data.strip() - def addItem(self, title, link = '', description = '', light = False): + def addItem(self, title, link = '', description = ''): self.itemscounter += 1 self.items[self.itemscounter] = {} self.items[self.itemscounter]['title'] = title @@ -6294,7 +6292,7 @@ class TriggerInterface: raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed") self.Entropy = EquoInstance - self.equoLog = self.Entropy.equoLog + self.clientLog = self.Entropy.equoLog self.validPhases = ("preinstall","postinstall","preremove","postremove") self.pkgdata = pkgdata self.prepared = False @@ -6789,13 +6787,13 @@ class TriggerInterface: item = etpConst['systemroot']+item if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item): # running? - running = os.path.isfile(etpConst['systemroot']+self.INITSERVICES_DIR+'/started/'+os.path.basename(item)) + #running = os.path.isfile(etpConst['systemroot']+self.INITSERVICES_DIR+'/started/'+os.path.basename(item)) if not etpConst['systemroot']: myroot = "/" else: myroot = etpConst['systemroot']+"/" scheduled = not os.system('ROOT="'+myroot+'" rc-update show | grep '+os.path.basename(item)+'&> /dev/null') - self.trigger_initdeactivate(item, running, scheduled) + self.trigger_initdeactivate(item, scheduled) def trigger_initinform(self): for item in self.pkgdata['content']: @@ -7604,7 +7602,7 @@ class TriggerInterface: @description: shuts down selected init script, and remove from runlevel @output: returns int() as exit status ''' - def trigger_initdeactivate(self, item, running, scheduled): + def trigger_initdeactivate(self, item, scheduled): if not etpConst['systemroot']: myroot = "/" ''' @@ -7685,7 +7683,7 @@ timeout=10 kernel, initramfs = (unicode(kernel,'raw_unicode_escape'),unicode(initramfs,'raw_unicode_escape')) except TypeError: pass - kernelname = os.path.basename(kernel) + #kernelname = os.path.basename(kernel) new_conf = [] skip = False for line in content: @@ -10768,10 +10766,13 @@ class ServerInterface(TextInterface): if etpConst['uid'] != 0: raise exceptionTools.PermissionDenied("PermissionDenied: Entropy ServerInterface must be run as root") + self.serverLog = LogFile(level = etpConst['entropyloglevel'],filename = etpConst['entropylogfile'], header = "[server]") + # settings etpSys['serverside'] = True self.indexing = False self.xcache = False + self.FtpInterface = FtpInterface self.serverDbCache = {} self.settings_to_backup = [] self.do_save_repository = save_repository @@ -10796,6 +10797,7 @@ class ServerInterface(TextInterface): self.setup_entropy_settings() self.backup_entropy_settings() self.ClientService = EquoInterface(indexing = self.indexing, xcache = self.xcache, repo_validation = False) + self.ClientService.FtpInterface = self.FtpInterface self.databaseTools = self.ClientService.databaseTools self.entropyTools = self.ClientService.entropyTools self.SpmService = self.ClientService.Spm() @@ -11169,6 +11171,12 @@ class ServerInterface(TextInterface): return self.SpmService.quickpkg(atom,storedir) + def remove_package(self, idpackage): + dbconn = self.openServerDatabase(read_only = False, no_upload = True) + dbconn.removePackage(idpackage) + self.close_server_database(dbconn) + + def bump_database(self): dbconn = self.openServerDatabase(read_only = False, no_upload = True) dbconn.taintDatabase() @@ -11330,7 +11338,7 @@ class ServerInterface(TextInterface): f.flush() f.close() - rc = Entropy.askQuestion("Would you like to sync packages first (important if you don't have them synced) ?") + rc = self.askQuestion("Would you like to sync packages first (important if you don't have them synced) ?") if rc == "Yes": activatorTools.packages(["sync","--ask"]) @@ -11819,4 +11827,131 @@ class ServerInterface(TextInterface): header = brown(" # ") ) - return fine, failed, downloaded_fine, downloaded_errors \ No newline at end of file + self.close_server_database(dbconn) + return fine, failed, downloaded_fine, downloaded_errors + + + def switch_packages_branch(self, idpackages, to_branch): + + self.updateProgress( + red("Switching selected packages..."), + importance = 1, + type = "info", + header = darkgreen(" * ") + ) + dbconn = self.openServerDatabase(read_only = False, no_upload = True) + + already_switched = set() + not_found = set() + switched = set() + ignored = set() + no_checksum = set() + + for idpackage in idpackages: + + cur_branch = dbconn.retrieveBranch(idpackage) + atom = dbconn.retrieveAtom(idpackage) + if cur_branch == to_branch: + already_switched.add(idpackage) + self.updateProgress( + red("Ignoring %s, already in branch %s" % (bold(atom),cur_branch,)), + importance = 0, + type = "info", + header = darkgreen(" * ") + ) + ignored.add(idpackage) + continue + old_filename = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) + # check if file exists + frompath = os.path.join(etpConst['packagesserverbindir'],cur_branch+"/"+old_filename) + if not os.path.isfile(frompath): + self.updateProgress( + red("[%s=>%s] %s, cannot switch, package not found!" % (cur_branch,to_branch,atom,)), + importance = 0, + type = "warning", + header = darkred(" !!! ") + ) + not_found.add(idpackage) + continue + + self.updateProgress( + red("[%s=>%s] %s, configuring package information..." % (cur_branch,to_branch,atom,)), + importance = 0, + type = "info", + header = darkgreen(" * "), + back = True + ) + dbconn.switchBranch(idpackage,to_branch) + dbconn.commitChanges() + + # LOCAL + self.updateProgress( + red("[%s -> %s] %s, moving file locally..." % (cur_branch,to_branch,atom,)), + importance = 0, + type = "info", + header = darkgreen(" * "), + back = True + ) + new_filename = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) + topath = os.path.join(etpConst['packagesserverbindir'],to_branch) + if not os.path.isdir(topath): + os.makedirs(topath) + + topath = os.path.join(topath,new_filename) + shutil.move(frompath,topath) + if os.path.isfile(frompath+etpConst['packageshashfileext']): + shutil.move(frompath+etpConst['packageshashfileext'],topath+etpConst['packageshashfileext']) + else: + self.updateProgress( + red("[%s=>%s] %s, cannot find checksum to migrate!" % (cur_branch,to_branch,atom,)), + importance = 0, + type = "warning", + header = darkred(" !!! ") + ) + no_checksum.add(idpackage) + + # REMOTE + self.updateProgress( + red("[%s=>%s] %s, moving file remotely..." % (cur_branch,to_branch,atom,)), + importance = 0, + type = "info", + header = darkgreen(" * "), + back = True + ) + + for uri in etpConst['activatoruploaduris']: + + crippled_uri = self.entropyTools.extractFTPHostFromUri(uri) + self.updateProgress( + red("[%s=>%s] %s, moving file remotely on: %s" % (cur_branch,to_branch,atom,crippled_uri,)), + importance = 0, + type = "info", + header = darkgreen(" * "), + back = True + ) + + ftp = self.FtpInterface(uri, self) + ftp.setCWD(etpConst['binaryurirelativepath']) + # create directory if it doesn't exist + if not ftp.isFileAvailable(to_branch): + ftp.mkdir(to_branch) + + fromuri = os.path.join(cur_branch,old_filename) + touri = os.path.join(to_branch,new_filename) + ftp.renameFile(fromuri,touri) + ftp.renameFile(fromuri+etpConst['packageshashfileext'],touri+etpConst['packageshashfileext']) + ftp.closeConnection() + + switched.add(idpackage) + + dbconn.commitChanges() + self.close_server_database(dbconn) + self.updateProgress( + red("[%s=>%s] migration loop completed." % (cur_branch,to_branch,)), + importance = 1, + type = "info", + header = darkgreen(" * ") + ) + + return switched, already_switched, ignored, not_found, no_checksum + diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index efba449f1..8b299b83f 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -581,8 +581,6 @@ def const_defaultSettings(rootdir): 'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools 'proxy': {}, # proxy configuration information, used system wide - 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) - 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) 'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info) 'equologlevel': 1, # # Equo log level (default: 1 - see equo.conf for more info) 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their stuff @@ -1209,15 +1207,6 @@ def const_readActivatorSettings(): format = line.split("database-format|")[1] if format in etpConst['etpdatabasesupportedcformats']: etpConst['etpdatabasefileformat'] = format - elif line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except ValueError: - pass - - if (loglevel > -1) and (loglevel < 3): - etpConst['activatorloglevel'] = loglevel def const_readReagentSettings(): @@ -1226,17 +1215,8 @@ def const_readReagentSettings(): reagentconf = f.readlines() f.close() for line in reagentconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except ValueError: - pass - if (loglevel > -1) and (loglevel < 3): - etpConst['reagentloglevel'] = loglevel - - elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): + if line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): feed = line.split("rss-feed|")[1] if feed in ("enable","enabled","true","1"): etpConst['rss-feed'] = True diff --git a/server/server_reagent.py b/server/server_reagent.py index 1e69da6df..0a66c1023 100644 --- a/server/server_reagent.py +++ b/server/server_reagent.py @@ -224,7 +224,7 @@ def database(options): _options.append(opt) options = _options - if len(options) == 0: + if not options: print_error(brown(" * ")+red("Not enough parameters")) return 1 @@ -240,7 +240,6 @@ def database(options): dbpath = None if myopts: dbpath = myopts[0] - print_info(darkgreen(" * ")+red("Creating empty database to: ")+dbpath) if os.path.isfile(dbpath): print_error(darkgreen(" * ")+red("Cannot overwrite already existing file: ")+dbpath) @@ -250,170 +249,101 @@ def database(options): elif (options[0] == "switchbranch"): - if (len(options) < 2): + if (len(options) < 3): print_error(brown(" * ")+red("Not enough parameters")) - return 6 + return 1 switchbranch = options[1] - print_info(green(" * ")+red("Collecting packages that would be marked '"+switchbranch+"' ..."), back = True) + print_info(darkgreen(" * ")+red("Switching branch, be sure to have your packages in sync.")) + print_info(darkgreen(" * ")+red("Collecting packages that would be marked %s..." % (switchbranch,) ), back = True) + dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True) + pkglist = set() myatoms = options[2:] - if not myatoms: - print_error(brown(" * ")+red("Not enough parameters")) - return 7 - - dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True) - # is world? - if myatoms[0] == "world": - pkglist = dbconn.listAllIdpackages() + if "world" in myatoms: + pkglist |= dbconn.listAllIdpackages() else: - pkglist = set() for atom in myatoms: - # validate atom match = dbconn.atomMatch(atom) if match == -1: print_warning(brown(" * ")+red("Cannot match: ")+bold(atom)) else: pkglist.add(match[0]) - # check if atoms were found if not pkglist: - print print_error(brown(" * ")+red("No packages found.")) - return 8 + return 3 - # show what would be done - print_info(green(" * ")+red("These are the packages that would be marked '"+switchbranch+"':")) + print_info(darkgreen(" * ")+red("These are the packages that would be marked %s:" % (switchbranch,))) + for idpackage in pkglist: + atom = dbconn.retrieveAtom(idpackage) + print_info(red(" # ")+bold(atom)) - for pkg in pkglist: - atom = dbconn.retrieveAtom(pkg) - print_info(red(" (*) ")+bold(atom)) - - rc = Entropy.askQuestion(" Would you like to continue ?") + rc = Entropy.askQuestion("Would you like to continue ?") if rc == "No": - return 9 + return 4 - # sync packages - import activatorTools - ask = etpUi['ask'] - etpUi['ask'] = True - activatorTools.packages(["sync"]) - etpUi['ask'] = ask - - print_info(green(" * ")+red("Switching selected packages ...")) - - for pkg in pkglist: - atom = dbconn.retrieveAtom(pkg) - currentbranch = dbconn.retrieveBranch(pkg) - currentdownload = dbconn.retrieveDownloadURL(pkg) - - if currentbranch == switchbranch: - print_warning(green(" * ")+red("Ignoring ")+bold(atom)+red(" since it is already in the chosen branch")) - continue - - print_info(green(" * ")+darkred(atom+": ")+red("Configuring package information..."), back = True) - # change branch and download URL - dbconn.switchBranch(pkg,switchbranch) - - # rename locally - filename = os.path.basename(dbconn.retrieveDownloadURL(pkg)) - topath = etpConst['packagesbindir']+"/"+switchbranch - if not os.path.isdir(topath): - os.makedirs(topath) - print_info(green(" * ")+darkred(atom+": ")+red("Moving file locally..."), back = True) - #print etpConst['entropyworkdir']+"/"+currentdownload+" --> "+topath+"/"+newdownload - os.rename(etpConst['entropyworkdir']+"/"+currentdownload,topath+"/"+filename) - # md5 - os.rename(etpConst['entropyworkdir']+"/"+currentdownload+etpConst['packageshashfileext'],topath+"/"+filename+etpConst['packageshashfileext']) - - # XXX: we can barely ignore branch info injected into .tbz2 since they'll be ignored too - - # rename remotely - print_info(green(" * ")+darkred(atom+": ")+red("Moving file remotely..."), back = True) - # change filename remotely - for uri in etpConst['activatoruploaduris']: - - print_info(green(" * ")+darkred(atom+": ")+red("Moving file remotely on: ")+Entropy.entropyTools.extractFTPHostFromUri(uri), back = True) - - ftp = FtpInterface(uri, Entropy) - ftp.setCWD(etpConst['binaryurirelativepath']) - # create directory if it doesn't exist - if (not ftp.isFileAvailable(switchbranch)): - ftp.mkdir(switchbranch) - # rename tbz2 - ftp.renameFile(currentbranch+"/"+filename,switchbranch+"/"+filename) - # rename md5 - ftp.renameFile(currentbranch+"/"+filename+etpConst['packageshashfileext'],switchbranch+"/"+filename+etpConst['packageshashfileext']) - ftp.closeConnection() - - dbconn.closeDB() - print_info(green(" * ")+red("All the selected packages have been marked as requested. Remember to run activator.")) + switched, already_switched, ignored, not_found, no_checksum = Entropy.switch_packages_branch(pkglist, to_branch = switchbranch) + if not_found or no_checksum: + return 1 return 0 elif (options[0] == "remove"): - print_info(green(" * ")+red("Scanning packages that would be removed ..."), back = True) - - myopts = options[1:] - _myopts = [] + print_info(darkgreen(" * ")+red("Matching packages to remove..."), back = True) + myopts = [] branch = None - for opt in myopts: + for opt in options[1:]: if (opt.startswith("--branch=")) and (len(opt.split("=")) == 2): branch = opt.split("=")[1] else: - _myopts.append(opt) - myopts = _myopts + myopts.append(opt) - if len(myopts) == 0: + if not myopts: print_error(brown(" * ")+red("Not enough parameters")) - return 10 + return 1 + dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True) pkglist = set() - dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True) - for atom in myopts: - if (branch): - pkg = dbconn.atomMatch(atom, matchBranches = (branch,)) + if branch: + pkg = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True) else: - pkg = dbconn.atomMatch(atom) - if pkg[0] != -1: - pkglist.add(pkg[0]) + pkg = dbconn.atomMatch(atom, multiMatch = True) + if pkg[1] == 0: + for idpackage in pkg[0]: + pkglist.add(idpackage) - # check if atoms were found if not pkglist: - print - dbconn.closeDB() print_error(brown(" * ")+red("No packages found.")) - return 11 + return 2 - print_info(green(" * ")+red("These are the packages that would be removed from the database:")) + print_info(darkgreen(" * ")+red("These are the packages that would be removed from the database:")) + for idpackage in pkglist: + pkgatom = dbconn.retrieveAtom(idpackage) + branch = dbconn.retrieveBranch(idpackage) + print_info(red(" # ")+blue("[")+red(branch)+blue("] ")+bold(pkgatom)) - for pkg in pkglist: - pkgatom = dbconn.retrieveAtom(pkg) - branch = dbconn.retrieveBranch(pkg) - print_info(red("\t (*) ")+bold(pkgatom)+blue(" [")+red(branch)+blue("]")) - # ask to continue - rc = Entropy.askQuestion(" Would you like to continue ?") + rc = Entropy.askQuestion("Would you like to continue ?") if rc == "No": return 0 - # now mark them as stable - print_info(green(" * ")+red("Removing selected packages ...")) + print_info(darkgreen(" * ")+red("Removing selected packages...")) + for idpackage in pkglist: + pkgatom = dbconn.retrieveAtom(idpackage) + print_info(darkgreen(" * ")+red("Removing package: ")+bold(pkgatom)+red("..."), back = True) + Entropy.remove_package(idpackage) + + Entropy.close_server_database(dbconn) + print_info(darkgreen(" * ")+red("Packages removed. To remove binary packages, run activator.")) - # open db - for pkg in pkglist: - pkgatom = dbconn.retrieveAtom(pkg) - print_info(green(" * ")+red("Removing package: ")+bold(pkgatom)+red(" ..."), back = True) - dbconn.removePackage(pkg) - print_info(green(" * ")+red("All the selected packages have been removed as requested. To remove online binary packages, just run Activator.")) - dbconn.closeDB() return 0 elif (options[0] == "multiremove"): - print_info(green(" * ")+red("Scanning packages that would be removed ..."), back = True) + print_info(darkgreen(" * ")+red("Searching injected packages to remove..."), back = True) branch = etpConst['branch'] atoms = [] @@ -423,58 +353,46 @@ def database(options): else: atoms.append(opt) - pkglist = set() dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True) - allidpackages = dbconn.listAllIdpackages() idpackages = set() if not atoms: - # choose all + allidpackages = dbconn.listAllIdpackages() for idpackage in allidpackages: if dbconn.isInjected(idpackage): idpackages.add(idpackage) else: for atom in atoms: - match = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True, packagesFilter = False) - if match[1] != 0: - print_warning(red("Attention, no match for: ")+bold(atom)) - else: + match = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True) + if match[1] == 0: for x in match[0]: if dbconn.isInjected(x): idpackages.add(x) - # check if atoms were found if not idpackages: - dbconn.closeDB() print_error(brown(" * ")+red("No packages found.")) - return 11 + return 1 - print_info(green(" * ")+blue("These are the packages that would be removed from the database:")) + print_info(darkgreen(" * ")+blue("These are the injected packages pulled in for removal:")) for idpackage in idpackages: pkgatom = dbconn.retrieveAtom(idpackage) branch = dbconn.retrieveBranch(idpackage) - print_info(darkred(" (*) ")+blue("[")+red(branch)+blue("] ")+brown(pkgatom)) + print_info(darkred(" # ")+blue("[")+red(branch)+blue("] ")+brown(pkgatom)) # ask to continue - rc = Entropy.askQuestion(" Would you like to continue ?") + rc = Entropy.askQuestion("Would you like to continue ?") if rc == "No": return 0 - dbconn.closeDB() - del dbconn - dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True) - print_info(green(" * ")+red("Removing selected packages ...")) - - # open db for idpackage in idpackages: pkgatom = dbconn.retrieveAtom(idpackage) - print_info(green(" * ")+red("Removing package: ")+bold(pkgatom)+red(" ...")) - dbconn.removePackage(idpackage) - print_info(green(" * ")+red("All the selected packages have been removed as requested.")) - dbconn.closeDB() - del dbconn + print_info(darkgreen(" * ")+red("Removing package: ")+bold(pkgatom)+red("..."), back = True) + Entropy.remove_package(idpackage) + + Entropy.close_server_database(dbconn) + print_info(darkgreen(" * ")+red("Packages removed. To remove binary packages, run activator.")) return 0 # used by reagent @@ -527,4 +445,3 @@ def spm(options): print ' '.join(["="+x for x in packages]) else: os.system(etpConst['spm']['exec']+" "+etpConst['spm']['ask_cmd']+" "+etpConst['spm']['verbose_cmd']+" "+" ".join(["="+x for x in packages])) -