From 100dafbcb2ff83f5d03937482a4b5575fe5d5ef5 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Wed, 12 Dec 2007 22:43:23 +0000 Subject: [PATCH] - fixed a problem with activator and directory tree generation - organized all entropy caches in a single location git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@852 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + client/equoTools.py | 21 ++-- libraries/activatorTools.py | 207 ++++++++++++++++++---------------- libraries/entropyConstants.py | 50 ++++++-- libraries/entropyTools.py | 20 ++-- 5 files changed, 174 insertions(+), 125 deletions(-) diff --git a/TODO b/TODO index 40066abc9..bdff6d759 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,7 @@ TODO list: - world: first fetch all, then upgrade - multipackage: clean tool - implement a sane exception infrastructure + - use cPickle for dumpTools - mirrors: one more and in random order - trigger: Regenerating cracklib dictionary - autotriggers: for enewgroup, enewuser and friends diff --git a/client/equoTools.py b/client/equoTools.py index 521990a91..e75ceb2df 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -52,7 +52,7 @@ def loadCaches(): if isinstance(mycache, dict): atomMatchCache = mycache.copy() except: - atomMatchCache = {} + atomMatchCache.clear() dumpTools.dumpobj(etpCache['atomMatch'],{}) # removal dependencies @@ -61,7 +61,7 @@ def loadCaches(): if isinstance(mycache3, dict): generateDependsTreeCache = mycache3.copy() except: - generateDependsTreeCache = {} + generateDependsTreeCache.clear() dumpTools.dumpobj(etpCache['generateDependsTree'],{}) @@ -143,7 +143,7 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (), # handle repoResults packageInformation = {} - + # nothing found if len(repoResults) == 0: atomMatchCache[atom] = {} @@ -165,7 +165,16 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (), elif len(repoResults) > 1: # we have to decide which version should be taken - + + # .tbz2 repos have always the precedence, so if we find them, we should second what user wants, installing his tbz2 + tbz2repos = [x for x in repoResults if x.endswith(".tbz2")] + if tbz2repos: + del tbz2repos + newrepos = repoResults.copy() + for x in newrepos: + if not x.endswith(".tbz2"): + del repoResults[x] + # get package information for all the entries for repo in repoResults: @@ -338,7 +347,6 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (), @input packageInfo: tuple composed by int(id) and str(repository name), if this one is int(0), the client database will be opened. @output: ordered dependency list ''' -getDependenciesCache = {} def getDependencies(packageInfo): ''' caching ''' @@ -373,8 +381,6 @@ def getDependencies(packageInfo): @input dependencies: list of dependencies to check @output: filtered list, aka the needed ones and the ones satisfied ''' -filterSatisfiedDependenciesCache = {} -filterSatisfiedDependenciesCmpResults = {} # 0: not installed, <0 a0 a>b def filterSatisfiedDependencies(dependencies, deepdeps = False): unsatisfiedDeps = set() @@ -462,7 +468,6 @@ def filterSatisfiedDependencies(dependencies, deepdeps = False): @input package: atomInfo (idpackage,reponame) @output: dependency tree dictionary, plus status code ''' -generateDependencyTreeCache = {} matchFilter = set() def generateDependencyTree(atomInfo, emptydeps = False, deepdeps = False, usefilter = False): diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index f05d74923..5a923a322 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -256,7 +256,8 @@ def packages(options): except Exception, e: if str(e).find("550") != -1: pass - raise + else: + raise ftp.setCWD(etpConst['binaryurirelativepath']) if (not ftp.isFileAvailable(mybranch)): @@ -720,118 +721,132 @@ def database(options): # lock tool if (options[0] == "lock"): - print_info(green(" * ")+green("Starting to lock mirrors' databases...")) - rc = lockDatabases(lock = True) - if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !") - print_info(green(" * ")+red("A problem occured on at least one mirror !")) - else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") - print_info(green(" * ")+green("Databases lock complete")) + print_info(green(" * ")+green("Starting to lock mirrors' databases...")) + rc = lockDatabases(lock = True) + if (rc): + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !") + print_info(green(" * ")+red("A problem occured on at least one mirror !")) + else: + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") + print_info(green(" * ")+green("Databases lock complete")) # unlock tool elif (options[0] == "unlock"): - print_info(green(" * ")+green("Starting to unlock mirrors' databases...")) - rc = lockDatabases(lock = False) - if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !") - print_info(green(" * ")+green("A problem occured on at least one mirror !")) - else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") - print_info(green(" * ")+green("Databases unlock complete")) + print_info(green(" * ")+green("Starting to unlock mirrors' databases...")) + rc = lockDatabases(lock = False) + if (rc): + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !") + print_info(green(" * ")+green("A problem occured on at least one mirror !")) + else: + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.") + print_info(green(" * ")+green("Databases unlock complete")) # download lock tool elif (options[0] == "download-lock"): - print_info(green(" * ")+green("Starting to lock download mirrors' databases...")) - rc = downloadLockDatabases(lock = True) - if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !") - print_info(green(" * ")+green("A problem occured on at least one mirror !")) - else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.") - print_info(green(" * ")+green("Download mirrors lock complete")) + print_info(green(" * ")+green("Starting to lock download mirrors' databases...")) + rc = downloadLockDatabases(lock = True) + if (rc): + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !") + print_info(green(" * ")+green("A problem occured on at least one mirror !")) + else: + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.") + print_info(green(" * ")+green("Download mirrors lock complete")) # download unlock tool elif (options[0] == "download-unlock"): - print_info(green(" * ")+green("Starting to unlock download mirrors' databases...")) - rc = downloadLockDatabases(lock = False) - if (rc): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !") - print_info(green(" * ")+green("A problem occured on at least one mirror...")) - else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.") - print_info(green(" * ")+green("Download mirrors unlock complete")) + print_info(green(" * ")+green("Starting to unlock download mirrors' databases...")) + rc = downloadLockDatabases(lock = False) + if (rc): + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !") + print_info(green(" * ")+green("A problem occured on at least one mirror...")) + else: + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.") + print_info(green(" * ")+green("Download mirrors unlock complete")) # lock status tool elif (options[0] == "lock-status"): - print_info(brown(" * ")+green("Mirrors status table:")) - dbstatus = getMirrorsLock() - for db in dbstatus: - if (db[1]): - db[1] = red("Locked") - else: - db[1] = green("Unlocked") - if (db[2]): - db[2] = red("Locked") - else: - db[2] = green("Unlocked") - print_info(bold("\t"+extractFTPHostFromUri(db[0])+": ")+red("[")+brown("DATABASE: ")+db[1]+red("] [")+brown("DOWNLOAD: ")+db[2]+red("]")) + print_info(brown(" * ")+green("Mirrors status table:")) + dbstatus = getMirrorsLock() + for db in dbstatus: + if (db[1]): + db[1] = red("Locked") + else: + db[1] = green("Unlocked") + if (db[2]): + db[2] = red("Locked") + else: + db[2] = green("Unlocked") + print_info(bold("\t"+extractFTPHostFromUri(db[0])+": ")+red("[")+brown("DATABASE: ")+db[1]+red("] [")+brown("DOWNLOAD: ")+db[2]+red("]")) # database sync tool elif (options[0] == "sync"): - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.") - - print_info(green(" * ")+red("Checking database status ..."), back = True) - - dbLockFile = False - # does the taint file exist? - if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): - dbLockFile = True - - # are online mirrors locked? - mirrorsLocked = False - for uri in etpConst['activatoruploaduris']: - ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['etpurirelativepath']) - if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])): - mirrorsLocked = True - ftp.closeConnection() - break - - if (mirrorsLocked): - # if the mirrors are locked, we need to check if we have - # the taint file in place. Because in this case, the one - # that tainted the db, was me. - if (dbLockFile): - print_info(green(" * ")+red("Updating mirrors with new information ...")) - # it's safe to sync - syncRemoteDatabases() - # remove the online lock file - lockDatabases(False) - # remove the taint file - if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): - os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) - else: - print - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...") - print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later...")) - exit(422) - - else: - if (dbLockFile): - activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) - print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])) - exit(423) - else: - activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...") - print_info(green(" * ")+red("Mirrors are not locked. Fetching data...")) - - syncRemoteDatabases() - + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.") + + print_info(green(" * ")+red("Checking database status ..."), back = True) + + dbLockFile = False + # does the taint file exist? + if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): + dbLockFile = True + + # are online mirrors locked? + mirrorsLocked = False + for uri in etpConst['activatoruploaduris']: + ftp = mirrorTools.handlerFTP(uri) + try: + ftp.setCWD(etpConst['etpurirelativepath']) + except: + bdir = "" + for mydir in etpConst['etpurirelativepath'].split("/"): + bdir += "/"+mydir + if (not ftp.isFileAvailable(bdir)): + try: + ftp.mkdir(bdir) + except Exception, e: + if str(e).find("550") != -1: + pass + else: + raise + ftp.setCWD(etpConst['etpurirelativepath']) + if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])): + mirrorsLocked = True + ftp.closeConnection() + break + + if (mirrorsLocked): + # if the mirrors are locked, we need to check if we have + # the taint file in place. Because in this case, the one + # that tainted the db, was me. + if (dbLockFile): + print_info(green(" * ")+red("Updating mirrors with new information ...")) + # it's safe to sync + syncRemoteDatabases() + # remove the online lock file + lockDatabases(False) + # remove the taint file + if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): + os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) + else: + print + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...") + print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later...")) + exit(422) + + else: + if (dbLockFile): + activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) + print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])) + exit(423) + else: + activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...") + print_info(green(" * ")+red("Mirrors are not locked. Fetching data...")) + + syncRemoteDatabases() + else: - print_error(red(" * ")+green("No valid tool specified.")) - exit(400) + print_error(red(" * ")+green("No valid tool specified.")) + exit(400) ######################################################## diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index d39406045..ad0e4460a 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -382,6 +382,46 @@ atomMatchCache = {} atomClientMatchCache = {} generateDependsTreeCache = {} idpackageValidatorCache = {} +filterSatisfiedDependenciesCache = {} +filterSatisfiedDependenciesCmpResults = {} +getDependenciesCache = {} +generateDependencyTreeCache = {} +ververifyCache = {} +isjustnameCache = {} +catpkgsplitCache = {} +dep_striptagCache = {} +dep_getkeyCache = {} +dep_getcpvCache = {} +dep_getslotCache = {} +dep_gettagCache = {} +removePackageOperatorsCache = {} +compareVersionsCache = {} +getNewerVersionCache = {} + + +### Application disk cache +def const_resetCache(): + for item in dbCacheStore: + dbCacheStore[item].clear() + atomMatchCache.clear() + atomClientMatchCache.clear() + generateDependsTreeCache.clear() + idpackageValidatorCache.clear() + filterSatisfiedDependenciesCache.clear() + filterSatisfiedDependenciesCmpResults.clear() + getDependenciesCache.clear() + generateDependencyTreeCache.clear() + ververifyCache.clear() + isjustnameCache.clear() + catpkgsplitCache.clear() + dep_striptagCache.clear() + dep_getkeyCache.clear() + dep_getcpvCache.clear() + dep_getslotCache.clear() + dep_gettagCache.clear() + removePackageOperatorsCache.clear() + compareVersionsCache.clear() + getNewerVersionCache.clear() # Client packages/database repositories etpRepositories = {} @@ -422,16 +462,6 @@ dbCONTENTS = "CONTENTS" dbCOUNTER = "COUNTER" -### Application disk cache -def const_resetCache(): - for item in dbCacheStore: - dbCacheStore[item].clear() - atomMatchCache.clear() - atomClientMatchCache.clear() - generateDependsTreeCache.clear() - idpackageValidatorCache.clear() - - # =============================================================================================== # BEGINNING OF DYNAMIC SECTION # =============================================================================================== diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 41535455b..05c505b4f 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -407,7 +407,6 @@ def isjustpkgname(mypkg): return 0 return 1 -ververifyCache = {} def ververify(myverx, silent=1): cached = ververifyCache.get(myverx) @@ -426,7 +425,7 @@ def ververify(myverx, silent=1): ververifyCache[myverx] = 0 return 0 -isjustnameCache = {} + def isjustname(mypkg): """ Checks to see if the depstring is only the package name (no version parts) @@ -482,7 +481,7 @@ def isspecific(mypkg): return 1 return 0 -catpkgsplitCache = {} + def catpkgsplit(mydata,silent=1): """ Takes a Category/Package-Version-Rev and returns a list of each. @@ -564,7 +563,7 @@ def pkgsplit(mypkg,silent=1): return None # FIXME: deprecated, use remove_tag - will be removed soonly -dep_striptagCache = {} + def dep_striptag(mydepx): cached = dep_striptagCache.get(mydepx) @@ -580,7 +579,7 @@ def dep_striptag(mydepx): dep_striptagCache[mydepx] = mydep return mydep -dep_getkeyCache = {} + def dep_getkey(mydepx): """ Return the category/package-name of a depstring. @@ -615,7 +614,7 @@ def dep_getkey(mydepx): dep_getkeyCache[mydepx] = mydep return mydep -dep_getcpvCache = {} + def dep_getcpv(mydep): """ Return the category-package-version with any operators/slot specifications stripped off @@ -652,7 +651,7 @@ def dep_getcpv(mydep): dep_getcpvCache[mydep] = mydep return mydep -dep_getslotCache = {} + def dep_getslot(dep): """ Retrieve the slot on a depend. @@ -708,7 +707,7 @@ def remove_entropy_revision(mydep): return mystring return mydep -dep_gettagCache = {} + def dep_gettag(dep): """ Retrieve the slot on a depend. @@ -732,7 +731,7 @@ def dep_gettag(dep): dep_gettagCache[dep] = None return None -removePackageOperatorsCache = {} + def removePackageOperators(atom): cached = removePackageOperatorsCache.get(atom) @@ -757,7 +756,6 @@ ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alph suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1} endversion_keys = ["pre", "p", "alpha", "beta", "rc"] -compareVersionsCache = {} def compareVersions(ver1, ver2, silent=1): cached = compareVersionsCache.get(tuple([ver1,ver2])) @@ -889,7 +887,7 @@ def entropyCompareVersions(listA,listB): @input versionlist: a list @output: the ordered list ''' -getNewerVersionCache = {} + def getNewerVersion(InputVersionlist): cached = getNewerVersionCache.get(tuple(InputVersionlist))