sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- fixed a problem with activator and directory tree generation

Browse Source 
- organized all entropy caches in a single location

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@852 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2007-12-12 22:43:23 +00:00
parent cd799929a4
commit 100dafbcb2
5 changed files with 174 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO
+1
View File
@@ -2,6 +2,7 @@ TODO list:
- world: first fetch all, then upgrade
- multipackage: clean tool
- implement a sane exception infrastructure
- use cPickle for dumpTools
- mirrors: one more and in random order
- trigger: Regenerating cracklib dictionary
- autotriggers: for enewgroup, enewuser and friends
client/equoTools.py
+13 -8
View File
@@ -52,7 +52,7 @@ def loadCaches():
if isinstance(mycache, dict):
atomMatchCache = mycache.copy()
except:
atomMatchCache = {}
atomMatchCache.clear()
dumpTools.dumpobj(etpCache['atomMatch'],{})
# removal dependencies
@@ -61,7 +61,7 @@ def loadCaches():
if isinstance(mycache3, dict):
generateDependsTreeCache = mycache3.copy()
except:
generateDependsTreeCache = {}
generateDependsTreeCache.clear()
dumpTools.dumpobj(etpCache['generateDependsTree'],{})
@@ -143,7 +143,7 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (),
# handle repoResults
packageInformation = {}
# nothing found
if len(repoResults) == 0:
atomMatchCache[atom] = {}
@@ -165,7 +165,16 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (),
elif len(repoResults) > 1:
# we have to decide which version should be taken
# .tbz2 repos have always the precedence, so if we find them, we should second what user wants, installing his tbz2
tbz2repos = [x for x in repoResults if x.endswith(".tbz2")]
if tbz2repos:
del tbz2repos
newrepos = repoResults.copy()
for x in newrepos:
if not x.endswith(".tbz2"):
del repoResults[x]
# get package information for all the entries
for repo in repoResults:
@@ -338,7 +347,6 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (),
@input packageInfo: tuple composed by int(id) and str(repository name), if this one is int(0), the client database will be opened.
@output: ordered dependency list
'''
getDependenciesCache = {}
def getDependencies(packageInfo):
''' caching '''
@@ -373,8 +381,6 @@ def getDependencies(packageInfo):
@input dependencies: list of dependencies to check
@output: filtered list, aka the needed ones and the ones satisfied
'''
filterSatisfiedDependenciesCache = {}
filterSatisfiedDependenciesCmpResults = {} # 0: not installed, <0 a<b | >0 a>b
def filterSatisfiedDependencies(dependencies, deepdeps = False):
unsatisfiedDeps = set()
@@ -462,7 +468,6 @@ def filterSatisfiedDependencies(dependencies, deepdeps = False):
@input package: atomInfo (idpackage,reponame)
@output: dependency tree dictionary, plus status code
'''
generateDependencyTreeCache = {}
matchFilter = set()
def generateDependencyTree(atomInfo, emptydeps = False, deepdeps = False, usefilter = False):
libraries/activatorTools.py
+111 -96
View File
@@ -256,7 +256,8 @@ def packages(options):
except Exception, e:
if str(e).find("550") != -1:
pass
raise
else:
raise
ftp.setCWD(etpConst['binaryurirelativepath'])
if (not ftp.isFileAvailable(mybranch)):
@@ -720,118 +721,132 @@ def database(options):
# lock tool
if (options[0] == "lock"):
print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
rc = lockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !")
print_info(green(" * ")+red("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases lock complete"))
print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
rc = lockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !")
print_info(green(" * ")+red("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases lock complete"))
# unlock tool
elif (options[0] == "unlock"):
print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
rc = lockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases unlock complete"))
print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
rc = lockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases unlock complete"))
# download lock tool
elif (options[0] == "download-lock"):
print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
rc = downloadLockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.")
print_info(green(" * ")+green("Download mirrors lock complete"))
print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
rc = downloadLockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.")
print_info(green(" * ")+green("Download mirrors lock complete"))
# download unlock tool
elif (options[0] == "download-unlock"):
print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
rc = downloadLockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.")
print_info(green(" * ")+green("Download mirrors unlock complete"))
print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
rc = downloadLockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.")
print_info(green(" * ")+green("Download mirrors unlock complete"))
# lock status tool
elif (options[0] == "lock-status"):
print_info(brown(" * ")+green("Mirrors status table:"))
dbstatus = getMirrorsLock()
for db in dbstatus:
if (db[1]):
db[1] = red("Locked")
else:
db[1] = green("Unlocked")
if (db[2]):
db[2] = red("Locked")
else:
db[2] = green("Unlocked")
print_info(bold("\t"+extractFTPHostFromUri(db[0])+": ")+red("[")+brown("DATABASE: ")+db[1]+red("] [")+brown("DOWNLOAD: ")+db[2]+red("]"))
print_info(brown(" * ")+green("Mirrors status table:"))
dbstatus = getMirrorsLock()
for db in dbstatus:
if (db[1]):
db[1] = red("Locked")
else:
db[1] = green("Unlocked")
if (db[2]):
db[2] = red("Locked")
else:
db[2] = green("Unlocked")
print_info(bold("\t"+extractFTPHostFromUri(db[0])+": ")+red("[")+brown("DATABASE: ")+db[1]+red("] [")+brown("DOWNLOAD: ")+db[2]+red("]"))
# database sync tool
elif (options[0] == "sync"):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.")
print_info(green(" * ")+red("Checking database status ..."), back = True)
dbLockFile = False
# does the taint file exist?
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
dbLockFile = True
# are online mirrors locked?
mirrorsLocked = False
for uri in etpConst['activatoruploaduris']:
ftp = mirrorTools.handlerFTP(uri)
ftp.setCWD(etpConst['etpurirelativepath'])
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
mirrorsLocked = True
ftp.closeConnection()
break
if (mirrorsLocked):
# if the mirrors are locked, we need to check if we have
# the taint file in place. Because in this case, the one
# that tainted the db, was me.
if (dbLockFile):
print_info(green(" * ")+red("Updating mirrors with new information ..."))
# it's safe to sync
syncRemoteDatabases()
# remove the online lock file
lockDatabases(False)
# remove the taint file
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
else:
print
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...")
print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later..."))
exit(422)
else:
if (dbLockFile):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']))
exit(423)
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...")
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
syncRemoteDatabases()
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.")
print_info(green(" * ")+red("Checking database status ..."), back = True)
dbLockFile = False
# does the taint file exist?
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
dbLockFile = True
# are online mirrors locked?
mirrorsLocked = False
for uri in etpConst['activatoruploaduris']:
ftp = mirrorTools.handlerFTP(uri)
try:
ftp.setCWD(etpConst['etpurirelativepath'])
except:
bdir = ""
for mydir in etpConst['etpurirelativepath'].split("/"):
bdir += "/"+mydir
if (not ftp.isFileAvailable(bdir)):
try:
ftp.mkdir(bdir)
except Exception, e:
if str(e).find("550") != -1:
pass
else:
raise
ftp.setCWD(etpConst['etpurirelativepath'])
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
mirrorsLocked = True
ftp.closeConnection()
break
if (mirrorsLocked):
# if the mirrors are locked, we need to check if we have
# the taint file in place. Because in this case, the one
# that tainted the db, was me.
if (dbLockFile):
print_info(green(" * ")+red("Updating mirrors with new information ..."))
# it's safe to sync
syncRemoteDatabases()
# remove the online lock file
lockDatabases(False)
# remove the taint file
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
else:
print
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...")
print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later..."))
exit(422)
else:
if (dbLockFile):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']))
exit(423)
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...")
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
syncRemoteDatabases()
else:
print_error(red(" * ")+green("No valid tool specified."))
exit(400)
print_error(red(" * ")+green("No valid tool specified."))
exit(400)
########################################################
libraries/entropyConstants.py
+40 -10
View File
@@ -382,6 +382,46 @@ atomMatchCache = {}
atomClientMatchCache = {}
generateDependsTreeCache = {}
idpackageValidatorCache = {}
filterSatisfiedDependenciesCache = {}
filterSatisfiedDependenciesCmpResults = {}
getDependenciesCache = {}
generateDependencyTreeCache = {}
ververifyCache = {}
isjustnameCache = {}
catpkgsplitCache = {}
dep_striptagCache = {}
dep_getkeyCache = {}
dep_getcpvCache = {}
dep_getslotCache = {}
dep_gettagCache = {}
removePackageOperatorsCache = {}
compareVersionsCache = {}
getNewerVersionCache = {}
### Application disk cache
def const_resetCache():
for item in dbCacheStore:
dbCacheStore[item].clear()
atomMatchCache.clear()
atomClientMatchCache.clear()
generateDependsTreeCache.clear()
idpackageValidatorCache.clear()
filterSatisfiedDependenciesCache.clear()
filterSatisfiedDependenciesCmpResults.clear()
getDependenciesCache.clear()
generateDependencyTreeCache.clear()
ververifyCache.clear()
isjustnameCache.clear()
catpkgsplitCache.clear()
dep_striptagCache.clear()
dep_getkeyCache.clear()
dep_getcpvCache.clear()
dep_getslotCache.clear()
dep_gettagCache.clear()
removePackageOperatorsCache.clear()
compareVersionsCache.clear()
getNewerVersionCache.clear()
# Client packages/database repositories
etpRepositories = {}
@@ -422,16 +462,6 @@ dbCONTENTS = "CONTENTS"
dbCOUNTER = "COUNTER"
### Application disk cache
def const_resetCache():
for item in dbCacheStore:
dbCacheStore[item].clear()
atomMatchCache.clear()
atomClientMatchCache.clear()
generateDependsTreeCache.clear()
idpackageValidatorCache.clear()
# ===============================================================================================
# BEGINNING OF DYNAMIC SECTION
# ===============================================================================================
libraries/entropyTools.py
+9 -11
View File
@@ -407,7 +407,6 @@ def isjustpkgname(mypkg):
return 0
return 1
ververifyCache = {}
def ververify(myverx, silent=1):
cached = ververifyCache.get(myverx)
@@ -426,7 +425,7 @@ def ververify(myverx, silent=1):
ververifyCache[myverx] = 0
return 0
isjustnameCache = {}
def isjustname(mypkg):
"""
Checks to see if the depstring is only the package name (no version parts)
@@ -482,7 +481,7 @@ def isspecific(mypkg):
return 1
return 0
catpkgsplitCache = {}
def catpkgsplit(mydata,silent=1):
"""
Takes a Category/Package-Version-Rev and returns a list of each.
@@ -564,7 +563,7 @@ def pkgsplit(mypkg,silent=1):
return None
# FIXME: deprecated, use remove_tag - will be removed soonly
dep_striptagCache = {}
def dep_striptag(mydepx):
cached = dep_striptagCache.get(mydepx)
@@ -580,7 +579,7 @@ def dep_striptag(mydepx):
dep_striptagCache[mydepx] = mydep
return mydep
dep_getkeyCache = {}
def dep_getkey(mydepx):
"""
Return the category/package-name of a depstring.
@@ -615,7 +614,7 @@ def dep_getkey(mydepx):
dep_getkeyCache[mydepx] = mydep
return mydep
dep_getcpvCache = {}
def dep_getcpv(mydep):
"""
Return the category-package-version with any operators/slot specifications stripped off
@@ -652,7 +651,7 @@ def dep_getcpv(mydep):
dep_getcpvCache[mydep] = mydep
return mydep
dep_getslotCache = {}
def dep_getslot(dep):
"""
Retrieve the slot on a depend.
@@ -708,7 +707,7 @@ def remove_entropy_revision(mydep):
return mystring
return mydep
dep_gettagCache = {}
def dep_gettag(dep):
"""
Retrieve the slot on a depend.
@@ -732,7 +731,7 @@ def dep_gettag(dep):
dep_gettagCache[dep] = None
return None
removePackageOperatorsCache = {}
def removePackageOperators(atom):
cached = removePackageOperatorsCache.get(atom)
@@ -757,7 +756,6 @@ ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alph
suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
compareVersionsCache = {}
def compareVersions(ver1, ver2, silent=1):
cached = compareVersionsCache.get(tuple([ver1,ver2]))
@@ -889,7 +887,7 @@ def entropyCompareVersions(listA,listB):
@input versionlist: a list
@output: the ordered list
'''
getNewerVersionCache = {}
def getNewerVersion(InputVersionlist):
cached = getNewerVersionCache.get(tuple(InputVersionlist))