sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

entropy.*: move atom matching package masking validation cache to

Browse Source 
SystemSettings
This commit is contained in:
Fabio Erculiani
2009-04-19 10:23:55 +02:00
parent e54c027534
commit 0a901a3de6
5 changed files with 70 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/APICHANGES
+1 -1
View File
@@ -110,7 +110,7 @@ From SVN 3112 there have been several API changes:
securityCache => __security_cache
QACache => __QA_cache
memoryDbInstances => _memory_db_instances
package_match_validator_cache => _package_match_validator_cache
package_match_validator_cache => SystemSettings[etpConst['system_settings_plugins_ids']['client_plugin']]['masking_validation']['cache']
spmCache => __spm_cache
- ServerInterface (now entropy.server.interfaces.Server) API changes:
libraries/entropy/client/interfaces/client.py
+6 -1
View File
@@ -76,6 +76,12 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin):
})
return parser_data
def masking_validation_parser(self, system_settings_instance):
data = {
'cache': {}, # package masking validation cache
}
return data
def repo_setup_parser(self, system_settings_instance):
# this makes sure that repository metadata is initialized
@@ -143,7 +149,6 @@ class Client(Singleton, TextInterface, LoadersMixin, CacheMixin, CalculatorsMixi
self.atomMatchCacheKey = etpCache['atomMatch']
self.dbapi2 = dbapi2 # export for third parties
self.FileUpdates = None
self._package_match_validator_cache = {}
self.validRepositories = []
self.UGC = None
# supporting external updateProgress stuff, you can point self.progress
libraries/entropy/client/interfaces/methods.py
+2 -2
View File
@@ -41,7 +41,7 @@ class RepositoryMixin:
def validate_repositories(self):
self.MirrorStatus.clear()
self.__repo_error_messages_cache.clear()
self._package_match_validator_cache.clear()
self.SystemSettings['package_masking']['cache'].clear()
# valid repositories
del self.validRepositories[:]
for repoid in self.SystemSettings['repositories']['order']:
@@ -1570,7 +1570,7 @@ class MatchMixin:
self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],))
self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],))
self._package_match_validator_cache.clear()
self.SystemSettings['package_masking']['cache'].clear()
return done
def unmask_match_by_atom(self, match, dry_run = False):
libraries/entropy/db.py
+53 -18
View File
@@ -354,6 +354,7 @@ class LocalRepository:
self.dbSearchCacheKey = etpCache['dbSearch']
self.dbname = dbname
self.lockRemote = lockRemote
self.client_settings_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin']
self.db_branch = self.SystemSettings['repositories']['branch']
if self.dbname == etpConst['clientdbid']:
self.db_branch = None
@@ -4516,7 +4517,11 @@ class LocalRepository:
if idpackage in user_package_mask_ids:
# sorry, masked
myr = self.SystemSettings['pkg_masking_reference']['user_package_mask']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = -1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _idpackageValidator_user_package_unmask(self, idpackage, reponame, live):
@@ -4532,7 +4537,11 @@ class LocalRepository:
self.SystemSettings[mykw] = user_package_unmask_ids
if idpackage in user_package_unmask_ids:
myr = self.SystemSettings['pkg_masking_reference']['user_package_unmask']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = idpackage, myr
except KeyError: # system settings client plugin not found
pass
return idpackage, myr
def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live):
@@ -4552,7 +4561,11 @@ class LocalRepository:
repos_mask[mask_repo_id] = repomask_ids
if idpackage in repomask_ids:
myr = self.SystemSettings['pkg_masking_reference']['repository_packages_db_mask']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = -1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _idpackageValidator_package_license_mask(self, idpackage, reponame, live):
@@ -4563,7 +4576,11 @@ class LocalRepository:
for mylicense in mylicenses:
if mylicense not in lic_mask: continue
myr = self.SystemSettings['pkg_masking_reference']['user_license_mask']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = -1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _idpackageValidator_keyword_mask(self, idpackage, reponame, live):
@@ -4576,7 +4593,11 @@ class LocalRepository:
for key in etpConst['keywords']:
if key not in mykeywords: continue
myr = self.SystemSettings['pkg_masking_reference']['system_keyword']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = idpackage, myr
except KeyError: # system settings client plugin not found
pass
return idpackage, myr
# if we get here, it means we didn't find mykeywords in etpConst['keywords']
@@ -4589,7 +4610,11 @@ class LocalRepository:
if not keyword_data: continue
if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok
myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords_all']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = idpackage, myr
except KeyError: # system settings client plugin not found
pass
return idpackage, myr
kwd_key = "%s_ids" % (keyword,)
keyword_data_ids = self.SystemSettings['keywords']['repositories'][reponame].get(kwd_key)
@@ -4602,7 +4627,11 @@ class LocalRepository:
self.SystemSettings['keywords']['repositories'][reponame][kwd_key] = keyword_data_ids
if idpackage in keyword_data_ids:
myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = idpackage, myr
except KeyError: # system settings client plugin not found
pass
return idpackage, myr
# if we get here, it means we didn't find a match in repositories
@@ -4625,29 +4654,35 @@ class LocalRepository:
if idpackage in keyword_data_ids:
# valid!
myr = self.SystemSettings['pkg_masking_reference']['user_package_keywords']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
validator_cache[(idpackage, reponame, live)] = idpackage, myr
except KeyError: # system settings client plugin not found
pass
return idpackage, myr
# function that validate one atom by reading keywords settings
# self.ServiceInterface._package_match_validator_cache = {} >> function cache
# validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
def idpackageValidator(self, idpackage, live = True):
if self.dbname == etpConst['clientdbid']:
return idpackage, 0
elif self.dbname.startswith(etpConst['serverdbid']):
return idpackage, 0
elif self.ServiceInterface == None:
return idpackage, 0
reponame = self.dbname[len(etpConst['dbnamerepoprefix']):]
cached = self.ServiceInterface._package_match_validator_cache.get((idpackage, reponame, live))
if cached != None:
return cached
# avoid memleaks
if len(self.ServiceInterface._package_match_validator_cache) > 10000:
self.ServiceInterface._package_match_validator_cache.clear()
try:
validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']
cached = validator_cache.get((idpackage, reponame, live))
if cached != None:
return cached
# avoid memleaks
if len(validator_cache) > 10000:
validator_cache.clear()
except KeyError: # plugin does not exist
pass
if live:
data = self._idpackageValidator_live(idpackage, reponame)
@@ -4670,7 +4705,7 @@ class LocalRepository:
# holy crap, can't validate
myr = self.SystemSettings['pkg_masking_reference']['completely_masked']
self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr
validator_cache[(idpackage, reponame, live)] = -1, myr
return -1, myr
# packages filter used by atomMatch, input must me foundIDs, a list like this:
libraries/tests/db.py
+8 -2
View File
@@ -14,6 +14,8 @@ class LocalRepositoryTest(unittest.TestCase):
self.Client = Client(noclientdb = 2, indexing = False, xcache = False,
repo_validation = False)
self.test_db_name = "%s_test_suite" % (etpConst['dbnamerepoprefix'],)
self.client_sysset_plugin_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
self.test_db = self.__open_test_db()
self.SystemSettings = SystemSettings()
@@ -55,16 +57,20 @@ class LocalRepositoryTest(unittest.TestCase):
self.test_db.atomMatch(pkg_atom))
# test package masking
plug_id = self.client_sysset_plugin_id
masking_validation = \
self.SystemSettings[plug_id]['masking_validation']['cache']
f_match_mask = (1,
self.test_db_name[len(etpConst['dbnamerepoprefix']):],)
self.SystemSettings['live_packagemasking']['mask_matches'].add(
f_match_mask)
self.Client._package_match_validator_cache.clear()
masking_validation.clear()
self.assertEqual((-1, 1),self.test_db.atomMatch(pkg_atom))
self.SystemSettings['live_packagemasking']['mask_matches'].discard(
f_match_mask)
self.Client._package_match_validator_cache.clear()
masking_validation.clear()
self.assertNotEqual((-1, 1),self.test_db.atomMatch(pkg_atom))
def test_db_package_sets(self):