From 0a901a3de62c719bcaf3739ee13ffddaee2e0c94 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Sun, 19 Apr 2009 10:23:55 +0200 Subject: [PATCH] entropy.*: move atom matching package masking validation cache to SystemSettings --- docs/APICHANGES | 2 +- libraries/entropy/client/interfaces/client.py | 7 +- .../entropy/client/interfaces/methods.py | 4 +- libraries/entropy/db.py | 71 ++++++++++++++----- libraries/tests/db.py | 10 ++- 5 files changed, 70 insertions(+), 24 deletions(-) diff --git a/docs/APICHANGES b/docs/APICHANGES index 831553060..77637eeb5 100644 --- a/docs/APICHANGES +++ b/docs/APICHANGES @@ -110,7 +110,7 @@ From SVN 3112 there have been several API changes: securityCache => __security_cache QACache => __QA_cache memoryDbInstances => _memory_db_instances - package_match_validator_cache => _package_match_validator_cache + package_match_validator_cache => SystemSettings[etpConst['system_settings_plugins_ids']['client_plugin']]['masking_validation']['cache'] spmCache => __spm_cache - ServerInterface (now entropy.server.interfaces.Server) API changes: diff --git a/libraries/entropy/client/interfaces/client.py b/libraries/entropy/client/interfaces/client.py index 36591cf67..0d5abcdb7 100644 --- a/libraries/entropy/client/interfaces/client.py +++ b/libraries/entropy/client/interfaces/client.py @@ -76,6 +76,12 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): }) return parser_data + def masking_validation_parser(self, system_settings_instance): + data = { + 'cache': {}, # package masking validation cache + } + return data + def repo_setup_parser(self, system_settings_instance): # this makes sure that repository metadata is initialized @@ -143,7 +149,6 @@ class Client(Singleton, TextInterface, LoadersMixin, CacheMixin, CalculatorsMixi self.atomMatchCacheKey = etpCache['atomMatch'] self.dbapi2 = dbapi2 # export for third parties self.FileUpdates = None - self._package_match_validator_cache = {} self.validRepositories = [] self.UGC = None # supporting external updateProgress stuff, you can point self.progress diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index 82d19ab48..8cc711b20 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -41,7 +41,7 @@ class RepositoryMixin: def validate_repositories(self): self.MirrorStatus.clear() self.__repo_error_messages_cache.clear() - self._package_match_validator_cache.clear() + self.SystemSettings['package_masking']['cache'].clear() # valid repositories del self.validRepositories[:] for repoid in self.SystemSettings['repositories']['order']: @@ -1570,7 +1570,7 @@ class MatchMixin: self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],)) self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],)) - self._package_match_validator_cache.clear() + self.SystemSettings['package_masking']['cache'].clear() return done def unmask_match_by_atom(self, match, dry_run = False): diff --git a/libraries/entropy/db.py b/libraries/entropy/db.py index a26dfef00..5d91405d8 100644 --- a/libraries/entropy/db.py +++ b/libraries/entropy/db.py @@ -354,6 +354,7 @@ class LocalRepository: self.dbSearchCacheKey = etpCache['dbSearch'] self.dbname = dbname self.lockRemote = lockRemote + self.client_settings_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin'] self.db_branch = self.SystemSettings['repositories']['branch'] if self.dbname == etpConst['clientdbid']: self.db_branch = None @@ -4516,7 +4517,11 @@ class LocalRepository: if idpackage in user_package_mask_ids: # sorry, masked myr = self.SystemSettings['pkg_masking_reference']['user_package_mask'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = -1, myr + except KeyError: # system settings client plugin not found + pass return -1, myr def _idpackageValidator_user_package_unmask(self, idpackage, reponame, live): @@ -4532,7 +4537,11 @@ class LocalRepository: self.SystemSettings[mykw] = user_package_unmask_ids if idpackage in user_package_unmask_ids: myr = self.SystemSettings['pkg_masking_reference']['user_package_unmask'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = idpackage, myr + except KeyError: # system settings client plugin not found + pass return idpackage, myr def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live): @@ -4552,7 +4561,11 @@ class LocalRepository: repos_mask[mask_repo_id] = repomask_ids if idpackage in repomask_ids: myr = self.SystemSettings['pkg_masking_reference']['repository_packages_db_mask'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = -1, myr + except KeyError: # system settings client plugin not found + pass return -1, myr def _idpackageValidator_package_license_mask(self, idpackage, reponame, live): @@ -4563,7 +4576,11 @@ class LocalRepository: for mylicense in mylicenses: if mylicense not in lic_mask: continue myr = self.SystemSettings['pkg_masking_reference']['user_license_mask'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = -1, myr + except KeyError: # system settings client plugin not found + pass return -1, myr def _idpackageValidator_keyword_mask(self, idpackage, reponame, live): @@ -4576,7 +4593,11 @@ class LocalRepository: for key in etpConst['keywords']: if key not in mykeywords: continue myr = self.SystemSettings['pkg_masking_reference']['system_keyword'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = idpackage, myr + except KeyError: # system settings client plugin not found + pass return idpackage, myr # if we get here, it means we didn't find mykeywords in etpConst['keywords'] @@ -4589,7 +4610,11 @@ class LocalRepository: if not keyword_data: continue if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords_all'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = idpackage, myr + except KeyError: # system settings client plugin not found + pass return idpackage, myr kwd_key = "%s_ids" % (keyword,) keyword_data_ids = self.SystemSettings['keywords']['repositories'][reponame].get(kwd_key) @@ -4602,7 +4627,11 @@ class LocalRepository: self.SystemSettings['keywords']['repositories'][reponame][kwd_key] = keyword_data_ids if idpackage in keyword_data_ids: myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = idpackage, myr + except KeyError: # system settings client plugin not found + pass return idpackage, myr # if we get here, it means we didn't find a match in repositories @@ -4625,29 +4654,35 @@ class LocalRepository: if idpackage in keyword_data_ids: # valid! myr = self.SystemSettings['pkg_masking_reference']['user_package_keywords'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = idpackage, myr + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + validator_cache[(idpackage, reponame, live)] = idpackage, myr + except KeyError: # system settings client plugin not found + pass return idpackage, myr # function that validate one atom by reading keywords settings - # self.ServiceInterface._package_match_validator_cache = {} >> function cache + # validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] def idpackageValidator(self, idpackage, live = True): if self.dbname == etpConst['clientdbid']: return idpackage, 0 elif self.dbname.startswith(etpConst['serverdbid']): return idpackage, 0 - elif self.ServiceInterface == None: - return idpackage, 0 reponame = self.dbname[len(etpConst['dbnamerepoprefix']):] - cached = self.ServiceInterface._package_match_validator_cache.get((idpackage, reponame, live)) - if cached != None: - return cached - # avoid memleaks - if len(self.ServiceInterface._package_match_validator_cache) > 10000: - self.ServiceInterface._package_match_validator_cache.clear() + try: + validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] + cached = validator_cache.get((idpackage, reponame, live)) + if cached != None: + return cached + # avoid memleaks + if len(validator_cache) > 10000: + validator_cache.clear() + except KeyError: # plugin does not exist + pass if live: data = self._idpackageValidator_live(idpackage, reponame) @@ -4670,7 +4705,7 @@ class LocalRepository: # holy crap, can't validate myr = self.SystemSettings['pkg_masking_reference']['completely_masked'] - self.ServiceInterface._package_match_validator_cache[(idpackage, reponame, live)] = -1, myr + validator_cache[(idpackage, reponame, live)] = -1, myr return -1, myr # packages filter used by atomMatch, input must me foundIDs, a list like this: diff --git a/libraries/tests/db.py b/libraries/tests/db.py index 7faeff4db..568d6c3a2 100644 --- a/libraries/tests/db.py +++ b/libraries/tests/db.py @@ -14,6 +14,8 @@ class LocalRepositoryTest(unittest.TestCase): self.Client = Client(noclientdb = 2, indexing = False, xcache = False, repo_validation = False) self.test_db_name = "%s_test_suite" % (etpConst['dbnamerepoprefix'],) + self.client_sysset_plugin_id = \ + etpConst['system_settings_plugins_ids']['client_plugin'] self.test_db = self.__open_test_db() self.SystemSettings = SystemSettings() @@ -55,16 +57,20 @@ class LocalRepositoryTest(unittest.TestCase): self.test_db.atomMatch(pkg_atom)) # test package masking + plug_id = self.client_sysset_plugin_id + masking_validation = \ + self.SystemSettings[plug_id]['masking_validation']['cache'] f_match_mask = (1, self.test_db_name[len(etpConst['dbnamerepoprefix']):],) + self.SystemSettings['live_packagemasking']['mask_matches'].add( f_match_mask) - self.Client._package_match_validator_cache.clear() + masking_validation.clear() self.assertEqual((-1, 1),self.test_db.atomMatch(pkg_atom)) self.SystemSettings['live_packagemasking']['mask_matches'].discard( f_match_mask) - self.Client._package_match_validator_cache.clear() + masking_validation.clear() self.assertNotEqual((-1, 1),self.test_db.atomMatch(pkg_atom)) def test_db_package_sets(self):