From 2d2a967ef892f0d123a9ac77b65ff91f0dc2b811 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Mon, 22 Jun 2009 06:04:23 +0200 Subject: [PATCH] [entropy.core/entropy.db/entropy.spm/entropy.client] move SystemSettings client metadata to entropy.client SystemSettings plugin --- libraries/entropy/client/interfaces/client.py | 222 +++++++++++++++++- libraries/entropy/client/interfaces/dep.py | 15 +- .../entropy/client/interfaces/methods.py | 5 +- libraries/entropy/core.py | 207 +--------------- libraries/entropy/db.py | 6 +- libraries/entropy/spm.py | 16 +- 6 files changed, 255 insertions(+), 216 deletions(-) diff --git a/libraries/entropy/client/interfaces/client.py b/libraries/entropy/client/interfaces/client.py index ad831b3b6..ed9ca12ad 100644 --- a/libraries/entropy/client/interfaces/client.py +++ b/libraries/entropy/client/interfaces/client.py @@ -41,6 +41,107 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): import entropy.tools as entropyTools + def __init__(self, plugin_id, helper_interface): + SystemSettingsPlugin.__init__(self, plugin_id, helper_interface) + self.__repos_files = {} + self.__repos_mtime = {} + + def __setup_repos_files(self, system_settings): + """ + This function collects available repositories configuration files + by filling internal dict() __repos_files and __repos_mtime. + + @param system_settings: SystemSettings instance + @type system_settings: instance of SystemSettings + @return: None + @rtype: None + """ + + self.__repos_mtime = { + 'repos_license_whitelist': {}, + 'repos_mask': {}, + 'repos_system_mask': {}, + 'repos_critical_updates': {}, + } + self.__repos_files = { + 'repos_license_whitelist': {}, + 'repos_mask': {}, + 'repos_system_mask': {}, + 'conflicting_tagged_packages': {}, + 'repos_critical_updates': {}, + } + + dmp_dir = etpConst['dumpstoragedir'] + for repoid in system_settings['repositories']['order']: + + repos_mask_setting = {} + repos_mask_mtime = {} + repos_lic_wl_setting = {} + repos_lic_wl_mtime = {} + repo_data = system_settings['repositories']['available'][repoid] + repos_sm_mask_setting = {} + repos_sm_mask_mtime = {} + confl_tagged = {} + repos_critical_updates_setting = {} + repos_critical_updates_mtime = {} + + maskpath = os.path.join(repo_data['dbpath'], + etpConst['etpdatabasemaskfile']) + wlpath = os.path.join(repo_data['dbpath'], + etpConst['etpdatabaselicwhitelistfile']) + sm_path = os.path.join(repo_data['dbpath'], + etpConst['etpdatabasesytemmaskfile']) + ct_path = os.path.join(repo_data['dbpath'], + etpConst['etpdatabaseconflictingtaggedfile']) + critical_path = os.path.join(repo_data['dbpath'], + etpConst['etpdatabasecriticalfile']) + + if os.access(maskpath, os.R_OK | os.F_OK): + repos_mask_setting[repoid] = maskpath + repos_mask_mtime[repoid] = dmp_dir + "/repo_" + \ + repoid + "_" + etpConst['etpdatabasemaskfile'] + ".mtime" + + if os.access(wlpath, os.R_OK | os.F_OK): + repos_lic_wl_setting[repoid] = wlpath + repos_lic_wl_mtime[repoid] = dmp_dir + "/repo_" + \ + repoid + "_" + etpConst['etpdatabaselicwhitelistfile'] + \ + ".mtime" + + if os.access(sm_path, os.R_OK | os.F_OK): + repos_sm_mask_setting[repoid] = sm_path + repos_sm_mask_mtime[repoid] = dmp_dir + "/repo_" + \ + repoid + "_" + etpConst['etpdatabasesytemmaskfile'] + \ + ".mtime" + if os.access(ct_path, os.R_OK | os.F_OK): + confl_tagged[repoid] = ct_path + + if os.access(critical_path, os.R_OK | os.F_OK): + repos_critical_updates_setting[repoid] = critical_path + repos_critical_updates_mtime[repoid] = dmp_dir + "/repo_" + \ + repoid + "_" + etpConst['etpdatabasecriticalfile'] + \ + ".mtime" + + self.__repos_files['repos_mask'].update(repos_mask_setting) + self.__repos_mtime['repos_mask'].update(repos_mask_mtime) + + self.__repos_files['repos_license_whitelist'].update( + repos_lic_wl_setting) + self.__repos_mtime['repos_license_whitelist'].update( + repos_lic_wl_mtime) + + self.__repos_files['repos_system_mask'].update( + repos_sm_mask_setting) + self.__repos_mtime['repos_system_mask'].update( + repos_sm_mask_mtime) + + self.__repos_files['conflicting_tagged_packages'].update( + confl_tagged) + + self.__repos_files['repos_critical_updates'].update( + repos_critical_updates_setting) + self.__repos_mtime['repos_critical_updates'].update( + repos_critical_updates_mtime) + def system_mask_parser(self, system_settings_instance): parser_data = {} @@ -53,8 +154,9 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): except SystemDatabaseError: break mc_cache = set() - m_list = system_settings_instance['repos_system_mask'] + \ - system_settings_instance['system_mask'] + repos_mask_list = self.__repositories_system_mask( + system_settings_instance) + m_list = repos_mask_list + system_settings_instance['system_mask'] for atom in m_list: m_ids, m_r = self._helper.clientDbconn.atomMatch(atom, multiMatch = True) @@ -83,6 +185,122 @@ class ClientSystemSettingsPlugin(SystemSettingsPlugin): } return data + def __repositories_system_mask(self, sys_settings_instance): + """ + Parser returning system packages mask metadata read from + packages.db.system_mask file inside the repository directory. + This file contains packages that should be always kept + installed, extending the already defined (in repository database) + set of atoms. + """ + system_mask = [] + for repoid in self.__repos_files['repos_system_mask']: + sys_settings_instance.validate_entropy_cache( + self.__repos_files['repos_system_mask'][repoid], + self.__repos_mtime['repos_system_mask'][repoid], + repoid = repoid) + system_mask += [x for x in \ + self.entropyTools.generic_file_content_parser( + self.__repos_files['repos_system_mask'][repoid]) if x \ + not in system_mask] + return system_mask + + def repositories_parser(self, sys_settings_instance): + """ + Parser that generates repository settings metadata. + + @param sys_settings_instance: SystemSettings instance + @type sys_settings_instance: instance of SystemSettings + @return: parsed metadata + @rtype: dict + """ + # fill repositories metadata dictionaries + self.__setup_repos_files(sys_settings_instance) + + data = { + 'license_whitelist': {}, + 'mask': {}, + 'system_mask': [], + 'critical_updates': {}, + 'conflicting_tagged_packages': {}, + } + + # parse license whitelist + """ + Parser returning licenses considered accepted by default + (= GPL compatibles) read from package.lic_whitelist. + """ + for repoid in self.__repos_files['repos_license_whitelist']: + sys_settings_instance.validate_entropy_cache( + self.__repos_files['repos_license_whitelist'][repoid], + self.__repos_mtime['repos_license_whitelist'][repoid], + repoid = repoid) + + data['license_whitelist'][repoid] = \ + self.entropyTools.generic_file_content_parser( + self.__repos_files['repos_license_whitelist'][repoid]) + + # package masking + """ + Parser returning packages masked at repository level read from + packages.db.mask inside the repository database directory. + """ + for repoid in self.__repos_files['repos_mask']: + sys_settings_instance.validate_entropy_cache( + self.__repos_files['repos_mask'][repoid], + self.__repos_mtime['repos_mask'][repoid], repoid = repoid) + + data['mask'][repoid] = \ + self.entropyTools.generic_file_content_parser( + self.__repos_files['repos_mask'][repoid]) + + # system masking + data['system_mask'] = self.__repositories_system_mask( + sys_settings_instance) + + # critical updates + """ + Parser returning critical packages list metadata read from + packages.db.critical file inside the repository directory. + This file contains packages that should be always updated + before anything else. + """ + for repoid in self.__repos_files['repos_critical_updates']: + sys_settings_instance.validate_entropy_cache( + self.__repos_files['repos_critical_updates'][repoid], + self.__repos_mtime['repos_critical_updates'][repoid], + repoid = repoid) + data['critical_updates'][repoid] = \ + self.entropyTools.generic_file_content_parser( + self.__repos_files['repos_critical_updates'][repoid]) + + + # conflicts map + """ + Parser returning packages that could have been installed because + they aren't in the same scope, but ending up creating critical + issues. You can see it as a configurable conflict map. + """ + # keep priority order + repoids = [x for x in sys_settings_instance['repositories']['order'] \ + if x in self.__repos_files['conflicting_tagged_packages']] + for repoid in repoids: + filepath = self.__repos_files['conflicting_tagged_packages'].get( + repoid) + if os.access(filepath, os.R_OK | os.F_OK): + confl_f = open(filepath,"r") + content = confl_f.readlines() + confl_f.close() + content = [x.strip().rsplit("#", 1)[0].strip().split() for x \ + in content if not x.startswith("#") and x.strip()] + for mydata in content: + if len(mydata) < 2: + continue + data['conflicting_tagged_packages'][mydata[0]] = mydata[1:] + + return data + + def misc_parser(self, sys_settings_instance): """ diff --git a/libraries/entropy/client/interfaces/dep.py b/libraries/entropy/client/interfaces/dep.py index a9d4c3e9b..5d0004f5f 100644 --- a/libraries/entropy/client/interfaces/dep.py +++ b/libraries/entropy/client/interfaces/dep.py @@ -992,8 +992,11 @@ class CalculatorsMixin: def _lookup_system_mask_repository_deps(self): - data = self.SystemSettings['repos_system_mask'] - if not data: return [] + client_settings = self.SystemSettings[self.sys_settings_client_plugin_id] + data = client_settings['repositories']['system_mask'] + + if not data: + return [] mydata = [] cached_items = set() for atom in data: @@ -1426,7 +1429,9 @@ class CalculatorsMixin: if cached != None: return cached - critical_data = self.SystemSettings['repos_critical_updates'] + client_settings = self.SystemSettings[self.sys_settings_client_plugin_id] + critical_data = client_settings['repositories']['critical_updates'] + atoms = set() atom_matches = {} for repoid in critical_data: @@ -1628,8 +1633,8 @@ class CalculatorsMixin: pkgatom = self.clientDbconn.retrieveAtom(idpackage) pkgkey = self.entropyTools.dep_getkey(pkgatom) - client_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin'] - mask_installed_keys = self.SystemSettings[client_plugin_id]['system_mask']['repos_installed_keys'] + client_settings = self.SystemSettings[self.sys_settings_client_plugin_id] + mask_installed_keys = client_settings['system_mask']['repos_installed_keys'] if self.is_installed_idpackage_in_system_mask(idpackage): idpackages = mask_installed_keys.get(pkgkey) diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index e394fe30a..7f0802933 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -946,10 +946,13 @@ class MiscMixin: if not install_queue: return {} licenses = {} + cl_id = self.sys_settings_client_plugin_id + repo_sys_data = self.SystemSettings[cl_id]['repositories'] + for match in install_queue: repoid = match[1] dbconn = self.open_repository(repoid) - wl = self.SystemSettings['repos_license_whitelist'].get(repoid) + wl = repo_sys_data['license_whitelist'].get(repoid) if not wl: continue keys = dbconn.retrieveLicensedataKeys(match[0]) diff --git a/libraries/entropy/core.py b/libraries/entropy/core.py index 5d0f75c47..dd5bbe092 100644 --- a/libraries/entropy/core.py +++ b/libraries/entropy/core.py @@ -348,19 +348,12 @@ class SystemSettings(Singleton): 'socket_service': etpConst['socketconf'], 'system': etpConst['entropyconf'], 'repositories': etpConst['repositoriesconf'], - 'repos_system_mask': {}, - 'repos_mask': {}, - 'repos_license_whitelist': {}, - 'repos_critical_updates': {}, 'system_package_sets': {}, - 'conflicting_tagged_packages': {}, }) self.__setting_files_order.extend([ 'keywords', 'unmask', 'mask', 'satisfied', 'license_mask', - 'repos_system_mask', 'system_mask', 'repos_mask', - 'repos_critical_updates', 'repos_license_whitelist', - 'system_package_sets', 'conflicting_tagged_packages', - 'system_dirs', 'system_dirs_mask', 'socket_service', 'system', + 'system_mask', 'system_package_sets', 'system_dirs', + 'system_dirs_mask', 'socket_service', 'system', 'system_rev_symlinks', 'hw_hash' ]) self.__setting_files_pre_run.extend(['repositories']) @@ -373,10 +366,6 @@ class SystemSettings(Singleton): 'satisfied_mtime': dmp_dir+"/satisfied.mtime", 'license_mask_mtime': dmp_dir+"/license_mask.mtime", 'system_mask_mtime': dmp_dir+"/system_mask.mtime", - 'repos_system_mask': {}, - 'repos_mask': {}, - 'repos_license_whitelist': {}, - 'repos_critical_updates': {}, }) @@ -459,12 +448,12 @@ class SystemSettings(Singleton): with self.__mutex: return len(self.__data) - def get(self, mykey): + def get(self, mykey, alt_obj = None): """ dict method. See Python dict API reference. """ with self.__mutex: - return self.__data.get(mykey) + return self.__data.get(mykey, alt_obj) def has_key(self, mykey): """ @@ -590,88 +579,6 @@ class SystemSettings(Singleton): del self.__persistent_settings[persistent_key] del self.__data[persistent_key] - def __setup_setting_vars(self): - - """ - This function setups the *mtimes* and *files* dictionaries - that will be read and parsed afterwards by respective - internal parsers. - - @return: None - @rtype: None - """ - - dmp_dir = etpConst['dumpstoragedir'] - for repoid in self['repositories']['order']: - - repos_mask_setting = {} - repos_mask_mtime = {} - repos_lic_wl_setting = {} - repos_lic_wl_mtime = {} - repo_data = self['repositories']['available'][repoid] - repos_sm_mask_setting = {} - repos_sm_mask_mtime = {} - confl_tagged = {} - repos_critical_updates_setting = {} - repos_critical_updates_mtime = {} - - maskpath = os.path.join(repo_data['dbpath'], - etpConst['etpdatabasemaskfile']) - wlpath = os.path.join(repo_data['dbpath'], - etpConst['etpdatabaselicwhitelistfile']) - sm_path = os.path.join(repo_data['dbpath'], - etpConst['etpdatabasesytemmaskfile']) - ct_path = os.path.join(repo_data['dbpath'], - etpConst['etpdatabaseconflictingtaggedfile']) - critical_path = os.path.join(repo_data['dbpath'], - etpConst['etpdatabasecriticalfile']) - - if os.access(maskpath, os.R_OK | os.F_OK): - repos_mask_setting[repoid] = maskpath - repos_mask_mtime[repoid] = dmp_dir + "/repo_" + \ - repoid + "_" + etpConst['etpdatabasemaskfile'] + ".mtime" - - if os.access(wlpath, os.R_OK | os.F_OK): - repos_lic_wl_setting[repoid] = wlpath - repos_lic_wl_mtime[repoid] = dmp_dir + "/repo_" + \ - repoid + "_" + etpConst['etpdatabaselicwhitelistfile'] + \ - ".mtime" - - if os.access(sm_path, os.R_OK | os.F_OK): - repos_sm_mask_setting[repoid] = sm_path - repos_sm_mask_mtime[repoid] = dmp_dir + "/repo_" + \ - repoid + "_" + etpConst['etpdatabasesytemmaskfile'] + \ - ".mtime" - if os.access(ct_path, os.R_OK | os.F_OK): - confl_tagged[repoid] = ct_path - - if os.access(critical_path, os.R_OK | os.F_OK): - repos_critical_updates_setting[repoid] = critical_path - repos_critical_updates_mtime[repoid] = dmp_dir + "/repo_" + \ - repoid + "_" + etpConst['etpdatabasecriticalfile'] + \ - ".mtime" - - self.__setting_files['repos_mask'].update(repos_mask_setting) - self.__mtime_files['repos_mask'].update(repos_mask_mtime) - - self.__setting_files['repos_license_whitelist'].update( - repos_lic_wl_setting) - self.__mtime_files['repos_license_whitelist'].update( - repos_lic_wl_mtime) - - self.__setting_files['repos_system_mask'].update( - repos_sm_mask_setting) - self.__mtime_files['repos_system_mask'].update( - repos_sm_mask_mtime) - - self.__setting_files['conflicting_tagged_packages'].update( - confl_tagged) - - self.__setting_files['repos_critical_updates'].update( - repos_critical_updates_setting) - self.__mtime_files['repos_critical_updates'].update( - repos_critical_updates_mtime) - def __setup_package_sets_vars(self): """ @@ -716,7 +623,6 @@ class SystemSettings(Singleton): self.__data[item] = func() # parse main settings - self.__setup_setting_vars() self.__setup_package_sets_vars() data = {} @@ -890,83 +796,6 @@ class SystemSettings(Singleton): self.__mtime_files['license_mask_mtime']) return self.__generic_parser(self.__setting_files['license_mask']) - def _repos_license_whitelist_parser(self): - """ - Parser returning licenses considered accepted by default - (= GPL compatibles) read from package.lic_whitelist. - - @return: parsed metadata - @rtype: dict - """ - data = {} - for repoid in self.__setting_files['repos_license_whitelist']: - self.validate_entropy_cache( - self.__setting_files['repos_license_whitelist'][repoid], - self.__mtime_files['repos_license_whitelist'][repoid], - repoid = repoid) - data[repoid] = self.__generic_parser( - self.__setting_files['repos_license_whitelist'][repoid]) - return data - - def _repos_mask_parser(self): - """ - Parser returning packages masked at repository level read from - packages.db.mask inside the repository database directory. - - @return: parsed metadata - @rtype: dict - """ - data = {} - for repoid in self.__setting_files['repos_mask']: - self.validate_entropy_cache( - self.__setting_files['repos_mask'][repoid], - self.__mtime_files['repos_mask'][repoid], repoid = repoid) - data[repoid] = self.__generic_parser( - self.__setting_files['repos_mask'][repoid]) - return data - - def _repos_system_mask_parser(self): - """ - Parser returning system packages mask metadata read from - packages.db.system_mask file inside the repository directory. - This file contains packages that should be always kept - installed, extending the already defined (in repository database) - set of atoms. - - @return: parsed metadata - @rtype: dict - """ - data = [] - for repoid in self.__setting_files['repos_system_mask']: - self.validate_entropy_cache( - self.__setting_files['repos_system_mask'][repoid], - self.__mtime_files['repos_system_mask'][repoid], - repoid = repoid) - data += [x for x in self.__generic_parser( - self.__setting_files['repos_system_mask'][repoid]) if x \ - not in data] - return data - - def _repos_critical_updates_parser(self): - """ - Parser returning critical packages list metadata read from - packages.db.critical file inside the repository directory. - This file contains packages that should be always updated - before anything else. - - @return: parsed metadata - @rtype: dict - """ - data = {} - for repoid in self.__setting_files['repos_critical_updates']: - self.validate_entropy_cache( - self.__setting_files['repos_critical_updates'][repoid], - self.__mtime_files['repos_critical_updates'][repoid], - repoid = repoid) - data[repoid] = self.__generic_parser( - self.__setting_files['repos_critical_updates'][repoid]) - return data - def _system_package_sets_parser(self): """ Parser returning system defined package sets read from @@ -1052,34 +881,6 @@ class SystemSettings(Singleton): data[line[0]] = frozenset(line[1:]) return data - def _conflicting_tagged_packages_parser(self): - """ - Parser returning packages that could have been installed because - they aren't in the same scope, but ending up creating critical - issues. You can see it as a configurable conflict map. - - @return: parsed metadata - @rtype: dict - """ - data = {} - # keep priority order - repoids = [x for x in self['repositories']['order'] if x in \ - self.__setting_files['conflicting_tagged_packages']] - for repoid in repoids: - filepath = self.__setting_files['conflicting_tagged_packages'].get( - repoid) - if os.path.isfile(filepath) and os.access(filepath, os.R_OK): - confl_f = open(filepath,"r") - content = confl_f.readlines() - confl_f.close() - content = [x.strip().rsplit("#", 1)[0].strip().split() for x \ - in content if not x.startswith("#") and x.strip()] - for mydata in content: - if len(mydata) < 2: - continue - data[mydata[0]] = mydata[1:] - return data - def _socket_service_parser(self): """ Parses socket service configuration file. diff --git a/libraries/entropy/db.py b/libraries/entropy/db.py index 17804b62e..df5c6e9cd 100644 --- a/libraries/entropy/db.py +++ b/libraries/entropy/db.py @@ -4843,7 +4843,11 @@ class LocalRepository(EntropyRepository): def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live): # check if repository packages.db.mask needs it masked - repos_mask = self.SystemSettings['repos_mask'] + repos_mask = {} + client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin'] + client_settings = self.SystemSettings.get(client_plg_id, {}) + if client_settings: + repos_mask = client_settings['repositories']['mask'] repomask = repos_mask.get(reponame) if isinstance(repomask, (list, set,)): # first, seek into generic masking, all branches diff --git a/libraries/entropy/spm.py b/libraries/entropy/spm.py index 63265a371..404a80683 100644 --- a/libraries/entropy/spm.py +++ b/libraries/entropy/spm.py @@ -2145,10 +2145,18 @@ class PortagePlugin: data['dependencies'][u"=sys-kernel/linux-"+kname+"-"+kver+"~-1"] = etpConst['spm']['(r)depend_id'] # Conflicting tagged packages support - key = data['category']+"/"+data['name'] - confl_data = system_settings['conflicting_tagged_packages'].get(key) - if confl_data != None: - for conflict in confl_data: data['conflicts'].append(conflict) + # Needs Entropy Client System Settings Plugin, + # but since entropy.server loads entropy.client, it's completely + # fine as of now. + key = data['category'] + "/" + data['name'] + client_sysset_plg_id = etpConst['system_settings_plugins_ids']['client_plugin'] + client_data = system_settings.get(client_sysset_plg_id, {}) + confl_data = None + if client_data: + confl_data = client_data['repositories']['conflicting_tagged_packages'].get(key) + if confl_data: + for conflict in confl_data: + data['conflicts'].append(conflict) # conflicts must be a set, which is what is returned # by entropy.db.getPackageData