# -*- coding: utf-8 -*- """ @author: Fabio Erculiani @contact: lxnay@sabayon.org @copyright: Fabio Erculiani @license: GPL-2 B{Entropy Package Manager Client EntropyRepository plugin code}. """ import os import sys import subprocess import tempfile import threading import shutil import time from entropy.const import const_debug_write, const_setup_perms, etpConst, \ etpUi, const_set_nice_level, const_setup_file from entropy.output import blue, darkred, red, darkgreen, purple, teal, brown, \ bold, TextInterface from entropy.dump import dumpobj, loadobj from entropy.cache import EntropyCacher from entropy.db import EntropyRepository from entropy.exceptions import RepositoryError, SystemDatabaseError, \ PermissionDenied from entropy.security import Repository as RepositorySecurity from entropy.misc import TimeScheduled, ParallelTask from entropy.i18n import _ from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase from entropy.db.exceptions import IntegrityError, OperationalError, Error, \ DatabaseError from entropy.core.settings.base import SystemSettings from entropy.services.client import WebService from entropy.client.services.interfaces import RepositoryWebService, \ RepositoryWebServiceFactory import entropy.dep import entropy.tools __all__ = ["CachedRepository", "ClientEntropyRepositoryPlugin", "InstalledPackagesRepository", "AvailablePackagesRepository", "GenericRepository"] class ClientEntropyRepositoryPlugin(EntropyRepositoryPlugin): def __init__(self, client_interface, metadata = None): """ Entropy client-side repository EntropyRepository Plugin class. This class will be instantiated and automatically added to EntropyRepository instances generated by Entropy Client. @param client_interface: Entropy Client interface instance @type client_interface: entropy.client.interfaces.Client class @param metadata: any dict form metadata map (key => value) @type metadata: dict """ EntropyRepositoryPlugin.__init__(self) self._client = client_interface if metadata is None: self._metadata = {} else: self._metadata = metadata def get_id(self): return "__client__" def get_metadata(self): return self._metadata def add_plugin_hook(self, entropy_repository_instance): const_debug_write(__name__, "ClientEntropyRepositoryPlugin: calling add_plugin_hook => %s" % ( self,) ) out_intf = self._metadata.get('output_interface') if out_intf is not None: entropy_repository_instance.output = out_intf.output entropy_repository_instance.ask_question = out_intf.ask_question return 0 class CachedRepository(EntropyRepository): """ This kind of repository cannot have close() called directly, without a valid token passed. This is because the class object is cached somewhere and calling close() would turn into a software bug. """ def setCloseToken(self, token): """ Set a token that can be used to validate close() calls. Calling close() on these repos is prohibited and considered a software bug. Only Entropy Client should be able to close them. """ self._close_token = token def close(self, _token = None): """ Reimplemented from EntropyRepository """ close_token = getattr(self, "_close_token", None) if close_token is not None: if (_token is None) or (_token != close_token): raise PermissionDenied( "cannot close this repository directly. Software bug!") return EntropyRepository.close(self) def __del__(self): """ Cannot honor the constraint in this case, sorry! """ return EntropyRepository.close(self) class InstalledPackagesRepository(CachedRepository): """ This class represents the installed packages repository and is a direct subclass of EntropyRepository. """ def __init__(self, *args, **kwargs): EntropyRepository.__init__(self, *args, **kwargs) # ensure proper repository file permissions if entropy.tools.is_root() and os.path.isfile(self._db_path): const_setup_file(self._db_path, etpConst['entropygid'], 0o644, uid = etpConst['uid']) def handlePackage(self, pkg_data, forcedRevision = -1, formattedContent = False): """ Reimplemented from EntropyRepository. """ removelist = self.getPackagesToRemove( pkg_data['name'], pkg_data['category'], pkg_data['slot'], pkg_data['injected'] ) for r_package_id in removelist: self.removePackage(r_package_id, do_cleanup = False, do_commit = False) return self.addPackage(pkg_data, revision = forcedRevision, formatted_content = formattedContent) class AvailablePackagesRepositoryUpdater(object): """ AvailablePackagesRepository update logic class. The required logic for updating a repository is stored here. """ WEBSERV_CACHE_ID = 'webserv_repo/segment_' def __init__(self, entropy_client, repository_id, force, gpg): self.__force = force self.__big_sock_timeout = 20 self._repository_id = repository_id self._cacher = EntropyCacher() self._last_rev = -1 self._entropy = entropy_client self._settings = SystemSettings() self._gpg_feature = gpg self._supported_apis = etpConst['supportedapis'] self._supported_download_items = ( "db", "dbck", "dblight", "ck", "cklight", "compck", "lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck", "meta_file", "meta_file_gpg", "notice_board" ) self._developer_repo = \ self._settings['repositories']['developer_repo'] self._differential_update = \ self._settings['repositories']['differential_update'] if self._developer_repo: const_debug_write(__name__, "__init__: developer repo mode enabled") self.__webservices = None self.__webservice = None self._repo_eapi = self.__get_repo_eapi() avail_data = self._settings['repositories']['available'] if self._repository_id not in avail_data: raise KeyError("Repository not available") @property def _webservices(self): if self.__webservices is None: if hasattr(self._entropy, "RepositoryWebServices"): self.__webservices = self._entropy.RepositoryWebServices() else: # in case self._entropy is a simple TextInterface() # like how it's called in remote_revision(). self.__webservices = RepositoryWebServiceFactory(self._entropy) # cross fingers! return self.__webservices @property def _webservice(self): if self.__webservice is None: self.__webservice = self._webservices.new(self._repository_id) self.__webservice._set_timeout(self.__big_sock_timeout) return self.__webservice def __get_webserv_repository_metadata(self): try: data = self._webservice.get_repository_metadata() except WebService.WebServiceException as err: const_debug_write(__name__, "__get_webserv_repository_metadata: error: %s" % (err,)) data = {} return data def __get_webserv_repository_revision(self): try: revision = self._webservice.get_revision() except WebService.WebServiceException as err: const_debug_write(__name__, "__get_webserv_repository_revision: error: %s" % (err,)) revision = None return revision def __check_webserv_availability(self): try: webserv = self._webservices.new(self._repository_id) except WebService.UnsupportedService: return False try: available = webserv.service_available(cache = False) except WebService.WebServiceException: available = False return available def __get_webserv_local_database(self): avail_data = self._settings['repositories']['available'] repo_data = avail_data[self._repository_id] dbfile = os.path.join(repo_data['dbpath'], etpConst['etpdatabasefile']) dbconn = None try: dbconn = self._entropy.open_generic_repository(dbfile, xcache = False, indexing_override = False) dbconn.validate() except (OperationalError, IntegrityError, SystemDatabaseError, IOError, OSError,): dbconn = None return dbconn def __get_webserv_database_differences(self, webserv, package_ids): try: remote_package_ids = webserv.get_package_ids() except WebService.WebServiceException as err: const_debug_write(__name__, "__get_webserv_database_differences: error: %s" % (err,)) return None, None added = [x for x in remote_package_ids if x not in package_ids] removed = [x for x in package_ids if x not in remote_package_ids] return added, removed def __eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old): dbconn = self._entropy.open_generic_repository(dbfile, xcache = False, indexing_override = False) old_dbconn = self._entropy.open_generic_repository(dbfile_old, xcache = False, indexing_override = False) upd_rc = 0 try: upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") except (OperationalError, IntegrityError, DatabaseError,): pass old_dbconn.close() dbconn.close() if upd_rc > 0: # -1 means no changes, == force used # 0 means too much hassle os.rename(dbfile_old, dbfile) return upd_rc def __eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod): # load the dump into database mytxt = "%s %s, %s %s" % ( red(_("Injecting downloaded dump")), darkgreen(etpConst['etpdatabasedumplight']), red(_("please wait")), red("..."), ) self._entropy.output( mytxt, importance = 0, level = "info", header = "\t" ) dbconn = self._entropy.open_generic_repository(dbfile, xcache = False, indexing_override = False) rc = dbconn.importRepository(dumpfile, dbfile) dbconn.close() return rc def __get_repo_eapi(self): eapi_env = os.getenv("FORCE_EAPI") sqlite3_access = os.access("/usr/bin/sqlite3", os.X_OK) sqlite3_rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True) try: eapi_env_clear = int(eapi_env) if eapi_env_clear not in self._supported_apis: raise ValueError() except (ValueError, TypeError,): eapi_env_clear = None repo_eapi = 2 eapi_avail = self.__check_webserv_availability() if eapi_avail: repo_eapi = 3 else: if not sqlite3_access or entropy.tools.islive(): repo_eapi = 1 elif sqlite3_rc != 0: repo_eapi = 1 # if differential update is disabled and FORCE_EAPI is not overriding # we cannot use EAPI=3 if (eapi_env_clear is None) and (not self._differential_update) and \ (repo_eapi == 3): const_debug_write(__name__, "__get_repo_eapi: differential update is disabled !") repo_eapi -= 1 # check EAPI if eapi_env_clear is not None: repo_eapi = eapi_env_clear # FORCE_EAPI is triggered, disable # developer_repo mode self._developer_repo = False const_debug_write(__name__, "__get_repo_eapi: developer repo mode disabled FORCE_EAPI") elif repo_eapi > 1 and self._developer_repo: # enforce EAPI=1 repo_eapi = 1 const_debug_write(__name__, "__get_repo_eapi: final eapi set to %s" % (repo_eapi,)) return repo_eapi def __is_repository_updatable(self): online = self.remote_revision() if online != -1: local = AvailablePackagesRepository.revision( self._repository_id) if (local == online) and (not self.__force): return False return True def __show_repository_information(self): avail_data = self._settings['repositories']['available'] repo_data = avail_data[self._repository_id] self._entropy.output( bold("%s") % ( repo_data['description'] ), importance = 2, level = "info", header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Repository URL")), darkgreen(repo_data['database']),) self._entropy.output( mytxt, importance = 1, level = "info", header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Repository local path")), darkgreen(repo_data['dbpath']),) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # ") ) mytxt = "%s: %s" % (red(_("Repository API")), darkgreen(str(self._repo_eapi)),) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # ") ) def __handle_repository_lock(self): # get database lock unlocked = self._is_repository_unlocked() if not unlocked: mytxt = "%s: %s. %s." % ( bold(_("Attention")), red(_("Repository is being updated")), red(_("Try again in a few minutes")), ) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) return True return False def __append_gpg_signature_to_path(self, path): return path + etpConst['etpgpgextension'] def __ensure_repository_path(self): avail_data = self._settings['repositories']['available'] repo_data = avail_data[self._repository_id] # create dir if it doesn't exist if not os.path.isdir(repo_data['dbpath']): os.makedirs(repo_data['dbpath'], 0o755) const_setup_perms(etpConst['etpdatabaseclientdir'], etpConst['entropygid'], f_perms = 0o644) def __validate_compression_method(self): repo = self._repository_id repo_settings = self._settings['repositories'] dbc_format = repo_settings['available'][repo]['dbcformat'] cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format) if cmethod is None: raise AttributeError("Wrong repository compression method") return cmethod def __remove_repository_files(self): sys_set = self._settings avail_data = sys_set['repositories']['available'] repo_dbpath = avail_data[self._repository_id]['dbpath'] shutil.rmtree(repo_dbpath, True) def __handle_database_download(self, cmethod): def show_repo_locked_message(): mytxt = "%s: %s." % ( bold(_("Attention")), red(_("remote repository got suddenly locked")), ) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) # starting to download mytxt = "%s ..." % (red(_("Downloading repository")),) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t" ) downloaded_item = None down_status = False sig_status = False if self._repo_eapi == 2: down_item = "dbdumplight" down_status = self._download_item(down_item, cmethod, disallow_redirect = True) if down_status: # get GPG file if available sig_status = self._download_item(down_item, cmethod, disallow_redirect = True, get_signature = True) downloaded_item = down_item if not down_status: # fallback to old db self._repo_eapi = 1 down_item = "dblight" if self._developer_repo: # if developer repo mode is enabled, fetch full-blown db down_item = "db" const_debug_write(__name__, "__handle_database_download: developer repo mode enabled") down_status = self._download_item(down_item, cmethod, disallow_redirect = True) if down_status: sig_status = self._download_item(down_item, cmethod, disallow_redirect = True, get_signature = True) downloaded_item = down_item if not down_status: mytxt = "%s: %s." % (bold(_("Attention")), red(_("repository does not exist online")),) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) return down_status, sig_status, downloaded_item def __handle_database_checksum_download(self, cmethod): downitem = 'cklight' if self._developer_repo: downitem = 'dbck' if self._repo_eapi == 2: # EAPI = 2 downitem = 'dbdumplightck' garbage_url, hashfile = self._construct_paths(downitem, cmethod) mytxt = "%s %s %s" % ( red(_("Downloading checksum")), darkgreen(os.path.basename(hashfile)), red("..."), ) # download checksum self._entropy.output( mytxt, importance = 0, level = "info", header = "\t" ) db_down_status = self._download_item(downitem, cmethod, disallow_redirect = True) if not db_down_status and (downitem not in ('cklight', 'dbck',)): # fallback to old method retryitem = 'cklight' if self._developer_repo: retryitem = 'dbck' db_down_status = self._download_item(retryitem, cmethod, disallow_redirect = True) if not db_down_status: mytxt = "%s %s !" % ( red(_("Cannot fetch checksum")), red(_("Cannot verify repository integrity")), ) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) return db_down_status def __verify_file_checksum(self, file_path, md5_checksum_path): with open(md5_checksum_path, "r") as ck_f: md5hash = ck_f.readline().strip() if not md5hash: # invalid !! => [] would cause IndexError return False md5hash = md5hash.split()[0] return entropy.tools.compare_md5(file_path, md5hash) def __verify_database_checksum(self, cmethod = None): sys_settings_repos = self._settings['repositories'] avail_config = sys_settings_repos['available'][self._repository_id] sep = os.path.sep if self._repo_eapi == 1: if self._developer_repo: remote_gb, dbfile = self._construct_paths('db', cmethod) remote_gb, md5file = self._construct_paths('dbck', cmethod) else: remote_gb, dbfile = self._construct_paths('dblight', cmethod) remote_gb, md5file = self._construct_paths('cklight', cmethod) elif self._repo_eapi == 2: remote_gb, dbfile = self._construct_paths('dbdumplight', cmethod) remote_gb, md5file = self._construct_paths('dbdumplightck', cmethod) else: raise AttributeError("EAPI must be = 1 or 2") if not (os.access(md5file, os.R_OK) and os.path.isfile(md5file)): return -1 return self.__verify_file_checksum(dbfile, md5file) def __unpack_downloaded_database(self, down_item, cmethod): rc = 0 path = None sys_set_repos = self._settings['repositories']['available'] repo_data = sys_set_repos[self._repository_id] garbage, myfile = self._construct_paths(down_item, cmethod) if self._repo_eapi in (1, 2,): try: myfunc = getattr(entropy.tools, cmethod[1]) path = myfunc(myfile) # rename path correctly if self._repo_eapi == 1: new_path = os.path.join(os.path.dirname(path), etpConst['etpdatabasefile']) os.rename(path, new_path) path = new_path except (OSError, EOFError): rc = 1 else: mytxt = "invalid EAPI must be = 1 or 2" raise AttributeError(mytxt) if rc == 0: const_setup_file(path, etpConst['entropygid'], 0o644, uid = etpConst['uid']) return rc def __handle_downloaded_database_unpack(self, cmethod): file_to_unpack = etpConst['etpdatabasedump'] if self._repo_eapi == 1: file_to_unpack = etpConst['etpdatabasefile'] elif self._repo_eapi == 2: file_to_unpack = etpConst['etpdatabasedumplight'] mytxt = "%s %s %s" % (red(_("Unpacking database to")), darkgreen(file_to_unpack), red("..."),) self._entropy.output( mytxt, importance = 0, level = "info", header = "\t" ) myitem = 'dblight' if self._repo_eapi == 2: myitem = 'dbdumplight' elif self._developer_repo: myitem = 'db' myrc = self.__unpack_downloaded_database(myitem, cmethod) if myrc != 0: mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")), red(_("Skipping repository")),) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) return False, myitem return True, myitem def __update_repository_revision(self): cur_rev = AvailablePackagesRepository.revision(self._repository_id) repo_data = self._settings['repositories'] db_data = repo_data['available'][self._repository_id] db_data['dbrevision'] = "0" if cur_rev != -1: db_data['dbrevision'] = str(cur_rev) # update repository revision file # self.remote_revision() output must be # written into packages.db.revision for consistency # otherwise WebService sync when WebService is on a separate # server (and uses rsync) doesn't work at its best # self._last_revs rev_file = os.path.join(db_data['dbpath'], etpConst['etpdatabaserevisionfile']) with open(rev_file, "w") as rev_f: rev_f.write(str(self._last_rev) + "\n") rev_f.flush() def __validate_database(self): def tell_error(err): mytxt = "%s: %s" % (darkred(_("Repository is invalid")), repr(err),) self._entropy.output( mytxt, importance = 1, level = "error", header = "\t" ) try: dbconn = self._entropy.open_repository(self._repository_id) except RepositoryError as err: tell_error(err) return False try: dbconn.validate() except SystemDatabaseError as err: tell_error(err) return False return True def __database_indexing(self): # renice a bit, to avoid eating resources old_prio = const_set_nice_level(15) mytxt = red("%s ...") % (_("Indexing Repository metadata"),) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t" ) dbconn = self._entropy.open_repository(self._repository_id) dbconn.createAllIndexes() dbconn.commit(force = True) if self._entropy.installed_repository() is not None: try: # client db can be absent self._entropy.installed_repository().createAllIndexes() except (DatabaseError, OperationalError, IntegrityError,): pass const_set_nice_level(old_prio) def _construct_paths(self, item, cmethod, get_signature = False): if item not in self._supported_download_items: raise AttributeError("Invalid item: %s" % (item,)) items_needing_cmethod = ( "db", "dbck", "dblight", "cklight", "dbdump", "dbdumpck", "dbdumplight", "dbdumplightck", "compck", ) if (item in items_needing_cmethod) and (cmethod is None): mytxt = "For %s, cmethod can't be None" % (item,) raise AttributeError(mytxt) avail_data = self._settings['repositories']['available'] repo_data = avail_data[self._repository_id] repo_db = repo_data['database'] repo_dbpath = repo_data['dbpath'] ec_hash = etpConst['etpdatabasehashfile'] repo_lock_file = etpConst['etpdatabasedownloadlockfile'] notice_board_filename = os.path.basename(repo_data['notice_board']) meta_file = etpConst['etpdatabasemetafilesfile'] meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \ etpConst['etpgpgextension'] md5_ext = etpConst['packagesmd5fileext'] ec_cm2 = None ec_cm3 = None ec_cm4 = None ec_cm5 = None ec_cm6 = None ec_cm7 = None ec_cm8 = None ec_cm9 = None if cmethod is not None: ec_cm2 = etpConst[cmethod[2]] ec_cm3 = etpConst[cmethod[3]] ec_cm4 = etpConst[cmethod[4]] ec_cm5 = etpConst[cmethod[5]] ec_cm6 = etpConst[cmethod[6]] ec_cm7 = etpConst[cmethod[7]] ec_cm8 = etpConst[cmethod[8]] ec_cm9 = etpConst[cmethod[9]] mymap = { 'db': ( "%s/%s" % (repo_db, ec_cm2,), "%s/%s" % (repo_dbpath, ec_cm2,), ), 'dbck': ( "%s/%s" % (repo_db, ec_cm9,), "%s/%s" % (repo_dbpath, ec_cm9,), ), 'dblight': ( "%s/%s" % (repo_db, ec_cm7,), "%s/%s" % (repo_dbpath, ec_cm7,), ), 'dbdump': ( "%s/%s" % (repo_db, ec_cm3,), "%s/%s" % (repo_dbpath, ec_cm3,), ), 'dbdumplight': ( "%s/%s" % (repo_db, ec_cm5,), "%s/%s" % (repo_dbpath, ec_cm5,), ), 'ck': ( "%s/%s" % (repo_db, ec_hash,), "%s/%s" % (repo_dbpath, ec_hash,), ), 'cklight': ( "%s/%s" % (repo_db, ec_cm8,), "%s/%s" % (repo_dbpath, ec_cm8,), ), 'compck': ( "%s/%s%s" % (repo_db, ec_cm2, md5_ext,), "%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,), ), 'dbdumpck': ( "%s/%s" % (repo_db, ec_cm4,), "%s/%s" % (repo_dbpath, ec_cm4,), ), 'dbdumplightck': ( "%s/%s" % (repo_db, ec_cm6,), "%s/%s" % (repo_dbpath, ec_cm6,), ), 'lock': ( "%s/%s" % (repo_db, repo_lock_file,), "%s/%s" % (repo_dbpath, repo_lock_file,), ), 'notice_board': ( repo_data['notice_board'], "%s/%s" % (repo_dbpath, notice_board_filename,), ), 'meta_file': ( "%s/%s" % (repo_db, meta_file,), "%s/%s" % (repo_dbpath, meta_file,), ), 'meta_file_gpg': ( "%s/%s" % (repo_db, meta_file_gpg,), "%s/%s" % (repo_dbpath, meta_file_gpg,), ), } url, path = mymap.get(item) if get_signature: url = self.__append_gpg_signature_to_path(url) path = self.__append_gpg_signature_to_path(path) return url, path def _download_item(self, item, cmethod = None, disallow_redirect = True, get_signature = False): url, filepath = self._construct_paths(item, cmethod, get_signature = get_signature) # to avoid having permissions issues # it's better to remove the file before, # otherwise new permissions won't be written if os.path.isfile(filepath): os.remove(filepath) filepath_dir = os.path.dirname(filepath) if not os.path.isdir(filepath_dir) and not \ os.path.lexists(filepath_dir): os.makedirs(filepath_dir, 0o755) const_setup_perms(filepath_dir, etpConst['entropygid'], f_perms = 0o644) fetcher = self._entropy._url_fetcher( url, filepath, resume = False, disallow_redirect = disallow_redirect ) rc = fetcher.download() if rc in ("-1", "-2", "-3", "-4"): return False const_setup_file(filepath, etpConst['entropygid'], 0o644, uid = etpConst['uid']) return True def _is_repository_unlocked(self): """ Returns whether the repository is remotely locked or not. @return: repository being remotely locked @rtype:bool """ rc = self._download_item("lock", disallow_redirect = True) if rc: # cannot download database return False return True def _standard_items_download(self): repos_data = self._settings['repositories'] repo_data = repos_data['available'][self._repository_id] notice_board = os.path.basename(repo_data['local_notice_board']) db_meta_file = etpConst['etpdatabasemetafilesfile'] db_meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \ etpConst['etpgpgextension'] objects_to_unpack = ("meta_file",) download_items = [ ( "meta_file", db_meta_file, False, "%s %s %s" % ( red(_("Downloading repository metafile")), darkgreen(db_meta_file), red("..."), ) ), ( "meta_file_gpg", db_meta_file_gpg, True, "%s %s %s" % ( red(_("Downloading GPG signature of repository metafile")), darkgreen(db_meta_file_gpg), red("..."), ) ), ( "notice_board", notice_board, True, "%s %s %s" % ( red(_("Downloading Notice Board")), darkgreen(notice_board), red("..."), ) ), ] def my_show_info(txt): self._entropy.output( txt, importance = 0, level = "info", header = "\t", back = True ) def my_show_down_status(message, mytype): self._entropy.output( message, importance = 0, level = mytype, header = "\t" ) def my_show_file_unpack(fp): self._entropy.output( "%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),), header = blue("\t << ") ) def my_show_file_rm(fp): self._entropy.output( "%s: %s" % (darkgreen(_("removed meta file")), purple(fp),), header = blue("\t << ") ) downloaded_files = [] for item, myfile, ignorable, mytxt in download_items: my_show_info(mytxt) mystatus = self._download_item(item, disallow_redirect = True) mytype = 'info' myurl, mypath = self._construct_paths(item, None) # download failed, is it critical? if not mystatus: if ignorable: message = "%s: %s." % (blue(myfile), red(_("not available, it's ok"))) else: mytype = 'warning' message = "%s: %s." % (blue(myfile), darkred(_("not available, not very ok!"))) my_show_down_status(message, mytype) # remove garbage if os.path.isfile(mypath): try: os.remove(mypath) except OSError: continue continue message = "%s: %s." % (blue(myfile), darkgreen(_("available, w00t!"))) my_show_down_status(message, mytype) downloaded_files.append(mypath) if item not in objects_to_unpack: continue if not (os.path.isfile(mypath) and os.access(mypath, os.R_OK)): continue tmpdir = tempfile.mkdtemp() repo_dir = repo_data['dbpath'] try: done = entropy.tools.universal_uncompress(mypath, tmpdir, catch_empty = True) if not done: mytype = 'warning' message = "%s: %s." % (blue(myfile), darkred(_("cannot be unpacked, not very ok!"))) my_show_down_status(message, mytype) continue myfiles_to_move = set(os.listdir(tmpdir)) # exclude files not available by default files_not_found_file = etpConst['etpdatabasemetafilesnotfound'] if files_not_found_file in myfiles_to_move: myfiles_to_move.remove(files_not_found_file) fnf_path = os.path.join(tmpdir, files_not_found_file) if os.path.isfile(fnf_path) and \ os.access(fnf_path, os.R_OK): with open(fnf_path, "r") as f: f_nf = [x.strip() for x in f.readlines()] for myfile in f_nf: myfile = os.path.basename(myfile) # avoid lamerz myfpath = os.path.join(repo_dir, myfile) if os.path.isfile(myfpath) and \ os.access(myfpath, os.W_OK): try: os.remove(myfpath) my_show_file_rm(myfile) except OSError: continue for myfile in sorted(myfiles_to_move): from_mypath = os.path.join(tmpdir, myfile) to_mypath = os.path.join(repo_dir, myfile) try: os.rename(from_mypath, to_mypath) my_show_file_unpack(myfile) except OSError: # try non atomic way try: shutil.copy2(from_mypath, to_mypath) my_show_file_unpack(myfile) os.remove(from_mypath) except (shutil.Error, IOError, OSError,): continue continue const_setup_file(to_mypath, etpConst['entropygid'], 0o644, uid = etpConst['uid']) finally: shutil.rmtree(tmpdir, True) repo_r = AvailablePackagesRepository.revision(self._repository_id) mytxt = "%s: %s" % ( red(_("Repository revision")), bold(str(repo_r)), ) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t" ) return downloaded_files def _check_downloaded_database(self, cmethod): dbitem = 'dblight' if self._repo_eapi == 2: dbitem = 'dbdumplight' elif self._developer_repo: dbitem = 'db' garbage, dbfilename = self._construct_paths(dbitem, cmethod) # verify checksum mytxt = "%s %s %s" % ( red(_("Checking downloaded repository")), darkgreen(os.path.basename(dbfilename)), red("..."), ) self._entropy.output( mytxt, importance = 0, back = True, level = "info", header = "\t" ) db_status = self.__verify_database_checksum(cmethod) if db_status == -1: mytxt = "%s. %s !" % ( red(_("Cannot open digest")), red(_("Cannot verify repository integrity")), ) self._entropy.output( mytxt, importance = 1, level = "warning", header = "\t" ) elif db_status: mytxt = "%s: %s" % ( red(_("Downloaded repository status")), bold(_("OK")), ) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t" ) else: mytxt = "%s: %s" % ( red(_("Downloaded repository status")), darkred(_("ERROR")), ) self._entropy.output( mytxt, importance = 1, level = "error", header = "\t" ) mytxt = "%s. %s" % ( red(_("An error occured while checking repository integrity")), red(_("Giving up")), ) self._entropy.output( mytxt, importance = 1, level = "error", header = "\t" ) return 1 return 0 def _install_gpg_key_if_available(self): my_repos = self._settings['repositories'] avail_data = my_repos['available'] repo_data = avail_data[self._repository_id] gpg_path = repo_data['gpg_pubkey'] if not (os.path.isfile(gpg_path) and os.access(gpg_path, os.R_OK)): return False # gpg key not available def do_warn_user(fingerprint): mytxt = purple(_("Make sure to verify the imported key and set an appropriate trust level")) self._entropy.output( mytxt + ":", level = "warning", header = "\t" ) mytxt = brown("gpg --homedir '%s' --edit-key '%s'" % ( etpConst['etpclientgpgdir'], fingerprint,) ) self._entropy.output( "$ " + mytxt, level = "warning", header = "\t" ) try: repo_sec = self._entropy.RepositorySecurity() except RepositorySecurity.GPGError: mytxt = "%s," % ( purple(_("This repository suports GPG-signed packages")), ) self._entropy.output( mytxt, level = "warning", header = "\t" ) mytxt = purple(_("you may want to install GnuPG to take advantage of this feature")) self._entropy.output( mytxt, level = "warning", header = "\t" ) return False # GPG not available pk_expired = False try: pk_avail = repo_sec.is_pubkey_available(self._repository_id) except repo_sec.KeyExpired: pk_avail = False pk_expired = True if pk_avail: tmp_dir = tempfile.mkdtemp() repo_tmp_sec = self._entropy.RepositorySecurity( keystore_dir = tmp_dir) # try to install and get fingerprint try: downloaded_key_fp = repo_tmp_sec.install_key( self._repository_id, gpg_path) except RepositorySecurity.GPGError: downloaded_key_fp = None fingerprint = repo_sec.get_key_metadata( self._repository_id)['fingerprint'] shutil.rmtree(tmp_dir, True) if downloaded_key_fp != fingerprint and \ (downloaded_key_fp is not None): mytxt = "%s: %s !!!" % ( purple(_("GPG key changed for")), bold(self._repository_id), ) self._entropy.output( mytxt, level = "warning", header = "\t" ) mytxt = "[%s => %s]" % ( darkgreen(fingerprint), purple(downloaded_key_fp), ) self._entropy.output( mytxt, level = "warning", header = "\t" ) do_warn_user(downloaded_key_fp) else: mytxt = "%s: %s" % ( purple(_("GPG key already installed for")), bold(self._repository_id), ) self._entropy.output( mytxt, level = "info", header = "\t" ) do_warn_user(fingerprint) return True # already installed elif pk_expired: mytxt = "%s: %s" % ( purple(_("GPG key EXPIRED for repository")), bold(self._repository_id), ) self._entropy.output( mytxt, level = "warning", header = "\t" ) # actually install mytxt = "%s: %s" % ( purple(_("Installing GPG key for repository")), brown(self._repository_id), ) self._entropy.output( mytxt, level = "info", header = "\t", back = True ) try_ignore = False while True: try: fingerprint = repo_sec.install_key(self._repository_id, gpg_path) except RepositorySecurity.NothingImported as err: if try_ignore: mytxt = "%s: %s" % ( darkred(_("Error during GPG key installation")), err, ) self._entropy.output( mytxt, level = "error", header = "\t" ) return False self._entropy.output( purple(_("GPG key seems already installed but not properly recorded, resetting")), level = "warning", header = "\t" ) target_fingerprint = repo_sec.get_key_fingerprint(gpg_path) if target_fingerprint is not None: # kill it, this is usually caused by shadow repos # like sabayon-weekly dead_repository_ids = set() for _repository_id, key_meta in repo_sec.get_keys().items(): if key_meta['fingerprint'] == target_fingerprint: dead_repository_ids.add(_repository_id) for _repository_id in dead_repository_ids: try: repo_sec.delete_pubkey(_repository_id) except KeyError: # wtf, fault tolerance pass try_ignore = True continue except RepositorySecurity.GPGError as err: mytxt = "%s: %s" % ( darkred(_("Error during GPG key installation")), err, ) self._entropy.output( mytxt, level = "error", header = "\t" ) return False break mytxt = "%s: %s" % ( purple(_("Successfully installed GPG key for repository")), brown(self._repository_id), ) self._entropy.output( mytxt, level = "info", header = "\t" ) mytxt = "%s: %s" % ( darkgreen(_("Fingerprint")), bold(fingerprint), ) self._entropy.output( mytxt, level = "info", header = "\t" ) do_warn_user(fingerprint) return True def _gpg_verify_downloaded_files(self, downloaded_files): try: repo_sec = self._entropy.RepositorySecurity() except RepositorySecurity.GPGServiceNotAvailable: # wtf! it was available a while ago! return 0 # GPG not available gpg_sign_ext = etpConst['etpgpgextension'] sign_files = [x for x in downloaded_files if \ x.endswith(gpg_sign_ext)] sign_files = [x for x in sign_files if os.path.isfile(x) and \ os.access(x, os.R_OK)] to_be_verified = [] for sign_path in sign_files: target_path = sign_path[:-len(gpg_sign_ext)] if os.path.isfile(target_path) and \ os.access(target_path, os.R_OK): to_be_verified.append((target_path, sign_path,)) gpg_rc = 0 for target_path, sign_path in to_be_verified: file_name = os.path.basename(target_path) mytxt = "%s: %s ..." % ( darkgreen(_("Verifying GPG signature of")), brown(file_name), ) self._entropy.output( mytxt, level = "info", header = blue("\t@@ "), back = True ) is_valid, err_msg = repo_sec.verify_file(self._repository_id, target_path, sign_path) if is_valid: mytxt = "%s: %s" % ( darkgreen(_("Verified GPG signature of")), brown(file_name), ) self._entropy.output( mytxt, level = "info", header = blue("\t@@ ") ) else: mytxt = "%s: %s" % ( darkred(_("Error during GPG verification of")), file_name, ) self._entropy.output( mytxt, level = "error", header = "\t%s " % (bold("!!!"),) ) mytxt = "%s: %s" % ( purple(_("It could mean a potential security risk")), err_msg, ) self._entropy.output( mytxt, level = "error", header = "\t%s " % (bold("!!!"),) ) gpg_rc = 1 return gpg_rc def _handle_webserv_database_sync(self): repo_db = None try: repo_db = self.__get_webserv_local_database() if repo_db is None: raise AttributeError() return self.__handle_webserv_database_sync(repo_db) except (DatabaseError, IntegrityError, OperationalError, AttributeError,): return False finally: if repo_db is not None: repo_db.close() return False def __handle_webserv_database_sync(self, mydbconn): try: webserv = self._webservice except WebService.UnsupportedService as err: const_debug_write(__name__, "__handle_webserv_database_sync: error: %s" % (err,)) return False try: myidpackages = mydbconn.listAllPackageIds() except (DatabaseError, IntegrityError, OperationalError,): return False added_ids, removed_ids = self.__get_webserv_database_differences( webserv, myidpackages) if (None in (added_ids, removed_ids)) or \ (not added_ids and not removed_ids and self.__force): # nothing to sync, it seems, if force is True, fallback to EAPI2 return False threshold = 500 # is it worth it? if len(added_ids) > threshold: mytxt = "%s: %s (%s: %s/%s)" % ( blue(_("Web Service")), darkred(_("skipping differential sync")), brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)), ) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # "), ) return False count = 0 chunk_size = RepositoryWebService.MAXIMUM_PACKAGE_REQUEST_SIZE added_segments = [] mytmp = [] for package_id in added_ids: count += 1 mytmp.append(package_id) if count % chunk_size == 0: added_segments.append(mytmp[:]) del mytmp[:] if mytmp: added_segments.append(mytmp[:]) def _do_fetch(fetch_sts_map, segment, count, maxcount): try: try: pkg_meta = webserv.get_packages_metadata(segment) except WebService.WebServiceException as err: const_debug_write(__name__, "__handle_webserv_database_sync: error: %s" % (err,)) mytxt = "%s: %s" % ( blue(_("Web Service communication error")), err, ) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t", count = (count, maxcount,) ) fetch_sts_map['error'] = True return except KeyboardInterrupt: const_debug_write(__name__, "__handle_webserv_database_sync: keyboard interrupt") fetch_sts_map['error'] = True return if not pkg_meta: const_debug_write(__name__, "__handle_webserv_database_sync: empty data: %s" % ( pkg_meta,)) self._entropy.output( _("Web Service data error"), importance = 1, level = "info", header = "\t", count = (count, maxcount,) ) fetch_sts_map['error'] = True return try: for package_id, pkg_data in pkg_meta.items(): dumpobj( "%s%s" % (self.WEBSERV_CACHE_ID, package_id,), pkg_data, ignore_exceptions = False ) except (IOError, EOFError, OSError,) as e: mytxt = "%s: %s: %s." % ( blue(_("Local status")), darkred("Error storing data"), e, ) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t", count = (count, maxcount,) ) fetch_sts_map['error'] = True return return finally: fetch_sts_map['sem'].release() # do not exagerate or you're going to need a way to block # further requests as long as some threads are still running # to avoid timeout errors max_threads = 4 fetch_sts_map = { 'sem': threading.Semaphore(max_threads), 'error': False, } # fetch and store count = 0 maxcount = len(added_segments) product = self._settings['repositories']['product'] segment = None threads = [] for segment in added_segments: count += 1 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) self._entropy.output( mytxt, importance = 0, level = "info", header = "\t", back = True, count = (count, maxcount,) ) fetch_sts_map['sem'].acquire() if len(threads) >= max_threads: const_debug_write(__name__, purple("joining all the parallel threads")) # give them the chance to complete # since long delays on socket could cause timeouts for th in threads: th.join() const_debug_write(__name__, purple("parallel threads joined")) del threads[:] if fetch_sts_map['error']: return None th = ParallelTask(_do_fetch, fetch_sts_map, segment, count, maxcount) th.daemon = True th.start() threads.append(th) for th in threads: th.join() if fetch_sts_map['error']: return None del added_segments # get repository metadata repo_metadata = self.__get_webserv_repository_metadata() # this gives us the "checksum" data too if not repo_metadata: mytxt = "%s: %s" % ( blue(_("Web Service status")), darkred(_("cannot fetch repository metadata")), ) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # "), ) return None # update treeupdates try: mydbconn.setRepositoryUpdatesDigest(self._repository_id, repo_metadata['treeupdates_digest']) mydbconn.bumpTreeUpdatesActions( repo_metadata['treeupdates_actions']) except (Error,): mytxt = "%s: %s" % ( blue(_("Web Service status")), darkred(_("cannot update treeupdates data")), ) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # "), ) return None # update package sets try: mydbconn.clearPackageSets() mydbconn.insertPackageSets(repo_metadata['sets']) except (Error,): mytxt = "%s: %s" % ( blue(_("Web Service status")), darkred(_("cannot update package sets data")), ) self._entropy.output( mytxt, importance = 0, level = "info", header = blue(" # "), ) return None # now that we have all stored, add for idpackage in added_ids: mydata = self._cacher.pop("%s%s" % (self.WEBSERV_CACHE_ID, idpackage,)) if mydata is None: mytxt = "%s: %s" % ( blue(_("Fetch error on segment while adding")), darkred(str(segment)), ) self._entropy.output( mytxt, importance = 1, level = "warning", header = " " ) return False mytxt = "%s %s" % ( darkgreen("++"), teal(mydata['atom']), ) self._entropy.output( mytxt, importance = 0, level = "info", header = " ", back = (not etpUi['verbose']) ) try: mydbconn.addPackage( mydata, revision = mydata['revision'], package_id = idpackage, do_commit = False, formatted_content = True ) except (Error,) as err: if etpUi['debug']: entropy.tools.print_traceback() self._entropy.output("%s: %s" % ( blue(_("repository error while adding packages")), err,), importance = 1, level = "warning", header = " " ) return False # now remove # preload atoms names to improve speed during removePackage atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids) for idpackage in removed_ids: myatom = atoms_map.get(idpackage) mytxt = "%s %s" % ( darkred("--"), purple(str(myatom)),) self._entropy.output( mytxt, importance = 0, level = "info", header = " ", back = (not etpUi['verbose']) ) try: mydbconn.removePackage(idpackage, do_cleanup = False, do_commit = False) except (Error,): self._entropy.output( blue(_("repository error while removing packages")), importance = 1, level = "warning", header = " " ) return False mydbconn.commit() mydbconn.clearCache() # now verify if both checksums match result = False mychecksum = mydbconn.checksum(do_order = True, strict = False, strings = True, include_signatures = True) if repo_metadata['checksum'] == mychecksum: result = True else: self._entropy.output( blue(_("Repository checksum doesn't match remote.")), importance = 0, level = "info", header = "\t", ) mytxt = "%s: %s" % (_('local'), mychecksum,) self._entropy.output( mytxt, importance = 0, level = "info", header = "\t", ) mytxt = "%s: %s" % (_('remote'), repo_metadata['checksum'],) self._entropy.output( mytxt, importance = 0, level = "info", header = "\t", ) return result def remote_revision(self): if self._repo_eapi == 3: # ask WebService then revision = self.__get_webserv_repository_revision() if revision is not None: try: revision = int(revision) except ValueError: revision = None if revision is not None: self._last_rev = revision return revision # otherwise, fallback to previous EAPI self._repo_eapi -= 1 avail_data = self._settings['repositories']['available'] repo_data = avail_data[self._repository_id] url = repo_data['database'] + "/" + \ etpConst['etpdatabaserevisionfile'] status = entropy.tools.get_remote_data(url, timeout = self.__big_sock_timeout) if status: status = status[0].strip() try: status = int(status) except ValueError: status = -1 else: status = -1 self._last_rev = status return status def update(self): # disallow unprivileged update if not entropy.tools.is_root(): raise PermissionDenied( "cannot update repository as unprivileged user") self.__show_repository_information() # this calls writes self._last_rev which is used to write back # updated repository revision, do not remove! updatable = self.__is_repository_updatable() if not self.__force: if not updatable: mytxt = "%s: %s." % (bold(_("Attention")), red(_("repository is already up to date")),) self._entropy.output( mytxt, importance = 1, level = "info", header = "\t" ) return EntropyRepositoryBase.REPOSITORY_ALREADY_UPTODATE locked = self.__handle_repository_lock() if locked: return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE # clear database interface cache belonging to this repository self.__ensure_repository_path() # dealing with EAPI # setting some vars db_checksum_down_status = False do_db_update_transfer = False rc = 0 my_repos = self._settings['repositories'] avail_data = my_repos['available'] repo_data = avail_data[self._repository_id] # some variables dumpfile = os.path.join(repo_data['dbpath'], etpConst['etpdatabasedumplight']) dbfile = os.path.join(repo_data['dbpath'], etpConst['etpdatabasefile']) dbfile_old = dbfile+".sync" cmethod = self.__validate_compression_method() while True: downloaded_db_item = None sig_down_status = False db_checksum_down_status = False if self._repo_eapi < 3: down_status, sig_down_status, downloaded_db_item = \ self.__handle_database_download(cmethod) if not down_status: return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE db_checksum_down_status = \ self.__handle_database_checksum_download(cmethod) break elif self._repo_eapi == 3 and not \ (os.path.isfile(dbfile) and os.access(dbfile, os.W_OK)): do_db_update_transfer = None self._repo_eapi -= 1 continue elif self._repo_eapi == 3: status = False try: status = self._handle_webserv_database_sync() except: # avoid broken entries, deal with every exception self.__remove_repository_files() raise if status is None: # remote db not available anymore ? time.sleep(5) locked = self.__handle_repository_lock() if locked: return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE do_db_update_transfer = None self._repo_eapi -= 1 continue elif not status: # (status == False) # set to none and completely skip database alignment do_db_update_transfer = None self._repo_eapi -= 1 continue break downloaded_files = self._standard_items_download() # also add db file to downloaded item # and md5 check repository if downloaded_db_item is not None: durl, dpath = self._construct_paths(downloaded_db_item, cmethod) downloaded_files.append(dpath) if sig_down_status: d_sig_path = self.__append_gpg_signature_to_path(dpath) downloaded_files.append(d_sig_path) # 1. we're always in EAPI1 or 2 here # 2. new policy, always deny repository if # its database checksum cannot be fetched if not db_checksum_down_status: # delete all self.__remove_repository_files() return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE rc = self._check_downloaded_database(cmethod) if rc != 0: # delete all self.__remove_repository_files() return EntropyRepositoryBase.REPOSITORY_CHECKSUM_ERROR # GPG pubkey install hook if self._gpg_feature: gpg_available = self._install_gpg_key_if_available() if gpg_available: gpg_rc = self._gpg_verify_downloaded_files(downloaded_files) # Now we can unpack files_to_remove = [] if self._repo_eapi in (1, 2,): # if do_db_update_transfer == False and not None if (do_db_update_transfer is not None) and not \ do_db_update_transfer: if os.access(dbfile, os.R_OK | os.W_OK) and \ os.path.isfile(dbfile): try: os.rename(dbfile, dbfile_old) do_db_update_transfer = True except OSError: do_db_update_transfer = False unpack_status, unpacked_item = \ self.__handle_downloaded_database_unpack(cmethod) if not unpack_status: # delete all self.__remove_repository_files() return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR unpack_url, unpack_path = self._construct_paths(unpacked_item, cmethod) files_to_remove.append(unpack_path) # re-validate if not os.path.isfile(dbfile): do_db_update_transfer = False elif os.path.isfile(dbfile) and not do_db_update_transfer and \ (self._repo_eapi != 1): os.remove(dbfile) if self._repo_eapi == 2: rc = self.__eapi2_inject_downloaded_dump(dumpfile, dbfile, cmethod) if do_db_update_transfer: self.__eapi1_eapi2_databases_alignment(dbfile, dbfile_old) if self._repo_eapi == 2: # remove the dump files_to_remove.append(dumpfile) if rc != 0: # delete all self.__remove_repository_files() files_to_remove.append(dbfile_old) for path in files_to_remove: try: os.remove(path) except OSError: continue return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR # make sure that all the repository files are stored with proper # permissions to avoid possible XSS and trust boundary problems. downloaded_files.append(dbfile) for downloaded_file in sorted(set(downloaded_files)): if os.path.isfile(downloaded_file) and \ os.access(downloaded_file, os.W_OK | os.R_OK): const_setup_file(downloaded_file, etpConst['entropygid'], 0o644, uid = etpConst['uid']) # remove garbage left around for path in files_to_remove: try: os.remove(path) except OSError: continue valid = self.__validate_database() if not valid: # repository failed validation return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR self.__update_repository_revision() if self._entropy._indexing: self.__database_indexing() try: spm_class = self._entropy.Spm_class() spm_class.entropy_client_post_repository_update_hook( self._entropy, self._repository_id) except Exception as err: entropy.tools.print_traceback() mytxt = "%s: %s" % ( blue(_("Configuration files update error, not critical, continuing")), err, ) self._entropy.output(mytxt, importance = 0, level = "info", header = blue(" # "),) # remove garbage if os.access(dbfile_old, os.R_OK) and os.path.isfile(dbfile_old): os.remove(dbfile_old) return EntropyRepositoryBase.REPOSITORY_UPDATED_OK _CL_PLUGIN_ID = etpConst['system_settings_plugins_ids']['client_plugin'] class MaskableRepository(EntropyRepositoryBase): """ Objects inheriting from this class support package masking. A masked package is a package that is not visible to user and thus not selectable in dependency calculation and also not directly installable. The only repositories that need to support the feature are those containing installable packages, like AvailablePackagesRepository. """ _MASK_FILTER_CACHE_ID = EntropyCacher.CACHE_IDS['mask_filter'] def _mask_filter_fetch_cache(self, package_id): if self._caching: return loadobj("%s/%s/%s" % ( MaskableRepository._MASK_FILTER_CACHE_ID, self.name, package_id,)) def _mask_filter_store_cache(self, package_id, value): if self._caching: dumpobj("%s/%s/%s" % (MaskableRepository._MASK_FILTER_CACHE_ID, self.name, package_id,), value) def _maskFilter_live(self, package_id): ref = self._settings['pkg_masking_reference'] if (package_id, self.name) in \ self._settings['live_packagemasking']['mask_matches']: # do not cache this return -1, ref['user_live_mask'] elif (package_id, self.name) in \ self._settings['live_packagemasking']['unmask_matches']: return package_id, ref['user_live_unmask'] def _maskFilter_user_package_mask(self, package_id, live): mykw = "%smask_ids" % (self.name,) user_package_mask_ids = self._settings.get(mykw) if not isinstance(user_package_mask_ids, (list, set, frozenset)): user_package_mask_ids = set() for atom in self._settings['mask']: atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom) if repository_ids is not None: if self.name not in repository_ids: # then the mask doesn't involve us continue # check if @repository is specified matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue user_package_mask_ids |= set(matches) self._settings[mykw] = user_package_mask_ids if package_id in user_package_mask_ids: # sorry, masked ref = self._settings['pkg_masking_reference'] myr = ref['user_package_mask'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = -1, myr except KeyError: # system settings client plugin not found pass return -1, myr def _maskFilter_user_package_unmask(self, package_id, live): # see if we can unmask by just lookin into user # package.unmask stuff -> self._settings['unmask'] mykw = "%sunmask_ids" % (self.name,) user_package_unmask_ids = self._settings.get(mykw) if not isinstance(user_package_unmask_ids, (list, set, frozenset)): user_package_unmask_ids = set() for atom in self._settings['unmask']: atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom) if repository_ids is not None: if self.name not in repository_ids: # then the mask doesn't involve us continue matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue user_package_unmask_ids |= set(matches) self._settings[mykw] = user_package_unmask_ids if package_id in user_package_unmask_ids: ref = self._settings['pkg_masking_reference'] myr = ref['user_package_unmask'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr except KeyError: # system settings client plugin not found pass return package_id, myr def _maskFilter_packages_db_mask(self, package_id, live): # check if repository packages.db.mask needs it masked repos_mask = {} client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin'] client_settings = self._settings.get(client_plg_id, {}) if client_settings: repos_mask = client_settings['repositories']['mask'] repomask = repos_mask.get(self.name) if isinstance(repomask, (list, set, frozenset)): # first, seek into generic masking, all branches # (below) avoid issues with repository names mask_repo_id = "%s_ids@@:of:%s" % (self.name, self.name,) repomask_ids = repos_mask.get(mask_repo_id) if not isinstance(repomask_ids, set): repomask_ids = set() for atom in repomask: matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue repomask_ids |= set(matches) repos_mask[mask_repo_id] = repomask_ids if package_id in repomask_ids: ref = self._settings['pkg_masking_reference'] myr = ref['repository_packages_db_mask'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ -1, myr except KeyError: # system settings client plugin not found pass return -1, myr def _maskFilter_package_license_mask(self, package_id, live): if not self._settings['license_mask']: return mylicenses = self.retrieveLicense(package_id) mylicenses = mylicenses.strip().split() lic_mask = self._settings['license_mask'] for mylicense in mylicenses: if mylicense not in lic_mask: continue ref = self._settings['pkg_masking_reference'] myr = ref['user_license_mask'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = -1, myr except KeyError: # system settings client plugin not found pass return -1, myr def _maskFilter_keyword_mask(self, package_id, live): # WORKAROUND for buggy entries # ** is fine then # TODO: remove this before 31-12-2011 mykeywords = self.retrieveKeywords(package_id) if mykeywords == set([""]): mykeywords = set(['**']) mask_ref = self._settings['pkg_masking_reference'] # firstly, check if package keywords are in etpConst['keywords'] # (universal keywords have been merged from package.keywords) same_keywords = etpConst['keywords'] & mykeywords if same_keywords: myr = mask_ref['system_keyword'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr except KeyError: # system settings client plugin not found pass return package_id, myr # if we get here, it means we didn't find mykeywords # in etpConst['keywords'] # we need to seek self._settings['keywords'] # seek in repository first keyword_repo = self._settings['keywords']['repositories'] for keyword in tuple(keyword_repo.get(self.name, {}).keys()): if keyword not in mykeywords: continue keyword_data = keyword_repo[self.name].get(keyword) if not keyword_data: continue if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok myr = mask_ref['user_repo_package_keywords_all'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr except KeyError: # system settings client plugin not found pass return package_id, myr kwd_key = "%s_ids" % (keyword,) keyword_data_ids = keyword_repo[self.name].get(kwd_key) if not isinstance(keyword_data_ids, set): keyword_data_ids = set() for atom in keyword_data: matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue keyword_data_ids |= matches keyword_repo[self.name][kwd_key] = keyword_data_ids if package_id in keyword_data_ids: myr = mask_ref['user_repo_package_keywords'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr except KeyError: # system settings client plugin not found pass return package_id, myr keyword_pkg = self._settings['keywords']['packages'] # if we get here, it means we didn't find a match in repositories # so we scan packages, last chance for keyword in keyword_pkg.keys(): # use .keys() because keyword_pkg gets modified during iteration # first of all check if keyword is in mykeywords if keyword not in mykeywords: continue keyword_data = keyword_pkg.get(keyword) if not keyword_data: continue kwd_key = "%s_ids" % (keyword,) keyword_data_ids = keyword_pkg.get(self.name+kwd_key) if not isinstance(keyword_data_ids, (list, set)): keyword_data_ids = set() for atom in keyword_data: # match atom matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue keyword_data_ids |= matches keyword_pkg[self.name+kwd_key] = keyword_data_ids if package_id in keyword_data_ids: # valid! myr = mask_ref['user_package_keywords'] try: cl_data = self._settings[_CL_PLUGIN_ID] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr except KeyError: # system settings client plugin not found pass return package_id, myr ## if we get here, it means that pkg it keyword masked ## and we should look at the very last resort, per-repository ## package keywords # check if repository contains keyword unmasking data cl_data = self._settings.get(_CL_PLUGIN_ID) if cl_data is None: # SystemSettings Entropy Client plugin not available return # let's see if something is available in repository config repo_keywords = cl_data['repositories']['repos_keywords'].get( self.name) if repo_keywords is None: # nopers, sorry! return # check universal keywords same_keywords = repo_keywords.get('universal') & mykeywords if same_keywords: # universal keyword matches! myr = mask_ref['repository_packages_db_keywords'] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr return package_id, myr ## if we get here, it means that even universal masking failed ## and we need to look at per-package settings repo_settings = repo_keywords.get('packages') if not repo_settings: # it's empty, not worth checking return cached_key = "packages_ids" keyword_data_ids = repo_keywords.get(cached_key) if not isinstance(keyword_data_ids, dict): # create cache keyword_data_ids = {} for atom, values in repo_settings.items(): matches, r = self.atomMatch(atom, multiMatch = True, maskFilter = False) if r != 0: continue for match in matches: obj = keyword_data_ids.setdefault(match, set()) obj.update(values) repo_keywords[cached_key] = keyword_data_ids pkg_keywords = keyword_data_ids.get(package_id, set()) if "**" in pkg_keywords: same_keywords = True else: same_keywords = pkg_keywords & etpConst['keywords'] if same_keywords: # found! this pkg is not masked, yay! myr = mask_ref['repository_packages_db_keywords'] validator_cache = cl_data['masking_validation']['cache'] validator_cache[(package_id, self.name, live)] = \ package_id, myr return package_id, myr def maskFilter(self, package_id, live = True): """ Reimplemented from EntropyRepositoryBase """ validator_cache = self._settings.get(_CL_PLUGIN_ID, {}).get( 'masking_validation', {}).get('cache', {}) cached = validator_cache.get((package_id, self.name, live)) if cached is not None: return cached # use on-disk cache? cached = self._mask_filter_fetch_cache(package_id) if cached is not None: return cached # avoid memleaks if len(validator_cache) > 100000: validator_cache.clear() if live: data = self._maskFilter_live(package_id) if data: return data data = self._maskFilter_user_package_mask(package_id, live) if data: self._mask_filter_store_cache(package_id, data) return data data = self._maskFilter_user_package_unmask(package_id, live) if data: self._mask_filter_store_cache(package_id, data) return data data = self._maskFilter_packages_db_mask(package_id, live) if data: self._mask_filter_store_cache(package_id, data) return data data = self._maskFilter_package_license_mask(package_id, live) if data: self._mask_filter_store_cache(package_id, data) return data data = self._maskFilter_keyword_mask(package_id, live) if data: self._mask_filter_store_cache(package_id, data) return data # holy crap, can't validate myr = self._settings['pkg_masking_reference']['completely_masked'] validator_cache[(package_id, self.name, live)] = -1, myr self._mask_filter_store_cache(package_id, data) return -1, myr class AvailablePackagesRepository(CachedRepository, MaskableRepository): """ This class represents the available packages repository and is a direct subclass of EntropyRepository. It implements the update() method in order to make possible to update the repository. """ def __init__(self, *args, **kwargs): EntropyRepository.__init__(self, *args, **kwargs) # ensure proper repository file permissions if entropy.tools.is_root() and os.path.isfile(self._db_path): const_setup_file(self._db_path, etpConst['entropygid'], 0o644, uid = etpConst['uid']) @staticmethod def update(entropy_client, repository_id, force, gpg): """ Reimplemented from EntropyRepositoryBase """ try: return AvailablePackagesRepositoryUpdater(entropy_client, repository_id, force, gpg).update() except KeyError: return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE @staticmethod def revision(repository_id): """ Reimplemented from EntropyRepositoryBase """ db_data = SystemSettings()['repositories']['available'][repository_id] fname = os.path.join(db_data['dbpath'], etpConst['etpdatabaserevisionfile']) revision = -1 if os.path.isfile(fname) and os.access(fname, os.R_OK): with open(fname, "r") as f: try: read_data = f.readline().strip() revision = int(read_data) except (OSError, IOError, ValueError,): pass return revision @staticmethod def remote_revision(repository_id): """ Reimplemented from EntropyRepositoryBase """ return AvailablePackagesRepositoryUpdater(TextInterface(), repository_id, False, False).remote_revision() def handlePackage(self, pkg_data, forcedRevision = -1, formattedContent = False): """ Reimplemented from EntropyRepository """ raise PermissionDenied( "cannot execute handlePackage on this repository") def addPackage(self, pkg_data, revision = -1, package_id = None, do_commit = True, formatted_content = False): """ Reimplemented from EntropyRepository """ raise PermissionDenied( "cannot execute addPackage on this repository") def removePackage(self, package_id, do_cleanup = True, do_commit = True, from_add_package = False): """ Reimplemented from EntropyRepository """ raise PermissionDenied( "cannot execute removePackage on this repository") def clearCache(self): # clear package masking filter cl_data = self._settings.get(_CL_PLUGIN_ID, {}) cl_data.get('masking_validation', {}).get('cache', {}).clear() EntropyRepository.clearCache(self) class GenericRepository(CachedRepository, MaskableRepository): """ This class represents a generic packages repository and is a direct subclass of EntropyRepository. Even GenericRepository is a CachedRepository because its object could get cached by 3rd party. Actually, we require this because our installed packages repository could end up being a GenericRepository, when running in fail-safe mode. """ def handlePackage(self, pkg_data, forcedRevision = -1, formattedContent = False): """ Reimplemented from EntropyRepository. It is supposed that a generic repository should not support handlePackage. You can override this (at your own risk) by setting the "override_handlePackage" property to True. In this case, a generic addPackage() call is issued. """ override = getattr(self, 'override_handlePackage', False) if not override: raise PermissionDenied( "cannot execute handlePackage on this repository") return self.addPackage(pkg_data, revision = forcedRevision, formatted_content = formattedContent) def maskFilter(self, package_id, live = True): """ Reimplemented from EntropyRepository. It is supposed that a generic repository doesn't support package masking. You can override this by setting the "enable_mask_filter" to True. """ enabled = getattr(self, 'enable_mask_filter', False) if not enabled: return package_id, 0 return MaskableRepository.maskFilter(self, package_id, live = live)